From e1cb675c04392bcc799cc58bc19120904828b01f Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Tue, 9 Nov 2021 17:31:53 +0000
Subject: [PATCH 01/24] Bump version to 1.5dev

---
 CHANGELOG.md    | 4 ++++
 nextflow.config | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b89d9ae..f3e67547 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,10 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unpublished Version / DEV]
+
+### Enhancements & fixes
+
 ## [[1.4](https://github.com/nf-core/fetchngs/releases/tag/1.4)] - 2021-11-09
 
 ### Enhancements & fixes
diff --git a/nextflow.config b/nextflow.config
index 1fb304d3..ea68a9b1 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -146,7 +146,7 @@ manifest {
     description     = 'Pipeline to fetch metadata and raw FastQ files from public databases'
     mainScript      = 'main.nf'
     nextflowVersion = '!>=21.04.0'
-    version         = '1.4'
+    version         = '1.5dev'
 }
 
 // Load modules.config for DSL2 module specific options

From 08d7c1e245ef726ed07b9c5ad60846d1030f494e Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 09:46:04 +0000
Subject: [PATCH 02/24] Install updated syntax module from nf-core/modules

---
 modules/local/dumpsoftwareversions.nf         | 96 -------------------
 .../custom/dumpsoftwareversions/main.nf       | 21 ++++
 .../custom/dumpsoftwareversions/meta.yml      | 34 +++++++
 .../templates/dumpsoftwareversions.py         | 89 +++++++++++++++++
 .../modules/sratools/fasterqdump/main.nf}     | 31 ++----
 .../modules/sratools/fasterqdump/meta.yml     | 42 ++++++++
 .../modules/sratools/prefetch/main.nf}        |  7 +-
 .../modules/sratools/prefetch/meta.yml        | 43 +++++++++
 8 files changed, 238 insertions(+), 125 deletions(-)
 delete mode 100644 modules/local/dumpsoftwareversions.nf
 create mode 100644 modules/nf-core/modules/custom/dumpsoftwareversions/main.nf
 create mode 100644 modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml
 create mode 100644 modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py
 rename modules/{local/sratools_fasterqdump.nf => nf-core/modules/sratools/fasterqdump/main.nf} (61%)
 create mode 100644 modules/nf-core/modules/sratools/fasterqdump/meta.yml
 rename modules/{local/sratools_prefetch.nf => nf-core/modules/sratools/prefetch/main.nf} (90%)
 create mode 100644 modules/nf-core/modules/sratools/prefetch/meta.yml

diff --git a/modules/local/dumpsoftwareversions.nf b/modules/local/dumpsoftwareversions.nf
deleted file mode 100644
index 69ebcbaa..00000000
--- a/modules/local/dumpsoftwareversions.nf
+++ /dev/null
@@ -1,96 +0,0 @@
-
-process DUMPSOFTWAREVERSIONS {
-    label 'process_low'
-
-    // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container
-    conda (params.enable_conda ? "bioconda::multiqc=1.11" : null)
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0' :
-        'quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0' }"
-
-    input:
-    path versions
-
-    output:
-    path "software_versions.yml"    , emit: yml
-    path "software_versions_mqc.yml", emit: mqc_yml
-    path "versions.yml"             , emit: versions
-
-    script:
-    """
-    #!/usr/bin/env python
-
-    import yaml
-    import platform
-    from textwrap import dedent
-
-    def _make_versions_html(versions):
-        html = [
-            dedent(
-                '''\\
-                <style>
-                #nf-core-versions tbody:nth-child(even) {
-                    background-color: #f2f2f2;
-                }
-                </style>
-                <table class="table" style="width:100%" id="nf-core-versions">
-                    <thead>
-                        <tr>
-                            <th> Process Name </th>
-                            <th> Software </th>
-                            <th> Version  </th>
-                        </tr>
-                    </thead>
-                '''
-            )
-        ]
-        for process, tmp_versions in sorted(versions.items()):
-            html.append("<tbody>")
-            for i, (tool, version) in enumerate(sorted(tmp_versions.items())):
-                html.append(
-                    dedent(
-                        f'''\\
-                        <tr>
-                            <td><samp>{process if (i == 0) else ''}</samp></td>
-                            <td><samp>{tool}</samp></td>
-                            <td><samp>{version}</samp></td>
-                        </tr>
-                        '''
-                    )
-                )
-            html.append("</tbody>")
-        html.append("</table>")
-        return "\\n".join(html)
-
-    module_versions = {}
-    module_versions["${task.process.tokenize(':').last()}"] = {
-        'python': platform.python_version(),
-        'yaml': yaml.__version__
-    }
-
-    with open("$versions") as f:
-        workflow_versions = yaml.load(f, Loader=yaml.BaseLoader) | module_versions
-
-    workflow_versions["Workflow"] = {
-        "Nextflow": "$workflow.nextflow.version",
-        "$workflow.manifest.name": "$workflow.manifest.version"
-    }
-
-    versions_mqc = {
-        'id': 'software_versions',
-        'section_name': '${workflow.manifest.name} Software Versions',
-        'section_href': 'https://github.com/${workflow.manifest.name}',
-        'plot_type': 'html',
-        'description': 'are collected at run time from the software output.',
-        'data': _make_versions_html(workflow_versions)
-    }
-
-    with open("software_versions.yml", 'w') as f:
-        yaml.dump(workflow_versions, f, default_flow_style=False)
-    with open("software_versions_mqc.yml", 'w') as f:
-        yaml.dump(versions_mqc, f, default_flow_style=False)
-
-    with open('versions.yml', 'w') as f:
-        yaml.dump(module_versions, f, default_flow_style=False)
-    """
-}
diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf b/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf
new file mode 100644
index 00000000..934bb467
--- /dev/null
+++ b/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf
@@ -0,0 +1,21 @@
+process CUSTOM_DUMPSOFTWAREVERSIONS {
+    label 'process_low'
+
+    // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container
+    conda (params.enable_conda ? "bioconda::multiqc=1.11" : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0' :
+        'quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0' }"
+
+    input:
+    path versions
+
+    output:
+    path "software_versions.yml"    , emit: yml
+    path "software_versions_mqc.yml", emit: mqc_yml
+    path "versions.yml"             , emit: versions
+
+    script:
+    def args = task.ext.args ?: ''
+    template 'dumpsoftwareversions.py'
+}
diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml
new file mode 100644
index 00000000..5b5b8a60
--- /dev/null
+++ b/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml
@@ -0,0 +1,34 @@
+name: custom_dumpsoftwareversions
+description: Custom module used to dump software versions within the nf-core pipeline template
+keywords:
+  - custom
+  - version
+tools:
+  - custom:
+      description: Custom module used to dump software versions within the nf-core pipeline template
+      homepage: https://github.com/nf-core/tools
+      documentation: https://github.com/nf-core/tools
+      licence: ['MIT']
+input:
+  - versions:
+      type: file
+      description: YML file containing software versions
+      pattern: "*.yml"
+
+output:
+  - yml:
+      type: file
+      description: Standard YML file containing software versions
+      pattern: "software_versions.yml"
+  - mqc_yml:
+      type: file
+      description: MultiQC custom content YML file containing software versions
+      pattern: "software_versions_mqc.yml"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@drpatelh"
+  - "@grst"
diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py
new file mode 100644
index 00000000..d1390392
--- /dev/null
+++ b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+import yaml
+import platform
+from textwrap import dedent
+
+
+def _make_versions_html(versions):
+    html = [
+        dedent(
+            """\\
+            <style>
+            #nf-core-versions tbody:nth-child(even) {
+                background-color: #f2f2f2;
+            }
+            </style>
+            <table class="table" style="width:100%" id="nf-core-versions">
+                <thead>
+                    <tr>
+                        <th> Process Name </th>
+                        <th> Software </th>
+                        <th> Version  </th>
+                    </tr>
+                </thead>
+            """
+        )
+    ]
+    for process, tmp_versions in sorted(versions.items()):
+        html.append("<tbody>")
+        for i, (tool, version) in enumerate(sorted(tmp_versions.items())):
+            html.append(
+                dedent(
+                    f"""\\
+                    <tr>
+                        <td><samp>{process if (i == 0) else ''}</samp></td>
+                        <td><samp>{tool}</samp></td>
+                        <td><samp>{version}</samp></td>
+                    </tr>
+                    """
+                )
+            )
+        html.append("</tbody>")
+    html.append("</table>")
+    return "\\n".join(html)
+
+
+versions_this_module = {}
+versions_this_module["${task.process}"] = {
+    "python": platform.python_version(),
+    "yaml": yaml.__version__,
+}
+
+with open("$versions") as f:
+    versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module
+
+# aggregate versions by the module name (derived from fully-qualified process name)
+versions_by_module = {}
+for process, process_versions in versions_by_process.items():
+    module = process.split(":")[-1]
+    try:
+        assert versions_by_module[module] == process_versions, (
+            "We assume that software versions are the same between all modules. "
+            "If you see this error-message it means you discovered an edge-case "
+            "and should open an issue in nf-core/tools. "
+        )
+    except KeyError:
+        versions_by_module[module] = process_versions
+
+versions_by_module["Workflow"] = {
+    "Nextflow": "$workflow.nextflow.version",
+    "$workflow.manifest.name": "$workflow.manifest.version",
+}
+
+versions_mqc = {
+    "id": "software_versions",
+    "section_name": "${workflow.manifest.name} Software Versions",
+    "section_href": "https://github.com/${workflow.manifest.name}",
+    "plot_type": "html",
+    "description": "are collected at run time from the software output.",
+    "data": _make_versions_html(versions_by_module),
+}
+
+with open("software_versions.yml", "w") as f:
+    yaml.dump(versions_by_module, f, default_flow_style=False)
+with open("software_versions_mqc.yml", "w") as f:
+    yaml.dump(versions_mqc, f, default_flow_style=False)
+
+with open("versions.yml", "w") as f:
+    yaml.dump(versions_this_module, f, default_flow_style=False)
diff --git a/modules/local/sratools_fasterqdump.nf b/modules/nf-core/modules/sratools/fasterqdump/main.nf
similarity index 61%
rename from modules/local/sratools_fasterqdump.nf
rename to modules/nf-core/modules/sratools/fasterqdump/main.nf
index f834eb8b..73e3673d 100644
--- a/modules/local/sratools_fasterqdump.nf
+++ b/modules/nf-core/modules/sratools/fasterqdump/main.nf
@@ -1,4 +1,3 @@
-
 process SRATOOLS_FASTERQDUMP {
     tag "$meta.id"
     label 'process_medium'
@@ -12,19 +11,17 @@ process SRATOOLS_FASTERQDUMP {
     tuple val(meta), path(sra)
 
     output:
-    tuple val(meta), path(fastq_output), emit: reads
-    tuple val(meta), path(md5_output)  , emit: md5
-    path "versions.yml"                , emit: versions
+    tuple val(meta), path(output), emit: reads
+    path "versions.yml"          , emit: versions
 
     script:
-    def args   = task.ext.args  ?: ''
-    def args2  = task.ext.args2 ?: ''
+    def args = task.ext.args ?: ''
+    def args2 = task.ext.args2 ?: ''
     def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
     // Paired-end data extracted by fasterq-dump (--split-3 the default) always creates
     // *_1.fastq *_2.fastq files but sometimes also an additional *.fastq file
     // for unpaired reads which we ignore here.
-    fastq_output = meta.single_end ? '*.fastq.gz'     : '*_{1,2}.fastq.gz'
-    md5_output   = meta.single_end ? '*.fastq.gz.md5' : '*_{1,2}.fastq.gz.md5'
+    output = meta.single_end ? '*.fastq.gz' : '*_{1,2}.fastq.gz'
     """
     eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
     if [[ ! -f "\${NCBI_SETTINGS}" ]]; then
@@ -43,24 +40,8 @@ process SRATOOLS_FASTERQDUMP {
         --processes $task.cpus \\
         *.fastq
 
-    ## Rename FastQ files by meta.id
-    if [ -f  ${sra.name}.fastq.gz ]; then
-        mv ${sra.name}.fastq.gz ${meta.id}.fastq.gz
-        md5sum ${meta.id}.fastq.gz > ${meta.id}.fastq.gz.md5
-    fi
-
-    if [ -f  ${sra.name}_1.fastq.gz ]; then
-        mv ${sra.name}_1.fastq.gz ${meta.id}_1.fastq.gz
-        md5sum ${meta.id}_1.fastq.gz > ${meta.id}_1.fastq.gz.md5
-    fi
-
-    if [ -f  ${sra.name}_2.fastq.gz ]; then
-        mv ${sra.name}_2.fastq.gz ${meta.id}_2.fastq.gz
-        md5sum ${meta.id}_2.fastq.gz > ${meta.id}_2.fastq.gz.md5
-    fi
-
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         sratools: \$(fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+')
         pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' )
     END_VERSIONS
diff --git a/modules/nf-core/modules/sratools/fasterqdump/meta.yml b/modules/nf-core/modules/sratools/fasterqdump/meta.yml
new file mode 100644
index 00000000..1478bed8
--- /dev/null
+++ b/modules/nf-core/modules/sratools/fasterqdump/meta.yml
@@ -0,0 +1,42 @@
+name: sratools_fasterqdump
+description: Extract sequencing reads in FASTQ format from a given NCBI Sequence Read Archive (SRA).
+keywords:
+  - sequencing
+  - FASTQ
+  - dump
+tools:
+  - sratools:
+      description: SRA Toolkit and SDK from NCBI
+      homepage: https://github.com/ncbi/sra-tools
+      documentation: https://github.com/ncbi/sra-tools/wiki
+      tool_dev_url: https://github.com/ncbi/sra-tools
+      licence: ['US-Government-Work']
+
+input:
+  - meta:
+      type: map
+      description: >
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - sra:
+      type: directory
+      description: Directory containing ETL data for the given SRA.
+      pattern: "*/*.sra"
+
+output:
+  - meta:
+      type: map
+      description: >
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+  - reads:
+      type: file
+      description: Extracted FASTQ file or files if the sequencing reads are paired-end.
+      pattern: "*.fastq.gz"
+
+authors:
+  - "@Midnighter"
diff --git a/modules/local/sratools_prefetch.nf b/modules/nf-core/modules/sratools/prefetch/main.nf
similarity index 90%
rename from modules/local/sratools_prefetch.nf
rename to modules/nf-core/modules/sratools/prefetch/main.nf
index 1edb16a3..1e1eb802 100644
--- a/modules/local/sratools_prefetch.nf
+++ b/modules/nf-core/modules/sratools/prefetch/main.nf
@@ -1,4 +1,3 @@
-
 process SRATOOLS_PREFETCH {
     tag "$id"
     label 'process_low'
@@ -17,7 +16,7 @@ process SRATOOLS_PREFETCH {
     path "versions.yml"         , emit: versions
 
     script:
-    def args   = task.ext.args ?: ''
+    def args = task.ext.args ?: ''
     def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
     """
     eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
@@ -26,7 +25,7 @@ process SRATOOLS_PREFETCH {
         printf '${config}' > "\${NCBI_SETTINGS}"
     fi
 
-    retry_with_backoff.sh prefetch \\
+    prefetch \\
         $args \\
         --progress \\
         $id
@@ -34,7 +33,7 @@ process SRATOOLS_PREFETCH {
     vdb-validate $id
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         sratools: \$(prefetch --version 2>&1 | grep -Eo '[0-9.]+')
     END_VERSIONS
     """
diff --git a/modules/nf-core/modules/sratools/prefetch/meta.yml b/modules/nf-core/modules/sratools/prefetch/meta.yml
new file mode 100644
index 00000000..22213b29
--- /dev/null
+++ b/modules/nf-core/modules/sratools/prefetch/meta.yml
@@ -0,0 +1,43 @@
+name: sratools_prefetch
+description: Download sequencing data from the NCBI Sequence Read Archive (SRA).
+keywords:
+  - sequencing
+  - fastq
+  - prefetch
+tools:
+  - sratools:
+      description: SRA Toolkit and SDK from NCBI
+      homepage: https://github.com/ncbi/sra-tools
+      documentation: https://github.com/ncbi/sra-tools/wiki
+      tool_dev_url: https://github.com/ncbi/sra-tools
+      licence: ['US-Government-Work']
+
+input:
+  - meta:
+      type: map
+      description: >
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - id:
+      type: val
+      description: >
+        A string denoting an SRA id.
+
+output:
+  - meta:
+      type: map
+      description: >
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - sra:
+      type: directory
+      description: >
+        Directory containing the ETL data for the given SRA id.
+      pattern: "*/*.sra"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@Midnighter"

From 1f307ce46f565be526f92d119dc16df01bbab8e5 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 09:46:34 +0000
Subject: [PATCH 03/24] Change version command to use task.process

---
 modules/local/multiqc_mappings_config.nf   | 2 +-
 modules/local/sra_fastq_ftp.nf             | 4 ++--
 modules/local/sra_ids_to_runinfo.nf        | 2 +-
 modules/local/sra_merge_samplesheet.nf     | 2 +-
 modules/local/sra_runinfo_to_ftp.nf        | 2 +-
 modules/local/synapse_get.nf               | 2 +-
 modules/local/synapse_list.nf              | 2 +-
 modules/local/synapse_merge_samplesheet.nf | 2 +-
 modules/local/synapse_show.nf              | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/local/multiqc_mappings_config.nf b/modules/local/multiqc_mappings_config.nf
index 15c30846..31f06dae 100644
--- a/modules/local/multiqc_mappings_config.nf
+++ b/modules/local/multiqc_mappings_config.nf
@@ -20,7 +20,7 @@ process MULTIQC_MAPPINGS_CONFIG {
         multiqc_config.yml
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         python: \$(python --version | sed 's/Python //g')
     END_VERSIONS
     """
diff --git a/modules/local/sra_fastq_ftp.nf b/modules/local/sra_fastq_ftp.nf
index dfbe72dd..2dc0c8ae 100644
--- a/modules/local/sra_fastq_ftp.nf
+++ b/modules/local/sra_fastq_ftp.nf
@@ -30,7 +30,7 @@ process SRA_FASTQ_FTP {
         md5sum -c ${meta.id}.fastq.gz.md5
 
         cat <<-END_VERSIONS > versions.yml
-        ${task.process.tokenize(':').last()}:
+        "${task.process}":
             curl: \$(echo \$(curl --version | head -n 1 | sed 's/^curl //; s/ .*\$//'))
         END_VERSIONS
         """
@@ -53,7 +53,7 @@ process SRA_FASTQ_FTP {
         md5sum -c ${meta.id}_2.fastq.gz.md5
 
         cat <<-END_VERSIONS > versions.yml
-        ${task.process.tokenize(':').last()}:
+        "${task.process}"
             curl: \$(echo \$(curl --version | head -n 1 | sed 's/^curl //; s/ .*\$//'))
         END_VERSIONS
         """
diff --git a/modules/local/sra_ids_to_runinfo.nf b/modules/local/sra_ids_to_runinfo.nf
index c8209690..57fdb10e 100644
--- a/modules/local/sra_ids_to_runinfo.nf
+++ b/modules/local/sra_ids_to_runinfo.nf
@@ -26,7 +26,7 @@ process SRA_IDS_TO_RUNINFO {
         $metadata_fields
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         python: \$(python --version | sed 's/Python //g')
     END_VERSIONS
     """
diff --git a/modules/local/sra_merge_samplesheet.nf b/modules/local/sra_merge_samplesheet.nf
index be3f1087..863dfaa7 100644
--- a/modules/local/sra_merge_samplesheet.nf
+++ b/modules/local/sra_merge_samplesheet.nf
@@ -28,7 +28,7 @@ process SRA_MERGE_SAMPLESHEET {
     done
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         sed: \$(echo \$(sed --version 2>&1) | sed 's/^.*GNU sed) //; s/ .*\$//')
     END_VERSIONS
     """
diff --git a/modules/local/sra_runinfo_to_ftp.nf b/modules/local/sra_runinfo_to_ftp.nf
index a6b6a829..13fa6c3e 100644
--- a/modules/local/sra_runinfo_to_ftp.nf
+++ b/modules/local/sra_runinfo_to_ftp.nf
@@ -20,7 +20,7 @@ process SRA_RUNINFO_TO_FTP {
         ${runinfo.toString().tokenize(".")[0]}.runinfo_ftp.tsv
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         python: \$(python --version | sed 's/Python //g')
     END_VERSIONS
     """
diff --git a/modules/local/synapse_get.nf b/modules/local/synapse_get.nf
index 5269286c..885a9a95 100644
--- a/modules/local/synapse_get.nf
+++ b/modules/local/synapse_get.nf
@@ -30,7 +30,7 @@ process SYNAPSE_GET {
     echo "${meta.md5} \t ${meta.name}" > ${meta.id}.md5
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         synapse: \$(synapse --version | sed -e "s/Synapse Client //g")
     END_VERSIONS
     """
diff --git a/modules/local/synapse_list.nf b/modules/local/synapse_list.nf
index 0255c925..58034038 100644
--- a/modules/local/synapse_list.nf
+++ b/modules/local/synapse_list.nf
@@ -29,7 +29,7 @@ process SYNAPSE_LIST {
         > ${id}.list.txt
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         syanpse: \$(synapse --version | sed -e "s/Synapse Client //g")
     END_VERSIONS
     """
diff --git a/modules/local/synapse_merge_samplesheet.nf b/modules/local/synapse_merge_samplesheet.nf
index e3f97fa8..01461e14 100644
--- a/modules/local/synapse_merge_samplesheet.nf
+++ b/modules/local/synapse_merge_samplesheet.nf
@@ -21,7 +21,7 @@ process SYNAPSE_MERGE_SAMPLESHEET {
     done
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         sed: \$(echo \$(sed --version 2>&1) | sed 's/^.*GNU sed) //; s/ .*\$//')
     END_VERSIONS
     """
diff --git a/modules/local/synapse_show.nf b/modules/local/synapse_show.nf
index 9a81493a..e03fb2f1 100644
--- a/modules/local/synapse_show.nf
+++ b/modules/local/synapse_show.nf
@@ -29,7 +29,7 @@ process SYNAPSE_SHOW {
         > ${id}.metadata.txt
 
     cat <<-END_VERSIONS > versions.yml
-    ${task.process.tokenize(':').last()}:
+    "${task.process}":
         synapse: \$(synapse --version | sed -e "s/Synapse Client //g")
     END_VERSIONS
     """

From ab3bf4ed865069fa0ae3cdb91dd8ff380af13240 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 09:46:46 +0000
Subject: [PATCH 04/24] Install nf-core modules

---
 modules.json | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/modules.json b/modules.json
index 0e4143ef..5d0b0fe9 100644
--- a/modules.json
+++ b/modules.json
@@ -1,5 +1,17 @@
 {
     "name": "nf-core/fetchngs",
     "homePage": "https://github.com/nf-core/fetchngs",
-    "repos": {}
+    "repos": {
+        "nf-core/modules": {
+            "custom/dumpsoftwareversions": {
+                "git_sha": "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41"
+            },
+            "sratools/fasterqdump": {
+                "git_sha": "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41"
+            },
+            "sratools/prefetch": {
+                "git_sha": "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41"
+            }
+        }
+    }
 }
\ No newline at end of file

From b1ced19ee6874ae89443323abf88d760288a1a8c Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 09:47:34 +0000
Subject: [PATCH 05/24] Add executor local for native groovy processes

---
 modules/local/sra_to_samplesheet.nf     | 1 +
 modules/local/synapse_to_samplesheet.nf | 1 +
 2 files changed, 2 insertions(+)

diff --git a/modules/local/sra_to_samplesheet.nf b/modules/local/sra_to_samplesheet.nf
index dbbc6613..465144d7 100644
--- a/modules/local/sra_to_samplesheet.nf
+++ b/modules/local/sra_to_samplesheet.nf
@@ -2,6 +2,7 @@
 process SRA_TO_SAMPLESHEET {
     tag "$meta.id"
 
+    executor 'local'
     memory 100.MB
 
     input:
diff --git a/modules/local/synapse_to_samplesheet.nf b/modules/local/synapse_to_samplesheet.nf
index 40519ec0..2e7945ed 100644
--- a/modules/local/synapse_to_samplesheet.nf
+++ b/modules/local/synapse_to_samplesheet.nf
@@ -2,6 +2,7 @@
 process SYNAPSE_TO_SAMPLESHEET {
     tag "$meta.id"
 
+    executor 'local'
     memory 100.MB
 
     input:

From 4fcd56b631835b81840d1102d1a726b2238c6415 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:53:40 +0000
Subject: [PATCH 06/24] Bump NF version from 21.04.0 to 21.10.3

---
 .github/ISSUE_TEMPLATE/bug_report.md | 2 +-
 .github/workflows/ci.yml             | 2 +-
 README.md                            | 4 ++--
 nextflow.config                      | 3 +--
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 11bb35bf..406ca8df 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -51,7 +51,7 @@ Have you provided the following extra information/files:
 
 ## Nextflow Installation
 
-- Version: <!-- [e.g. 21.04.0] -->
+- Version: <!-- [e.g. 21.10.3] -->
 
 ## Container engine
 
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4b3fe636..eb26bd07 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and latest edge version
-        nxf_ver: ["NXF_VER=21.04.0", "NXF_EDGE=1"]
+        nxf_ver: ["NXF_VER=21.10.3", "NXF_EDGE=1"]
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
diff --git a/README.md b/README.md
index 58e4a8a7..eb046d87 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)
 [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.04.0-23aa62.svg?labelColor=000000)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
@@ -48,7 +48,7 @@ The columns in the auto-created samplesheet can be tailored to be accepted out-o
 
 ## Quick Start
 
-1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.04.0`)
+1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`)
 
 2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(please only use [`Conda`](https://conda.io/miniconda.html) as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_
 
diff --git a/nextflow.config b/nextflow.config
index ea68a9b1..5cc0cdac 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -22,7 +22,6 @@ params {
     // Boilerplate options
     outdir                     = './results'
     tracedir                   = "${params.outdir}/pipeline_info"
-    publish_dir_mode           = 'copy'
     email                      = null
     email_on_fail              = null
     plaintext_email            = false
@@ -145,7 +144,7 @@ manifest {
     homePage        = 'https://github.com/nf-core/fetchngs'
     description     = 'Pipeline to fetch metadata and raw FastQ files from public databases'
     mainScript      = 'main.nf'
-    nextflowVersion = '!>=21.04.0'
+    nextflowVersion = '!>=21.10.3'
     version         = '1.5dev'
 }
 

From 2c28be248ced1ebb7d2a199434c9546cb10ac95c Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:55:08 +0000
Subject: [PATCH 07/24] Move sratools modules to local due to customisation

---
 modules.json                                  |  6 ---
 .../main.nf => local/sratools_fasterqdump.nf} | 27 ++++++++++--
 .../main.nf => local/sratools_prefetch.nf}    |  3 +-
 .../modules/sratools/fasterqdump/meta.yml     | 42 ------------------
 .../modules/sratools/prefetch/meta.yml        | 43 -------------------
 subworkflows/local/sra_fastq_sratools.nf      |  4 +-
 6 files changed, 27 insertions(+), 98 deletions(-)
 rename modules/{nf-core/modules/sratools/fasterqdump/main.nf => local/sratools_fasterqdump.nf} (64%)
 rename modules/{nf-core/modules/sratools/prefetch/main.nf => local/sratools_prefetch.nf} (96%)
 delete mode 100644 modules/nf-core/modules/sratools/fasterqdump/meta.yml
 delete mode 100644 modules/nf-core/modules/sratools/prefetch/meta.yml

diff --git a/modules.json b/modules.json
index 5d0b0fe9..eba88bd4 100644
--- a/modules.json
+++ b/modules.json
@@ -5,12 +5,6 @@
         "nf-core/modules": {
             "custom/dumpsoftwareversions": {
                 "git_sha": "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41"
-            },
-            "sratools/fasterqdump": {
-                "git_sha": "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41"
-            },
-            "sratools/prefetch": {
-                "git_sha": "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41"
             }
         }
     }
diff --git a/modules/nf-core/modules/sratools/fasterqdump/main.nf b/modules/local/sratools_fasterqdump.nf
similarity index 64%
rename from modules/nf-core/modules/sratools/fasterqdump/main.nf
rename to modules/local/sratools_fasterqdump.nf
index 73e3673d..7a75e551 100644
--- a/modules/nf-core/modules/sratools/fasterqdump/main.nf
+++ b/modules/local/sratools_fasterqdump.nf
@@ -1,3 +1,4 @@
+
 process SRATOOLS_FASTERQDUMP {
     tag "$meta.id"
     label 'process_medium'
@@ -11,17 +12,19 @@ process SRATOOLS_FASTERQDUMP {
     tuple val(meta), path(sra)
 
     output:
-    tuple val(meta), path(output), emit: reads
-    path "versions.yml"          , emit: versions
+    tuple val(meta), path(fastq_output), emit: reads
+    tuple val(meta), path(md5_output)  , emit: md5
+    path "versions.yml"                , emit: versions
 
     script:
-    def args = task.ext.args ?: ''
+    def args = task.ext.args  ?: ''
     def args2 = task.ext.args2 ?: ''
     def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
     // Paired-end data extracted by fasterq-dump (--split-3 the default) always creates
     // *_1.fastq *_2.fastq files but sometimes also an additional *.fastq file
     // for unpaired reads which we ignore here.
-    output = meta.single_end ? '*.fastq.gz' : '*_{1,2}.fastq.gz'
+    fastq_output = meta.single_end ? '*.fastq.gz'     : '*_{1,2}.fastq.gz'
+    md5_output = meta.single_end ? '*.fastq.gz.md5' : '*_{1,2}.fastq.gz.md5'
     """
     eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
     if [[ ! -f "\${NCBI_SETTINGS}" ]]; then
@@ -40,6 +43,22 @@ process SRATOOLS_FASTERQDUMP {
         --processes $task.cpus \\
         *.fastq
 
+    ## Rename FastQ files by meta.id
+    if [ -f  ${sra.name}.fastq.gz ]; then
+        mv ${sra.name}.fastq.gz ${meta.id}.fastq.gz
+        md5sum ${meta.id}.fastq.gz > ${meta.id}.fastq.gz.md5
+    fi
+
+    if [ -f  ${sra.name}_1.fastq.gz ]; then
+        mv ${sra.name}_1.fastq.gz ${meta.id}_1.fastq.gz
+        md5sum ${meta.id}_1.fastq.gz > ${meta.id}_1.fastq.gz.md5
+    fi
+
+    if [ -f  ${sra.name}_2.fastq.gz ]; then
+        mv ${sra.name}_2.fastq.gz ${meta.id}_2.fastq.gz
+        md5sum ${meta.id}_2.fastq.gz > ${meta.id}_2.fastq.gz.md5
+    fi
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         sratools: \$(fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+')
diff --git a/modules/nf-core/modules/sratools/prefetch/main.nf b/modules/local/sratools_prefetch.nf
similarity index 96%
rename from modules/nf-core/modules/sratools/prefetch/main.nf
rename to modules/local/sratools_prefetch.nf
index 1e1eb802..76679a93 100644
--- a/modules/nf-core/modules/sratools/prefetch/main.nf
+++ b/modules/local/sratools_prefetch.nf
@@ -1,3 +1,4 @@
+
 process SRATOOLS_PREFETCH {
     tag "$id"
     label 'process_low'
@@ -25,7 +26,7 @@ process SRATOOLS_PREFETCH {
         printf '${config}' > "\${NCBI_SETTINGS}"
     fi
 
-    prefetch \\
+    retry_with_backoff.sh prefetch \\
         $args \\
         --progress \\
         $id
diff --git a/modules/nf-core/modules/sratools/fasterqdump/meta.yml b/modules/nf-core/modules/sratools/fasterqdump/meta.yml
deleted file mode 100644
index 1478bed8..00000000
--- a/modules/nf-core/modules/sratools/fasterqdump/meta.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: sratools_fasterqdump
-description: Extract sequencing reads in FASTQ format from a given NCBI Sequence Read Archive (SRA).
-keywords:
-  - sequencing
-  - FASTQ
-  - dump
-tools:
-  - sratools:
-      description: SRA Toolkit and SDK from NCBI
-      homepage: https://github.com/ncbi/sra-tools
-      documentation: https://github.com/ncbi/sra-tools/wiki
-      tool_dev_url: https://github.com/ncbi/sra-tools
-      licence: ['US-Government-Work']
-
-input:
-  - meta:
-      type: map
-      description: >
-        Groovy Map containing sample information
-        e.g. [ id:'test', single_end:false ]
-  - sra:
-      type: directory
-      description: Directory containing ETL data for the given SRA.
-      pattern: "*/*.sra"
-
-output:
-  - meta:
-      type: map
-      description: >
-        Groovy Map containing sample information
-        e.g. [ id:'test', single_end:false ]
-  - versions:
-      type: file
-      description: File containing software versions
-      pattern: "versions.yml"
-  - reads:
-      type: file
-      description: Extracted FASTQ file or files if the sequencing reads are paired-end.
-      pattern: "*.fastq.gz"
-
-authors:
-  - "@Midnighter"
diff --git a/modules/nf-core/modules/sratools/prefetch/meta.yml b/modules/nf-core/modules/sratools/prefetch/meta.yml
deleted file mode 100644
index 22213b29..00000000
--- a/modules/nf-core/modules/sratools/prefetch/meta.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-name: sratools_prefetch
-description: Download sequencing data from the NCBI Sequence Read Archive (SRA).
-keywords:
-  - sequencing
-  - fastq
-  - prefetch
-tools:
-  - sratools:
-      description: SRA Toolkit and SDK from NCBI
-      homepage: https://github.com/ncbi/sra-tools
-      documentation: https://github.com/ncbi/sra-tools/wiki
-      tool_dev_url: https://github.com/ncbi/sra-tools
-      licence: ['US-Government-Work']
-
-input:
-  - meta:
-      type: map
-      description: >
-        Groovy Map containing sample information
-        e.g. [ id:'test', single_end:false ]
-  - id:
-      type: val
-      description: >
-        A string denoting an SRA id.
-
-output:
-  - meta:
-      type: map
-      description: >
-        Groovy Map containing sample information
-        e.g. [ id:'test', single_end:false ]
-  - sra:
-      type: directory
-      description: >
-        Directory containing the ETL data for the given SRA id.
-      pattern: "*/*.sra"
-  - versions:
-      type: file
-      description: File containing software versions
-      pattern: "versions.yml"
-
-authors:
-  - "@Midnighter"
diff --git a/subworkflows/local/sra_fastq_sratools.nf b/subworkflows/local/sra_fastq_sratools.nf
index e093fe2f..1d59eb61 100644
--- a/subworkflows/local/sra_fastq_sratools.nf
+++ b/subworkflows/local/sra_fastq_sratools.nf
@@ -2,8 +2,8 @@
 // Download FASTQ sequencing reads from the NCBI's Sequence Read Archive (SRA).
 //
 
-include { SRATOOLS_PREFETCH    } from '../../modules/local/sratools_prefetch.nf'
-include { SRATOOLS_FASTERQDUMP } from '../../modules/local/sratools_fasterqdump.nf'
+include { SRATOOLS_PREFETCH    } from '../../modules/local/sratools_prefetch'
+include { SRATOOLS_FASTERQDUMP } from '../../modules/local/sratools_fasterqdump'
 
 workflow SRA_FASTQ_SRATOOLS {
     take:

From 25640bd6aae7b2e7b8625883b109c4c4880a00b0 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:55:17 +0000
Subject: [PATCH 08/24] Update CHANGELOG

---
 CHANGELOG.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3e67547..4d6341df 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,11 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unpublished Version / DEV]
+## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5] - 2021-11-29
+
+* Finish porting the pipeline to the updated Nextflow DSL2 syntax adopted on nf-core/modules
+    * Bump minimum Nextflow version from `21.04.0` -> `21.10.3`
+    * Removed `--publish_dir_mode` as it is no longer required for the new syntax
 
 ### Enhancements & fixes
 

From 6e3259a057975c93a30361f5b75208f92aba02fa Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:55:34 +0000
Subject: [PATCH 09/24] Remove params.publish_dir_mode

---
 conf/modules.config | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/conf/modules.config b/conf/modules.config
index 753ab27d..c08c1fe6 100644
--- a/conf/modules.config
+++ b/conf/modules.config
@@ -17,14 +17,14 @@ process {
 
     publishDir = [
         path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
-        mode: params.publish_dir_mode,
+        mode: 'copy',
         saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
     ]
 
-    withName: DUMPSOFTWAREVERSIONS {
+    withName: CUSTOM_DUMPSOFTWAREVERSIONS {
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },
-            mode: params.publish_dir_mode,
+            mode: 'copy',
             pattern: '*_versions.yml'
         ]
     }
@@ -48,7 +48,7 @@ if (params.input_type == 'sra') {
         withName: SRA_RUNINFO_TO_FTP {
             publishDir = [
                 path: { "${params.outdir}/metadata" },
-                mode: params.publish_dir_mode,
+                mode: 'copy',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }
@@ -58,12 +58,12 @@ if (params.input_type == 'sra') {
             publishDir = [
                 [
                     path: { "${params.outdir}/fastq" },
-                    mode: params.publish_dir_mode,
+                    mode: 'copy',
                     pattern: "*gz"
                 ],
                 [
                     path: { "${params.outdir}/fastq/md5" },
-                    mode: params.publish_dir_mode,
+                    mode: 'copy',
                     pattern: "*.md5"
                 ]
             ]
@@ -80,12 +80,12 @@ if (params.input_type == 'sra') {
             publishDir = [
                 [
                     path: { "${params.outdir}/fastq" },
-                    mode: params.publish_dir_mode,
+                    mode: 'copy',
                     pattern: "*gz"
                 ],
                 [
                     path: { "${params.outdir}/fastq/md5" },
-                    mode: params.publish_dir_mode,
+                    mode: 'copy',
                     pattern: "*.md5"
                 ]
             ]
@@ -101,7 +101,7 @@ if (params.input_type == 'sra') {
         withName: SRA_MERGE_SAMPLESHEET {
             publishDir = [
                 path: { "${params.outdir}/samplesheet" },
-                mode: params.publish_dir_mode,
+                mode: 'copy',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }
@@ -109,7 +109,7 @@ if (params.input_type == 'sra') {
         withName: MULTIQC_MAPPINGS_CONFIG {
             publishDir = [
                 path: { "${params.outdir}/samplesheet" },
-                mode: params.publish_dir_mode,
+                mode: 'copy',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }
@@ -129,7 +129,7 @@ if (params.input_type == 'synapse') {
             ext.args = '--long'
             publishDir = [
                 path: { "${params.outdir}/metadata" },
-                mode: params.publish_dir_mode,
+                mode: 'copy',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }
@@ -138,12 +138,12 @@ if (params.input_type == 'synapse') {
             publishDir = [
                 [
                     path: { "${params.outdir}/fastq" },
-                    mode: params.publish_dir_mode,
+                    mode: 'copy',
                     pattern: "*gz"
                 ],
                 [
                     path: { "${params.outdir}/fastq/md5" },
-                    mode: params.publish_dir_mode,
+                    mode: 'copy',
                     pattern: "*.md5"
                 ]
             ]
@@ -152,7 +152,7 @@ if (params.input_type == 'synapse') {
         withName: SYNAPSE_SHOW {
             publishDir = [
                 path: { "${params.outdir}/metadata" },
-                mode: params.publish_dir_mode,
+                mode: 'copy',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }
@@ -167,7 +167,7 @@ if (params.input_type == 'synapse') {
         withName: SYNAPSE_MERGE_SAMPLESHEET {
             publishDir = [
                 path: { "${params.outdir}/samplesheet" },
-                mode: params.publish_dir_mode,
+                mode: 'copy',
                 saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
             ]
         }

From 12fcb97082ba84016fd525f895a819daecedbca2 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:55:38 +0000
Subject: [PATCH 10/24] Remove params.publish_dir_mode

---
 nextflow_schema.json | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/nextflow_schema.json b/nextflow_schema.json
index 2831dac7..ae439a7b 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -177,22 +177,6 @@
                     "fa_icon": "fas fa-question-circle",
                     "hidden": true
                 },
-                "publish_dir_mode": {
-                    "type": "string",
-                    "default": "copy",
-                    "description": "Method used to save pipeline results to output directory.",
-                    "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.",
-                    "fa_icon": "fas fa-copy",
-                    "enum": [
-                        "symlink",
-                        "rellink",
-                        "link",
-                        "copy",
-                        "copyNoFollow",
-                        "move"
-                    ],
-                    "hidden": true
-                },
                 "email_on_fail": {
                     "type": "string",
                     "description": "Email address for completion summary, only when pipeline fails.",

From 194fb72e63592819a03a61657126a02e0dedd5c3 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:55:55 +0000
Subject: [PATCH 11/24] Fix tyop in versions syntax

---
 modules/local/sra_fastq_ftp.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/local/sra_fastq_ftp.nf b/modules/local/sra_fastq_ftp.nf
index 2dc0c8ae..edb5241a 100644
--- a/modules/local/sra_fastq_ftp.nf
+++ b/modules/local/sra_fastq_ftp.nf
@@ -53,7 +53,7 @@ process SRA_FASTQ_FTP {
         md5sum -c ${meta.id}_2.fastq.gz.md5
 
         cat <<-END_VERSIONS > versions.yml
-        "${task.process}"
+        "${task.process}":
             curl: \$(echo \$(curl --version | head -n 1 | sed 's/^curl //; s/ .*\$//'))
         END_VERSIONS
         """

From 4ee548901ab86a757cf7359cbfd78aeace977e3c Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 20:56:10 +0000
Subject: [PATCH 12/24] Rename dumpsoftwareversions module

---
 workflows/sra.nf     | 14 +++++++++++---
 workflows/synapse.nf | 11 +++++++++--
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/workflows/sra.nf b/workflows/sra.nf
index 4012a571..9336fcc1 100644
--- a/workflows/sra.nf
+++ b/workflows/sra.nf
@@ -22,11 +22,19 @@ WorkflowSra.initialise(params, log, valid_params)
 include { SRA_IDS_TO_RUNINFO      } from '../modules/local/sra_ids_to_runinfo'
 include { SRA_RUNINFO_TO_FTP      } from '../modules/local/sra_runinfo_to_ftp'
 include { SRA_FASTQ_FTP           } from '../modules/local/sra_fastq_ftp'
-include { SRA_FASTQ_SRATOOLS      } from '../subworkflows/local/sra_fastq_sratools'
 include { SRA_TO_SAMPLESHEET      } from '../modules/local/sra_to_samplesheet'
 include { SRA_MERGE_SAMPLESHEET   } from '../modules/local/sra_merge_samplesheet'
 include { MULTIQC_MAPPINGS_CONFIG } from '../modules/local/multiqc_mappings_config'
-include { DUMPSOFTWAREVERSIONS    } from '../modules/local/dumpsoftwareversions'
+
+include { SRA_FASTQ_SRATOOLS      } from '../subworkflows/local/sra_fastq_sratools'
+
+/*
+========================================================================================
+    IMPORT NF-CORE MODULES/SUBWORKFLOWS
+========================================================================================
+*/
+
+include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main'
 
 /*
 ========================================================================================
@@ -126,7 +134,7 @@ workflow SRA {
     //
     // MODULE: Dump software versions for all tools used in the workflow
     //
-    DUMPSOFTWAREVERSIONS (
+    CUSTOM_DUMPSOFTWAREVERSIONS (
         ch_versions.unique().collectFile(name: 'collated_versions.yml')
     )
 }
diff --git a/workflows/synapse.nf b/workflows/synapse.nf
index 5a7e3fe7..2a673623 100644
--- a/workflows/synapse.nf
+++ b/workflows/synapse.nf
@@ -24,7 +24,14 @@ include { SYNAPSE_SHOW              } from '../modules/local/synapse_show'
 include { SYNAPSE_GET               } from '../modules/local/synapse_get'
 include { SYNAPSE_TO_SAMPLESHEET    } from '../modules/local/synapse_to_samplesheet'
 include { SYNAPSE_MERGE_SAMPLESHEET } from '../modules/local/synapse_merge_samplesheet'
-include { DUMPSOFTWAREVERSIONS      } from '../modules/local/dumpsoftwareversions'
+
+/*
+========================================================================================
+    IMPORT NF-CORE MODULES/SUBWORKFLOWS
+========================================================================================
+*/
+
+include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main'
 
 /*
 ========================================================================================
@@ -122,7 +129,7 @@ workflow SYNAPSE {
     //
     // MODULE: Dump software versions for all tools used in the workflow
     //
-    DUMPSOFTWAREVERSIONS (
+    CUSTOM_DUMPSOFTWAREVERSIONS (
         ch_versions.unique().collectFile(name: 'collated_versions.yml')
     )
 }

From 27f7827e579551bd9768dd11ac8fa8919bc42619 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sat, 27 Nov 2021 21:30:57 +0000
Subject: [PATCH 13/24] Bump version to 1.5

---
 nextflow.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nextflow.config b/nextflow.config
index 5cc0cdac..f3284c52 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -145,7 +145,7 @@ manifest {
     description     = 'Pipeline to fetch metadata and raw FastQ files from public databases'
     mainScript      = 'main.nf'
     nextflowVersion = '!>=21.10.3'
-    version         = '1.5dev'
+    version         = '1.5'
 }
 
 // Load modules.config for DSL2 module specific options

From c12cee0800398762ab9737d7ab35586c1fc7a01f Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sun, 28 Nov 2021 14:14:35 +0000
Subject: [PATCH 14/24] Upgrade CI tests to those on dev

---
 .github/workflows/awsfulltest.yml     | 11 ++++++++---
 .github/workflows/awstest.yml         |  9 +++++----
 .github/workflows/ci.yml              |  7 +++----
 .github/workflows/linting_comment.yml |  1 +
 4 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml
index a40a53f4..dc26b72d 100644
--- a/.github/workflows/awsfulltest.yml
+++ b/.github/workflows/awsfulltest.yml
@@ -14,10 +14,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@master
+        uses: nf-core/tower-action@v2
+        # TODO nf-core: You can customise AWS full pipeline tests as required
+        # Add full size test data (but still relatively small datasets for few samples)
+        # on the `test_full.config` test runs with only one set of parameters
+        
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          bearer_token: ${{ secrets.TOWER_BEARER_TOKEN }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
           pipeline: ${{ github.repository }}
           revision: ${{ github.sha }}
@@ -26,4 +30,5 @@ jobs:
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/fetchngs/results-${{ github.sha }}"
             }
-          profiles: '[ "test_full", "aws_tower" ]'
+          profiles: test_full,aws_tower
+
diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml
index 6f008aed..df53dcf3 100644
--- a/.github/workflows/awstest.yml
+++ b/.github/workflows/awstest.yml
@@ -11,11 +11,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@master
-
+        uses: nf-core/tower-action@v2
+        
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          bearer_token: ${{ secrets.TOWER_BEARER_TOKEN }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
           pipeline: ${{ github.repository }}
           revision: ${{ github.sha }}
@@ -24,4 +24,5 @@ jobs:
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/fetchngs/results-${{ github.sha }}"
             }
-          profiles: '[ "test", "aws_tower" ]'
+          profiles: test,aws_tower
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index eb26bd07..56193da8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,9 +8,6 @@ on:
   release:
     types: [published]
 
-# Uncomment if we need an edge release of Nextflow again
-# env: NXF_EDGE: 1
-
 jobs:
   test:
     name: Run workflow tests
@@ -22,7 +19,9 @@ jobs:
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and latest edge version
-        nxf_ver: ["NXF_VER=21.10.3", "NXF_EDGE=1"]
+        nxf_ver:
+         - 'NXF_VER=21.10.3'
+        #  - 'NXF_EDGE=1'
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml
index 90f03c6f..44d72994 100644
--- a/.github/workflows/linting_comment.yml
+++ b/.github/workflows/linting_comment.yml
@@ -15,6 +15,7 @@ jobs:
         uses: dawidd6/action-download-artifact@v2
         with:
           workflow: linting.yml
+          workflow_conclusion: completed
 
       - name: Get PR number
         id: pr_number

From e141707b26b2c07732eb2fe2ef71e88baa2c823b Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sun, 28 Nov 2021 14:18:57 +0000
Subject: [PATCH 15/24] Fix linting

---
 .github/workflows/ci.yml | 2 +-
 .nf-core.yml             | 2 ++
 CHANGELOG.md             | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 56193da8..a159501f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
       matrix:
         # Nextflow versions: check pipeline minimum and latest edge version
         nxf_ver:
-         - 'NXF_VER=21.10.3'
+          - 'NXF_VER=21.10.3'
         #  - 'NXF_EDGE=1'
     steps:
       - name: Check out pipeline code
diff --git a/.nf-core.yml b/.nf-core.yml
index b8e4d216..efb97174 100644
--- a/.nf-core.yml
+++ b/.nf-core.yml
@@ -1,6 +1,8 @@
 lint:
   files_unchanged:
     - .github/CONTRIBUTING.md
+    - .github/ISSUE_TEMPLATE/bug_report.md
+    - .github/workflows/linting_comment.yml
     - assets/sendmail_template.txt
     - lib/NfcoreSchema.groovy
     - lib/NfcoreTemplate.groovy
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d6341df..37feff4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,7 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5] - 2021-11-29
+## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5)] - 2021-11-29
 
 * Finish porting the pipeline to the updated Nextflow DSL2 syntax adopted on nf-core/modules
     * Bump minimum Nextflow version from `21.04.0` -> `21.10.3`

From b21a617b8402c0ec43fbd182854f94550360ddcc Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sun, 28 Nov 2021 20:06:48 +0000
Subject: [PATCH 16/24] Lint and fix repo relative to latest dev

---
 .gitattributes                             |  2 +
 .github/CONTRIBUTING.md                    | 56 ++++++++++++++-----
 .github/ISSUE_TEMPLATE/bug_report.md       | 63 ----------------------
 .github/ISSUE_TEMPLATE/bug_report.yml      | 52 ++++++++++++++++++
 .github/ISSUE_TEMPLATE/config.yml          |  1 -
 .github/ISSUE_TEMPLATE/feature_request.md  | 32 -----------
 .github/ISSUE_TEMPLATE/feature_request.yml | 11 ++++
 .github/workflows/awsfulltest.yml          |  5 +-
 .nf-core.yml                               | 16 +++---
 lib/NfcoreSchema.groovy                    | 19 +++++--
 10 files changed, 133 insertions(+), 124 deletions(-)
 delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml
 delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml

diff --git a/.gitattributes b/.gitattributes
index 7fe55006..050bb120 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1,3 @@
 *.config linguist-language=nextflow
+modules/nf-core/** linguist-generated
+subworkflows/nf-core/** linguist-generated
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index f294492d..1d7943a5 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -61,16 +61,23 @@ For further information/help, please consult the [nf-core/fetchngs documentation
 
 To make the nf-core/fetchngs code and processing logic more understandable for new contributors and to ensure quality, we semi-standardise the way the code and other contributions are written.
 
-### Adding a new step or module
-
-If you wish to contribute a new step or module please see the [official guidelines](https://nf-co.re/developers/adding_modules#new-module-guidelines-and-pr-review-checklist) and use the following coding standards:
-
-1. Add any new flags/options to `nextflow.config` with a default (see section below).
-2. Add any new flags/options to `nextflow_schema.json` with help text via `nf-core schema build`.
-3. Add sanity checks for all relevant parameters.
-4. Perform local tests to validate that the new code works as expected.
-5. If applicable, add a new test command in `.github/workflow/ci.yml`.
-6. Add any descriptions of output files to `docs/output.md`.
+### Adding a new step
+
+If you wish to contribute a new step, please use the following coding standards:
+
+1. Define the corresponding input channel into your new process from the expected previous process channel
+2. Write the process block (see below).
+3. Define the output channel if needed (see below).
+4. Add any new flags/options to `nextflow.config` with a default (see below).
+5. Add any new flags/options to `nextflow_schema.json` with help text (with `nf-core schema build`).
+6. Add any new flags/options to the help message (for integer/text parameters, print to help the corresponding `nextflow.config` parameter).
+7. Add sanity checks for all relevant parameters.
+8. Add any new software to the `scrape_software_versions.py` script in `bin/` and the version command to the `scrape_software_versions` process in `main.nf`.
+9. Do local tests that the new code works properly and as expected.
+10. Add a new test command in `.github/workflow/ci.yml`.
+11. If applicable add a [MultiQC](https://https://multiqc.info/) module.
+12. Update MultiQC config `assets/multiqc_config.yaml` so relevant suffixes, name clean up, General Statistics Table column order, and module figures are in the right order.
+13. Optional: Add any descriptions of MultiQC report sections and output files to `docs/output.md`.
 
 ### Default values
 
@@ -82,17 +89,40 @@ Once there, use `nf-core schema build` to add to `nextflow_schema.json`.
 
 Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels.
 
-### Channel naming convention
+The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block.
+
+### Naming schemes
 
 Please use the following naming schemes, to make it easy to understand what is going where.
 
-* Initial process channel: `ch_output_from_<process>`
-* Intermediate and terminal channels: `ch_<previousprocess>_for_<nextprocess>`
+* initial process channel: `ch_output_from_<process>`
+* intermediate and terminal channels: `ch_<previousprocess>_for_<nextprocess>`
 
 ### Nextflow version bumping
 
 If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core bump-version --nextflow . [min-nf-version]`
 
+### Software version reporting
+
+If you add a new tool to the pipeline, please ensure you add the information of the tool to the `get_software_version` process.
+
+Add to the script block of the process, something like the following:
+
+```bash
+<YOUR_TOOL> --version &> v_<YOUR_TOOL>.txt 2>&1 || true
+```
+
+or
+
+```bash
+<YOUR_TOOL> --help | head -n 1 &> v_<YOUR_TOOL>.txt 2>&1 || true
+```
+
+You then need to edit the script `bin/scrape_software_versions.py` to:
+
+1. Add a Python regex for your tool's `--version` output (as in stored in the `v_<YOUR_TOOL>.txt` file), to ensure the version is reported as a `v` and the version number e.g. `v2.1.1`
+2. Add a HTML entry to the `OrderedDict` for formatting in MultiQC.
+
 ### Images and figures
 
 For overview images and other documents we follow the nf-core [style guidelines and examples](https://nf-co.re/developers/design_guidelines).
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
deleted file mode 100644
index 406ca8df..00000000
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ /dev/null
@@ -1,63 +0,0 @@
----
-name: Bug report
-about: Report something that is broken or incorrect
-labels: bug
----
-
-<!--
-# nf-core/fetchngs bug report
-
-Hi there!
-
-Thanks for telling us about a problem with the pipeline.
-Please delete this text and anything that's not relevant from the template below:
--->
-
-## Check Documentation
-
-I have checked the following places for your error:
-
-- [ ] [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting)
-- [ ] [nf-core/fetchngs pipeline documentation](https://nf-co.re/fetchngs/usage)
-
-## Description of the bug
-
-<!-- A clear and concise description of what the bug is. -->
-
-## Steps to reproduce
-
-Steps to reproduce the behaviour:
-
-1. Command line: <!-- [e.g. `nextflow run ...`] -->
-2. See error: <!-- [Please provide your error message] -->
-
-## Expected behaviour
-
-<!-- A clear and concise description of what you expected to happen. -->
-
-## Log files
-
-Have you provided the following extra information/files:
-
-- [ ] The command used to run the pipeline
-- [ ] The `.nextflow.log` file <!-- this is a hidden file in the directory where you launched the pipeline -->
-
-## System
-
-- Hardware: <!-- [e.g. HPC, Desktop, Cloud...] -->
-- Executor: <!-- [e.g. slurm, local, awsbatch...] -->
-- OS: <!-- [e.g. CentOS Linux, macOS, Linux Mint...] -->
-- Version <!-- [e.g. 7, 10.13.6, 18.3...] -->
-
-## Nextflow Installation
-
-- Version: <!-- [e.g. 21.10.3] -->
-
-## Container engine
-
-- Engine: <!-- [e.g. Conda, Docker, Singularity, Podman, Shifter or Charliecloud] -->
-- version: <!-- [e.g. 1.0.0] -->
-
-## Additional context
-
-<!-- Add any other context about the problem here. -->
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
new file mode 100644
index 00000000..7fc6710e
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,52 @@
+
+name: Bug report
+description: Report something that is broken or incorrect
+labels: bug
+body:
+
+  - type: markdown
+    attributes:
+      value: |
+        Before you post this issue, please check the documentation:
+
+        - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting)
+        - [nf-core/fetchngs pipeline documentation](https://nf-co.re/fetchngs/usage)
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Description of the bug
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+
+  - type: textarea
+    id: command_used
+    attributes:
+      label: Command used and terminal output
+      description: Steps to reproduce the behaviour. Please paste the command you used to launch the pipeline and the output from your terminal.
+      render: console
+      placeholder: |
+        $ nextflow run ...
+
+        Some output where something broke
+
+  - type: textarea
+    id: files
+    attributes:
+      label: Relevant files
+      description: |
+        Please upload (drag and drop) and relevant files. Make into a `.zip` file if the extension is not allowed.
+        Your verbose log file `.nextflow.log` is often useful _(this is a hidden file in the directory where you launched the pipeline)_ as well as custom Nextflow configuration files.
+
+  - type: textarea
+    id: system
+    attributes:
+      label: System information
+      description: |
+        * Nextflow version _(eg. 21.10.3)_
+        * Hardware _(eg. HPC, Desktop, Cloud)_
+        * Executor _(eg. slurm, local, awsbatch)_
+        * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_
+        * OS _(eg. CentOS Linux, macOS, Linux Mint)_
+        * Version of nf-core/fetchngs _(eg. 1.1, 1.5, 1.8.2)_
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
index 7a59f73a..41a8beb7 100644
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,4 +1,3 @@
-blank_issues_enabled: false
 contact_links:
   - name: Join nf-core
     url: https://nf-co.re/join
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
deleted file mode 100644
index c554f29b..00000000
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ /dev/null
@@ -1,32 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for the nf-core/fetchngs pipeline
-labels: enhancement
----
-
-<!--
-# nf-core/fetchngs feature request
-
-Hi there!
-
-Thanks for suggesting a new feature for the pipeline!
-Please delete this text and anything that's not relevant from the template below:
--->
-
-## Is your feature request related to a problem? Please describe
-
-<!-- A clear and concise description of what the problem is. -->
-
-<!-- e.g. [I'm always frustrated when ...] -->
-
-## Describe the solution you'd like
-
-<!-- A clear and concise description of what you want to happen. -->
-
-## Describe alternatives you've considered
-
-<!-- A clear and concise description of any alternative solutions or features you've considered. -->
-
-## Additional context
-
-<!-- Add any other context about the feature request here. -->
diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
new file mode 100644
index 00000000..f8ea51da
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -0,0 +1,11 @@
+name: Feature request
+description: Suggest an idea for the nf-core/fetchngs pipeline
+labels: enhancement
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description of feature
+      description: Please describe your suggestion for a new feature. It might help to describe a problem or use case, plus any alternatives that you have considered.
+    validations:
+      required: true
diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml
index dc26b72d..70f39ce3 100644
--- a/.github/workflows/awsfulltest.yml
+++ b/.github/workflows/awsfulltest.yml
@@ -15,10 +15,7 @@ jobs:
     steps:
       - name: Launch workflow via tower
         uses: nf-core/tower-action@v2
-        # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples)
-        # on the `test_full.config` test runs with only one set of parameters
-        
+
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
diff --git a/.nf-core.yml b/.nf-core.yml
index efb97174..a660c3f8 100644
--- a/.nf-core.yml
+++ b/.nf-core.yml
@@ -1,12 +1,12 @@
 lint:
   files_unchanged:
-    - .github/CONTRIBUTING.md
-    - .github/ISSUE_TEMPLATE/bug_report.md
-    - .github/workflows/linting_comment.yml
+#     - .github/CONTRIBUTING.md
+#     - .github/ISSUE_TEMPLATE/bug_report.md
+#     - .github/workflows/linting_comment.yml
     - assets/sendmail_template.txt
-    - lib/NfcoreSchema.groovy
+#     - lib/NfcoreSchema.groovy
     - lib/NfcoreTemplate.groovy
-  files_exist:
-    - bin/scrape_software_versions.py
-    - modules/local/get_software_versions.nf
-  actions_ci: False
+#   files_exist:
+#     - bin/scrape_software_versions.py
+#     - modules/local/get_software_versions.nf
+#   actions_ci: False
diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy
index dcb39c83..5f681078 100755
--- a/lib/NfcoreSchema.groovy
+++ b/lib/NfcoreSchema.groovy
@@ -105,9 +105,13 @@ class NfcoreSchema {
 
         // Collect expected parameters from the schema
         def expectedParams = []
+        def enums = [:]
         for (group in schemaParams) {
             for (p in group.value['properties']) {
                 expectedParams.push(p.key)
+                if (group.value['properties'][p.key].containsKey('enum')) {
+                    enums[p.key] = group.value['properties'][p.key]['enum']
+                }
             }
         }
 
@@ -155,7 +159,7 @@ class NfcoreSchema {
             println ''
             log.error 'ERROR: Validation of pipeline parameters failed!'
             JSONObject exceptionJSON = e.toJSON()
-            printExceptions(exceptionJSON, params_json, log)
+            printExceptions(exceptionJSON, params_json, log, enums)
             println ''
             has_error = true
         }
@@ -329,7 +333,7 @@ class NfcoreSchema {
     //
     // Loop over nested exceptions and print the causingException
     //
-    private static void printExceptions(ex_json, params_json, log) {
+    private static void printExceptions(ex_json, params_json, log, enums, limit=5) {
         def causingExceptions = ex_json['causingExceptions']
         if (causingExceptions.length() == 0) {
             def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/
@@ -345,7 +349,16 @@ class NfcoreSchema {
             else {
                 def param = ex_json['pointerToViolation'] - ~/^#\//
                 def param_val = params_json[param].toString()
-                log.error "* --${param}: ${ex_json['message']} (${param_val})"
+                if (enums.containsKey(param)) {
+                    def error_msg = "* --${param}: '${param_val}' is not a valid choice (Available choices"
+                    if (enums[param].size() > limit) {
+                        log.error "${error_msg} (${limit} of ${enums[param].size()}): ${enums[param][0..limit-1].join(', ')}, ... )"
+                    } else {
+                        log.error "${error_msg}: ${enums[param].join(', ')})"
+                    }
+                } else {
+                    log.error "* --${param}: ${ex_json['message']} (${param_val})"
+                }
             }
         }
         for (ex in causingExceptions) {

From 881e1f897c5d7fdbb39afabd4f5d31d9992b7502 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Sun, 28 Nov 2021 20:14:39 +0000
Subject: [PATCH 17/24] Fix nf-core lint

---
 .nf-core.yml | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/.nf-core.yml b/.nf-core.yml
index a660c3f8..27864ada 100644
--- a/.nf-core.yml
+++ b/.nf-core.yml
@@ -1,12 +1,14 @@
 lint:
   files_unchanged:
-#     - .github/CONTRIBUTING.md
-#     - .github/ISSUE_TEMPLATE/bug_report.md
-#     - .github/workflows/linting_comment.yml
+    - .gitattributes
+    - .github/ISSUE_TEMPLATE/config.yml
+    - .github/workflows/linting_comment.yml
     - assets/sendmail_template.txt
-#     - lib/NfcoreSchema.groovy
+    - lib/NfcoreSchema.groovy
     - lib/NfcoreTemplate.groovy
-#   files_exist:
-#     - bin/scrape_software_versions.py
-#     - modules/local/get_software_versions.nf
-#   actions_ci: False
+  files_exist:
+    - .github/ISSUE_TEMPLATE/bug_report.md
+    - .github/ISSUE_TEMPLATE/feature_request.md
+    - bin/scrape_software_versions.py
+    - modules/local/get_software_versions.nf
+  actions_ci: False

From eab36c98b6362ecf810ec0dbed06423c8766a334 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Mon, 29 Nov 2021 10:51:32 +0000
Subject: [PATCH 18/24] Address review comments

---
 .github/CONTRIBUTING.md | 36 +++++++-----------------------------
 1 file changed, 7 insertions(+), 29 deletions(-)

diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 1d7943a5..e401c046 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -70,14 +70,13 @@ If you wish to contribute a new step, please use the following coding standards:
 3. Define the output channel if needed (see below).
 4. Add any new flags/options to `nextflow.config` with a default (see below).
 5. Add any new flags/options to `nextflow_schema.json` with help text (with `nf-core schema build`).
-6. Add any new flags/options to the help message (for integer/text parameters, print to help the corresponding `nextflow.config` parameter).
-7. Add sanity checks for all relevant parameters.
-8. Add any new software to the `scrape_software_versions.py` script in `bin/` and the version command to the `scrape_software_versions` process in `main.nf`.
-9. Do local tests that the new code works properly and as expected.
-10. Add a new test command in `.github/workflow/ci.yml`.
-11. If applicable add a [MultiQC](https://https://multiqc.info/) module.
-12. Update MultiQC config `assets/multiqc_config.yaml` so relevant suffixes, name clean up, General Statistics Table column order, and module figures are in the right order.
-13. Optional: Add any descriptions of MultiQC report sections and output files to `docs/output.md`.
+6. Add sanity checks for all relevant parameters.
+7. Add any new software to the `scrape_software_versions.py` script in `bin/` and the version command to the `scrape_software_versions` process in `main.nf`.
+8. Perform local tests to validate that the new code works as expected.
+9. Add a new test command in `.github/workflow/ci.yml`.
+10. If applicable add a [MultiQC](https://https://multiqc.info/) module.
+11. Update MultiQC config `assets/multiqc_config.yaml` so relevant suffixes, name clean up, General Statistics Table column order, and module figures are in the right order.
+12. Optional: Add any descriptions of MultiQC report sections and output files to `docs/output.md`.
 
 ### Default values
 
@@ -102,27 +101,6 @@ Please use the following naming schemes, to make it easy to understand what is g
 
 If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core bump-version --nextflow . [min-nf-version]`
 
-### Software version reporting
-
-If you add a new tool to the pipeline, please ensure you add the information of the tool to the `get_software_version` process.
-
-Add to the script block of the process, something like the following:
-
-```bash
-<YOUR_TOOL> --version &> v_<YOUR_TOOL>.txt 2>&1 || true
-```
-
-or
-
-```bash
-<YOUR_TOOL> --help | head -n 1 &> v_<YOUR_TOOL>.txt 2>&1 || true
-```
-
-You then need to edit the script `bin/scrape_software_versions.py` to:
-
-1. Add a Python regex for your tool's `--version` output (as in stored in the `v_<YOUR_TOOL>.txt` file), to ensure the version is reported as a `v` and the version number e.g. `v2.1.1`
-2. Add a HTML entry to the `OrderedDict` for formatting in MultiQC.
-
 ### Images and figures
 
 For overview images and other documents we follow the nf-core [style guidelines and examples](https://nf-co.re/developers/design_guidelines).

From 0c5a67b2b0e7f4620d3fa3253fa48e89776e8bf0 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Mon, 29 Nov 2021 10:55:09 +0000
Subject: [PATCH 19/24] Fix nf-core lint

---
 .nf-core.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.nf-core.yml b/.nf-core.yml
index 27864ada..4403ba5c 100644
--- a/.nf-core.yml
+++ b/.nf-core.yml
@@ -3,6 +3,7 @@ lint:
     - .gitattributes
     - .github/ISSUE_TEMPLATE/config.yml
     - .github/workflows/linting_comment.yml
+    - .github/CONTRIBUTING.md
     - assets/sendmail_template.txt
     - lib/NfcoreSchema.groovy
     - lib/NfcoreTemplate.groovy

From 541ab9a8cd31c713570174e3426e554013b89e5e Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Mon, 29 Nov 2021 17:41:02 +0000
Subject: [PATCH 20/24] Change ext.suffix to ext.prefix

---
 conf/modules.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conf/modules.config b/conf/modules.config
index c08c1fe6..eb3c936c 100644
--- a/conf/modules.config
+++ b/conf/modules.config
@@ -6,7 +6,7 @@
         ext.args            = Additional arguments appended to command in module.
         ext.args2           = Second set of arguments appended to command in module (multi-tool modules).
         ext.args3           = Third set of arguments appended to command in module (multi-tool modules).
-        ext.suffix          = File name suffix for output files.
+        ext.prefix          = File name prefix for output files.
 ----------------------------------------------------------------------------------------
 */
 

From d1602635309d835951f502b2e5ca3408ace637ec Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Tue, 30 Nov 2021 09:19:00 +0000
Subject: [PATCH 21/24] Update date in CHANGELOG

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37feff4a..056c3602 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,7 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5)] - 2021-11-29
+## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5)] - 2021-11-30
 
 * Finish porting the pipeline to the updated Nextflow DSL2 syntax adopted on nf-core/modules
     * Bump minimum Nextflow version from `21.04.0` -> `21.10.3`

From 8688a82879cd41c13cfa6a2689d554ab8f63ea5e Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Tue, 30 Nov 2021 09:43:29 +0000
Subject: [PATCH 22/24] Update contributing bullets

---
 .github/CONTRIBUTING.md | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index e401c046..9aa27773 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -68,15 +68,13 @@ If you wish to contribute a new step, please use the following coding standards:
 1. Define the corresponding input channel into your new process from the expected previous process channel
 2. Write the process block (see below).
 3. Define the output channel if needed (see below).
-4. Add any new flags/options to `nextflow.config` with a default (see below).
-5. Add any new flags/options to `nextflow_schema.json` with help text (with `nf-core schema build`).
-6. Add sanity checks for all relevant parameters.
-7. Add any new software to the `scrape_software_versions.py` script in `bin/` and the version command to the `scrape_software_versions` process in `main.nf`.
-8. Perform local tests to validate that the new code works as expected.
-9. Add a new test command in `.github/workflow/ci.yml`.
-10. If applicable add a [MultiQC](https://https://multiqc.info/) module.
-11. Update MultiQC config `assets/multiqc_config.yaml` so relevant suffixes, name clean up, General Statistics Table column order, and module figures are in the right order.
-12. Optional: Add any descriptions of MultiQC report sections and output files to `docs/output.md`.
+4. Add any new parameters to `nextflow.config` with a default (see below).
+5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core schema build` tool).
+6. Add sanity checks and validation for all relevant parameters.
+7. Perform local tests to validate that the new code works as expected.
+8. If applicable, add a new test command in `.github/workflow/ci.yml`.
+9. Update MultiQC config `assets/multiqc_config.yaml` so relevant suffixes, file name clean up and module plots are in the appropriate order. If applicable, add a [MultiQC](https://https://multiqc.info/) module.
+10. Add a description of the output files and if relevant any appropriate images from the MultiQC report to `docs/output.md`.
 
 ### Default values
 

From ad5c88a537ebd6f57a504ae5b6556bcfc3166dac Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelhh@gmail.com>
Date: Tue, 30 Nov 2021 11:34:34 +0000
Subject: [PATCH 23/24] Add pre-run script with min NF version for Tower Action

---
 .github/workflows/awsfulltest.yml | 1 +
 .github/workflows/awstest.yml     | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml
index 70f39ce3..83e2793d 100644
--- a/.github/workflows/awsfulltest.yml
+++ b/.github/workflows/awsfulltest.yml
@@ -28,4 +28,5 @@ jobs:
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/fetchngs/results-${{ github.sha }}"
             }
           profiles: test_full,aws_tower
+          pre_run_script: 'export NXF_VER=21.10.3'
 
diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml
index df53dcf3..7cf6d570 100644
--- a/.github/workflows/awstest.yml
+++ b/.github/workflows/awstest.yml
@@ -12,7 +12,7 @@ jobs:
     steps:
       - name: Launch workflow via tower
         uses: nf-core/tower-action@v2
-        
+
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
@@ -25,4 +25,5 @@ jobs:
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/fetchngs/results-${{ github.sha }}"
             }
           profiles: test,aws_tower
+          pre_run_script: 'export NXF_VER=21.10.3'
 

From 187c1169038228d45a13a06c405cca57178f4867 Mon Sep 17 00:00:00 2001
From: Harshil Patel <drpatelh@users.noreply.github.com>
Date: Wed, 1 Dec 2021 08:08:34 +0000
Subject: [PATCH 24/24] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 056c3602..edcf2fa5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,7 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5)] - 2021-11-30
+## [[1.5](https://github.com/nf-core/fetchngs/releases/tag/1.5)] - 2021-12-01
 
 * Finish porting the pipeline to the updated Nextflow DSL2 syntax adopted on nf-core/modules
     * Bump minimum Nextflow version from `21.04.0` -> `21.10.3`