From a12c5a3b2452de4b16fb1d03b91f85ea81eacd0c Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:51:54 +0200 Subject: [PATCH 1/2] Use nf-core nfvalidation subworkflow --- modules.json | 5 + .../local/nf_core_fetchngs_utils/main.nf | 4 +- .../local/nf_validation_plugin_utils/main.nf | 52 ----- .../nf-core/nfvalidation_plugin_utils/main.nf | 62 ++++++ .../nfvalidation_plugin_utils/meta.yml | 49 +++++ .../tests/main.nf.test | 199 ++++++++++++++++++ .../tests/main.nf.test.snap | 26 +++ .../tests/nextflow_schema.json | 96 +++++++++ 8 files changed, 439 insertions(+), 54 deletions(-) delete mode 100644 subworkflows/local/nf_validation_plugin_utils/main.nf create mode 100644 subworkflows/nf-core/nfvalidation_plugin_utils/main.nf create mode 100644 subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml create mode 100644 subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test create mode 100644 subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap create mode 100644 subworkflows/nf-core/nfvalidation_plugin_utils/tests/nextflow_schema.json diff --git a/modules.json b/modules.json index 3cc9d6dd..b8801dc6 100644 --- a/modules.json +++ b/modules.json @@ -38,6 +38,11 @@ "branch": "master", "git_sha": "76cbb00305f6b045c15c1e202a5bdd516d6f64a3", "installed_by": ["subworkflows"] + }, + "nfvalidation_plugin_utils": { + "branch": "master", + "git_sha": "a8223a72def6cc46f4652a348feba21845ce68c6", + "installed_by": ["subworkflows"] } } } diff --git a/subworkflows/local/nf_core_fetchngs_utils/main.nf b/subworkflows/local/nf_core_fetchngs_utils/main.nf index fd0e9855..05f9a2ba 100644 --- a/subworkflows/local/nf_core_fetchngs_utils/main.nf +++ b/subworkflows/local/nf_core_fetchngs_utils/main.nf @@ -9,8 +9,8 @@ */ include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../../../modules/nf-core/custom/dumpsoftwareversions' -include { NEXTFLOW_PIPELINE_UTILS; getWorkflowVersion } from '../../nf-core/nextflowpipelineutils/main' -include { NF_VALIDATION_PLUGIN_UTILS } from '../nf_validation_plugin_utils' +include { NEXTFLOW_PIPELINE_UTILS; getWorkflowVersion } from '../../nf-core/nextflowpipelineutils/main' +include { NF_VALIDATION_PLUGIN_UTILS } from '../../nf-core/nfvalidation_plugin_utils/main.nf' include { NF_CORE_PIPELINE_UTILS; workflowCitation; nfCoreLogo; dashedLine; completionEmail; completionSummary; imNotification } from '../nf_core_pipeline_utils' /* diff --git a/subworkflows/local/nf_validation_plugin_utils/main.nf b/subworkflows/local/nf_validation_plugin_utils/main.nf deleted file mode 100644 index be815a7f..00000000 --- a/subworkflows/local/nf_validation_plugin_utils/main.nf +++ /dev/null @@ -1,52 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsHelp; paramsSummaryLog; paramsSummaryMap; validateParameters } from 'plugin/nf-validation' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow NF_VALIDATION_PLUGIN_UTILS { - - take: - print_help // bool - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log - validate_params // bool - - main: - - // - // Print help message if needed - // - if (print_help && workflow_command) { - log.info pre_help_text + paramsHelp(workflow_command) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow) + post_help_text - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters() - } - - emit: - summary_params = paramsSummaryMap(workflow) -} diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/main.nf b/subworkflows/nf-core/nfvalidation_plugin_utils/main.nf new file mode 100644 index 00000000..22531719 --- /dev/null +++ b/subworkflows/nf-core/nfvalidation_plugin_utils/main.nf @@ -0,0 +1,62 @@ +// +// Subworkflow that uses the nf-validation plugin to render help text and parameter summary +// + +/* +======================================================================================== + IMPORT NF-VALIDATION PLUGIN +======================================================================================== +*/ + +include { paramsHelp; paramsSummaryLog; paramsSummaryMap; validateParameters } from 'plugin/nf-validation' + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow NF_VALIDATION_PLUGIN_UTILS { + + take: + print_help // bool + workflow_command // string: default commmand used to run pipeline + pre_help_text // string: string to be printed before help text and summary log + post_help_text // string: string to be printed after help text and summary log + validate_params // bool: Validate parameters + schema_filename // path: JSON schema file, null to use default value + + main: + + log.debug "Using schema file: ${schema_filename}" + + // Default values for strings + pre_help_text = pre_help_text ?: '' + post_help_text = post_help_text ?: '' + workflow_command = workflow_command ?: '' + + // + // Print help message if needed + // + if (print_help) { + log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text + System.exit(0) + } + + // + // Print parameter summary to stdout + // + log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text + + // + // Validate parameters relative to the parameter JSON schema + // + if (validate_params){ + validateParameters(parameters_schema: schema_filename) + } + + summary_params = paramsSummaryMap(workflow, parameters_schema: schema_filename) + + emit: + summary_params = summary_params +} diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml b/subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml new file mode 100644 index 00000000..6117b455 --- /dev/null +++ b/subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "nfvalidationpluginutils" +## TODO nf-core: Add a description of the subworkflow and list keywords +description: Sort SAM/BAM/CRAM file +keywords: + - sort + - bam + - sam + - cram +## TODO nf-core: Add a list of the modules and/or subworkflows used in the subworkflow +components: + - samtools/sort + - samtools/index +## TODO nf-core: List all of the channels used as input with a description and their structure +input: + - ch_bam: + type: file + description: | + The input channel containing the BAM/CRAM/SAM files + Structure: [ val(meta), path(bam) ] + pattern: "*.{bam/cram/sam}" +## TODO nf-core: List all of the channels used as output with a descriptions and their structure +output: + - bam: + type: file + description: | + Channel containing BAM files + Structure: [ val(meta), path(bam) ] + pattern: "*.bam" + - bai: + type: file + description: | + Channel containing indexed BAM (BAI) files + Structure: [ val(meta), path(bai) ] + pattern: "*.bai" + - csi: + type: file + description: | + Channel containing CSI files + Structure: [ val(meta), path(csi) ] + pattern: "*.csi" + - versions: + type: file + description: | + File containing software versions + Structure: [ path(versions.yml) ] + pattern: "versions.yml" +authors: + - "@adamrtalbot" diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test new file mode 100644 index 00000000..6a0efd1e --- /dev/null +++ b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test @@ -0,0 +1,199 @@ +nextflow_workflow { + + name "Test Workflow NF_VALIDATION_PLUGIN_UTILS" + script "../main.nf" + workflow "NF_VALIDATION_PLUGIN_UTILS" + tag "subworkflow" + tag "subworkflow_nfcore" + tag "nfvalidationpluginutils" + tag "subworkflows/nfvalidationpluginutils" + + test("Should run nothing") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assert workflow.success + } + + } + + test("Should run help") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assert workflow.success + assert workflow.exitStatus == 0 + assert workflow.stdout.any { it.contains('Input/output options') } + assert workflow.stdout.any { it.contains('--outdir') } + assert workflow.stdout.any { it.contains('--outdir') } + } + + } + + test("Should run help with command") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assert workflow.success + assert workflow.exitStatus == 0 + assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } + assert workflow.stdout.any { it.contains('Input/output options') } + assert workflow.stdout.any { it.contains('--outdir') } + assert workflow.stdout.any { it.contains('--outdir') } + } + + } + + test("Should run help with extra text") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = "pre-help-text" + post_help_text = "post-help-text" + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assert workflow.success + assert workflow.exitStatus == 0 + assert workflow.stdout.any { it.contains('pre-help-text') } + assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } + assert workflow.stdout.any { it.contains('Input/output options') } + assert workflow.stdout.any { it.contains('--outdir') } + assert workflow.stdout.any { it.contains('--outdir') } + assert workflow.stdout.any { it.contains('post-help-text') } + } + + } + + test("Should validate params") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = true + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assert workflow.failed + assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } + assert workflow.stdout.any { it.contains('The following invalid input values have been detected:') } + } + + } + +} diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap new file mode 100644 index 00000000..ae33a930 --- /dev/null +++ b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap @@ -0,0 +1,26 @@ +{ + "Should run help": { + "content": [ + { + "stderr": [ + + ], + "errorReport": "", + "exitStatus": 0, + "failed": false, + "stdout": [ + + ], + "errorMessage": "", + "trace": { + "tasksFailed": 0, + "tasksCount": 0, + "tasksSucceeded": 0 + }, + "name": "workflow", + "success": true + } + ], + "timestamp": "2023-10-13T13:18:16.933251413" + } +} diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/nextflow_schema.json b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/nextflow_schema.json new file mode 100644 index 00000000..7626c1c9 --- /dev/null +++ b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "definitions": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/input_output_options" + }, + { + "$ref": "#/definitions/generic_options" + } + ] +} From 94d75cb01b0545a4cc2edf992e073a83707990fb Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:00:54 +0200 Subject: [PATCH 2/2] fix --- subworkflows/local/nf_core_fetchngs_utils/main.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subworkflows/local/nf_core_fetchngs_utils/main.nf b/subworkflows/local/nf_core_fetchngs_utils/main.nf index 05f9a2ba..f20bdbe7 100644 --- a/subworkflows/local/nf_core_fetchngs_utils/main.nf +++ b/subworkflows/local/nf_core_fetchngs_utils/main.nf @@ -44,7 +44,8 @@ workflow PIPELINE_INITIALISATION { workflow_command, pre_help_text, post_help_text, - params.validate_params + params.validate_params, + "nextflow_schema.json" ) //