From 2333e5f049ef11aeb0fab6f8eab19f4c842e168d Mon Sep 17 00:00:00 2001 From: David Goss Date: Mon, 11 Oct 2021 08:54:40 +0100 Subject: [PATCH] runtime: don't fail the test run for undefined/ambiguous when in dry run (#1814) * update scenario (failing) * clarify * change logic * refactor to share logic across serial+parallel * add changelog * add doco for dry run * add link to changelog entry --- CHANGELOG.md | 2 ++ README.md | 1 + docs/dry_run.md | 20 ++++++++++++++++++++ features/dryrun_mode.feature | 22 ++++++++++++++++++++-- src/runtime/helpers.ts | 18 ++++++++++++++++++ src/runtime/index.ts | 15 ++------------- src/runtime/parallel/coordinator.ts | 11 ++--------- 7 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 docs/dry_run.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b89f226ec..975cb4e98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO ### Removed ### Fixed + +* When running with `--dry-run`, undefined or ambiguous steps no longer cause the process to exit with code 1. ([#1814](https://github.com/cucumber/cucumber-js/pull/1814)) * When running the help command, it now shows all available formatters under the --format option. [#1798](https://github.com/cucumber/cucumber-js/pull/1798) diff --git a/README.md b/README.md index 2075d0ba4..549aa49c8 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ The following documentation is for master. See below the documentation for older * [Attachments](/docs/support_files/attachments.md) * [API Reference](/docs/support_files/api_reference.md) * Guides + * [Dry Run](./docs/dry_run.md) * [ES Modules](./docs/esm.md) * [Formatters](./docs/formatters.md) * [Running in parallel](./docs/parallel.md) diff --git a/docs/dry_run.md b/docs/dry_run.md new file mode 100644 index 000000000..1f0f8ea0b --- /dev/null +++ b/docs/dry_run.md @@ -0,0 +1,20 @@ +# Dry Run + +You can run cucumber-js in "Dry Run" mode like this: + +```shell +$ cucumber-js --dry-run +``` + +The effect is that cucumber-js will still do all the aggregation work of looking at your feature files, loading your support code etc but without actually executing the tests. Specifically: + +- No [hooks](./support_files/hooks.md) are executed +- Steps are reported as "skipped" instead of being executed +- Undefined and ambiguous steps are reported, but don't cause the process to fail + +A few examples where this is useful: + +- Finding unused step definitions with the [usage formatter](./formatters.md#usage) +- Generating [snippets](./snippets.md) for all undefined steps with the [snippets formatter](./formatters.md#snippets) +- Checking if your path, tag expression etc matches the scenarios you expect it to + diff --git a/features/dryrun_mode.feature b/features/dryrun_mode.feature index a0f52ebda..c71ff163c 100644 --- a/features/dryrun_mode.feature +++ b/features/dryrun_mode.feature @@ -18,6 +18,7 @@ Feature: Dryrun mode Given('a step', function() {}) """ When I run cucumber-js with `--dry-run` + And it passes Then scenario "some scenario" step "Given a step" has status "skipped" And scenario "some scenario" has status "skipped" @@ -30,12 +31,29 @@ Feature: Dryrun mode Given('a(n) step', function() {}); """ When I run cucumber-js with `--dry-run` - Then it fails + Then it passes And scenario "some scenario" step "Given a step" has status "ambiguous" + Scenario: pending step + + Since steps aren't actually executed in dry run, a step that would resolve to pending + will still show up as skipped. + + Given a file named "features/step_definitions/cucumber_steps.js" with: + """ + const {Given} = require('@cucumber/cucumber') + + Given('a step', function() { + return 'pending'; + }); + """ + When I run cucumber-js with `--dry-run` + Then it passes + And scenario "some scenario" step "Given a step" has status "skipped" + Scenario: undefined step When I run cucumber-js with `--dry-run` - Then it fails + Then it passes And scenario "some scenario" step "Given a step" has status "undefined" Scenario: hooks should not execute in dry run, serial runtime diff --git a/src/runtime/helpers.ts b/src/runtime/helpers.ts index 3f1558105..5ecc031cb 100644 --- a/src/runtime/helpers.ts +++ b/src/runtime/helpers.ts @@ -63,3 +63,21 @@ export function retriesForPickle( } return 0 } + +export function shouldCauseFailure( + status: messages.TestStepResultStatus, + options: IRuntimeOptions +): boolean { + if (options.dryRun) { + return false + } + const failureStatuses: messages.TestStepResultStatus[] = [ + messages.TestStepResultStatus.AMBIGUOUS, + messages.TestStepResultStatus.FAILED, + messages.TestStepResultStatus.UNDEFINED, + ] + if (options.strict) { + failureStatuses.push(messages.TestStepResultStatus.PENDING) + } + return failureStatuses.includes(status) +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 884a8017d..5efced4ca 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -2,7 +2,7 @@ import { EventDataCollector, formatLocation } from '../formatter/helpers' import StackTraceFilter from '../stack_trace_filter' import UserCodeRunner from '../user_code_runner' import VError from 'verror' -import { retriesForPickle } from './helpers' +import { retriesForPickle, shouldCauseFailure } from './helpers' import { IdGenerator } from '@cucumber/messages' import * as messages from '@cucumber/messages' import TestCaseRunner from './test_case_runner' @@ -109,7 +109,7 @@ export default class Runtime { worldParameters: this.options.worldParameters, }) const status = await testCaseRunner.run() - if (this.shouldCauseFailure(status)) { + if (shouldCauseFailure(status, this.options)) { this.success = false } } @@ -157,15 +157,4 @@ export default class Runtime { } return this.success } - - shouldCauseFailure(status: messages.TestStepResultStatus): boolean { - const failureStatuses: messages.TestStepResultStatus[] = [ - messages.TestStepResultStatus.AMBIGUOUS, - messages.TestStepResultStatus.FAILED, - messages.TestStepResultStatus.UNDEFINED, - ] - if (this.options.strict) - failureStatuses.push(messages.TestStepResultStatus.PENDING) - return failureStatuses.includes(status) - } } diff --git a/src/runtime/parallel/coordinator.ts b/src/runtime/parallel/coordinator.ts index 0cce0aeae..73e840f63 100644 --- a/src/runtime/parallel/coordinator.ts +++ b/src/runtime/parallel/coordinator.ts @@ -1,6 +1,6 @@ import { ChildProcess, fork } from 'child_process' import path from 'path' -import { retriesForPickle } from '../helpers' +import { retriesForPickle, shouldCauseFailure } from '../helpers' import * as messages from '@cucumber/messages' import { EventEmitter } from 'events' import { EventDataCollector } from '../../formatter/helpers' @@ -157,7 +157,7 @@ export default class Coordinator { ) if ( !testCaseFinished.willBeRetried && - this.shouldCauseFailure(worstTestStepResult.status) + shouldCauseFailure(worstTestStepResult.status, this.options) ) { this.success = false } @@ -214,11 +214,4 @@ export default class Coordinator { } worker.process.send(runCommand) } - - shouldCauseFailure(status: messages.TestStepResultStatus): boolean { - return ( - ['AMBIGUOUS', 'FAILED', 'UNDEFINED'].includes(status) || - (status === 'PENDING' && this.options.strict) - ) - } }