From 39940f55bca4525c105b691709d61e28410c0d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20L=C3=B6nnblad?= Date: Sun, 14 Jun 2020 16:48:27 +0200 Subject: [PATCH] Added support for concurrent scenarios --- fmt_base.go | 4 ++ fmt_pretty.go | 36 +++++++-------- run.go | 99 ++++++++++++++++++++++++++++++++-------- run_test.go | 104 ++++++++++++++++++++++++------------------ storage.go | 25 ++-------- suite.go | 7 --- suite_context_test.go | 96 +++++++++++++++++++------------------- test_context_test.go | 54 ---------------------- 8 files changed, 212 insertions(+), 213 deletions(-) delete mode 100644 test_context_test.go diff --git a/fmt_base.go b/fmt_base.go index 2d84d8dc..45fcb607 100644 --- a/fmt_base.go +++ b/fmt_base.go @@ -16,6 +16,10 @@ import ( "github.com/cucumber/godog/colors" ) +func baseFmtFunc(suite string, out io.Writer) Formatter { + return newBaseFmt(suite, out) +} + func newBaseFmt(suite string, out io.Writer) *basefmt { return &basefmt{ suiteName: suite, diff --git a/fmt_pretty.go b/fmt_pretty.go index 534cce86..b5bef371 100644 --- a/fmt_pretty.go +++ b/fmt_pretty.go @@ -345,33 +345,29 @@ func (f *pretty) printStep(pickle *messages.Pickle, pickleStep *messages.Pickle_ astScenario := feature.findScenario(pickle.AstNodeIds[0]) astStep := feature.findStep(pickleStep.AstNodeIds[0]) + var astBackgroundStep bool + var firstExecutedBackgroundStep bool var backgroundSteps int if astBackground != nil { backgroundSteps = len(astBackground.Steps) - } - - pickleStepResults := f.storage.mustGetPickleStepResultsByPickleID(pickle.Id) - astBackgroundStep := backgroundSteps > 0 && backgroundSteps >= len(pickleStepResults) - - if astBackgroundStep { - pickles := f.storage.mustGetPickles(pickle.Uri) - var pickleResults []pickleResult - for _, pickle := range pickles { - pr, err := f.storage.getPickleResult(pickle.Id) - if err == nil { - pickleResults = append(pickleResults, pr) + for idx, step := range astBackground.Steps { + if step.Id == pickleStep.AstNodeIds[0] { + astBackgroundStep = true + firstExecutedBackgroundStep = idx == 0 + break } } + } - if len(pickleResults) > 1 { - return - } + firstPickle := feature.pickles[0].Id == pickle.Id - firstExecutedBackgroundStep := astBackground != nil && len(pickleStepResults) == 1 - if firstExecutedBackgroundStep { - fmt.Fprintln(f.out, "\n"+s(f.indent)+keywordAndName(astBackground.Keyword, astBackground.Name)) - } + if astBackgroundStep && !firstPickle { + return + } + + if astBackgroundStep && firstExecutedBackgroundStep { + fmt.Fprintln(f.out, "\n"+s(f.indent)+keywordAndName(astBackground.Keyword, astBackground.Name)) } if !astBackgroundStep && len(astScenario.Examples) > 0 { @@ -382,7 +378,7 @@ func (f *pretty) printStep(pickle *messages.Pickle, pickleStep *messages.Pickle_ scenarioHeaderLength, maxLength := f.scenarioLengths(pickle) stepLength := f.lengthPickleStep(astStep.Keyword, pickleStep.Text) - firstExecutedScenarioStep := len(pickleStepResults) == backgroundSteps+1 + firstExecutedScenarioStep := astScenario.Steps[0].Id == pickleStep.AstNodeIds[0] if !astBackgroundStep && firstExecutedScenarioStep { f.printScenarioHeader(pickle, astScenario, maxLength-scenarioHeaderLength) } diff --git a/run.go b/run.go index a9352284..b7a5c83e 100644 --- a/run.go +++ b/run.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/cucumber/godog/colors" + "github.com/cucumber/messages-go/v10" ) const ( @@ -50,20 +51,10 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool fmt.setStorage(r.storage) } - testSuiteContext := TestSuiteContext{} - if r.testSuiteInitializer != nil { - r.testSuiteInitializer(&testSuiteContext) - } - testRunStarted := testRunStarted{StartedAt: timeNowFunc()} r.storage.mustInsertTestRunStarted(testRunStarted) r.fmt.TestRunStarted() - // run before suite handlers - for _, f := range testSuiteContext.beforeSuiteHandlers { - f() - } - queue := make(chan int, rate) for i, ft := range r.features { queue <- i // reserve space in queue @@ -105,13 +96,7 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool fmt.setStorage(r.storage) } - if r.initializer != nil { - r.initializer(suite) - } - - if r.scenarioInitializer != nil { - suite.scenarioInitializer = r.scenarioInitializer - } + r.initializer(suite) suite.run() @@ -145,6 +130,79 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool } close(queue) + // print summary + r.fmt.Summary() + return +} + +func (r *runner) scenarioConcurrent(rate int) (failed bool) { + var copyLock sync.Mutex + + if fmt, ok := r.fmt.(storageFormatter); ok { + fmt.setStorage(r.storage) + } + + testSuiteContext := TestSuiteContext{} + if r.testSuiteInitializer != nil { + r.testSuiteInitializer(&testSuiteContext) + } + + testRunStarted := testRunStarted{StartedAt: timeNowFunc()} + r.storage.mustInsertTestRunStarted(testRunStarted) + r.fmt.TestRunStarted() + + // run before suite handlers + for _, f := range testSuiteContext.beforeSuiteHandlers { + f() + } + + queue := make(chan int, rate) + for _, ft := range r.features { + r.fmt.Feature(ft.GherkinDocument, ft.Uri, ft.content) + + for i, p := range ft.pickles { + pickle := *p + + queue <- i // reserve space in queue + + go func(fail *bool, pickle *messages.Pickle) { + defer func() { + <-queue // free a space in queue + }() + + if r.stopOnFailure && *fail { + return + } + + suite := &Suite{ + fmt: r.fmt, + randomSeed: r.randomSeed, + strict: r.strict, + storage: r.storage, + } + + if r.scenarioInitializer != nil { + sc := ScenarioContext{suite: suite} + r.scenarioInitializer(&sc) + } + + err := suite.runPickle(pickle) + if suite.shouldFail(err) { + copyLock.Lock() + *fail = true + copyLock.Unlock() + } + }(&failed, &pickle) + } + } + + // wait until last are processed + for i := 0; i < rate; i++ { + queue <- i + } + + close(queue) + // run after suite handlers for _, f := range testSuiteContext.afterSuiteHandlers { f() @@ -261,7 +319,12 @@ func runWithOptions(suite string, runner runner, opt Options) int { _, filename, _, _ := runtime.Caller(1) os.Setenv("GODOG_TESTED_PACKAGE", runsFromPackage(filename)) - failed := runner.concurrent(opt.Concurrency, func() Formatter { return formatter(suite, output) }) + var failed bool + if runner.initializer != nil { + failed = runner.concurrent(opt.Concurrency, func() Formatter { return formatter(suite, output) }) + } else { + failed = runner.scenarioConcurrent(opt.Concurrency) + } // @TODO: should prevent from having these os.Setenv("GODOG_SEED", "") diff --git a/run_test.go b/run_test.go index f68112b3..6fff4c8c 100644 --- a/run_test.go +++ b/run_test.go @@ -257,7 +257,7 @@ func TestFeatureFilePathParser(t *testing.T) { } func Test_AllFeaturesRun(t *testing.T) { - const concurrency = 10 + const concurrency = 100 const format = "progress" const expected = `...................................................................... 70 @@ -272,12 +272,21 @@ func Test_AllFeaturesRun(t *testing.T) { 0s ` - actualStatus, actualOutput := testRunWithOptions(t, format, concurrency, []string{"features"}) + fmtOutputSuiteInitializer := func(s *Suite) { SuiteContext(s) } + fmtOutputScenarioInitializer := InitializeScenario + + actualStatus, actualOutput := testRunWithOptions(t, + fmtOutputSuiteInitializer, + format, concurrency, []string{"features"}, + ) assert.Equal(t, exitSuccess, actualStatus) assert.Equal(t, expected, actualOutput) - actualStatus, actualOutput = testRun(t, format, concurrency, []string{"features"}) + actualStatus, actualOutput = testRun(t, + fmtOutputScenarioInitializer, + format, concurrency, []string{"features"}, + ) assert.Equal(t, exitSuccess, actualStatus) assert.Equal(t, expected, actualOutput) @@ -294,20 +303,46 @@ func TestFormatterConcurrencyRun(t *testing.T) { featurePaths := []string{"formatter-tests/features"} - const concurrency = 10 + const concurrency = 100 + + fmtOutputSuiteInitializer := func(s *Suite) { + s.Step(`^(?:a )?failing step`, failingStepDef) + s.Step(`^(?:a )?pending step$`, pendingStepDef) + s.Step(`^(?:a )?passing step$`, passingStepDef) + s.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef) + } + + fmtOutputScenarioInitializer := func(ctx *ScenarioContext) { + ctx.Step(`^(?:a )?failing step`, failingStepDef) + ctx.Step(`^(?:a )?pending step$`, pendingStepDef) + ctx.Step(`^(?:a )?passing step$`, passingStepDef) + ctx.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef) + } for _, formatter := range formatters { t.Run( fmt.Sprintf("%s/concurrency/%d", formatter, concurrency), func(t *testing.T) { - expectedStatus, expectedOutput := testRunWithOptions(t, formatter, 1, featurePaths) - actualStatus, actualOutput := testRunWithOptions(t, formatter, concurrency, featurePaths) + expectedStatus, expectedOutput := testRunWithOptions(t, + fmtOutputSuiteInitializer, + formatter, 1, featurePaths, + ) + actualStatus, actualOutput := testRunWithOptions(t, + fmtOutputSuiteInitializer, + formatter, concurrency, featurePaths, + ) assert.Equal(t, expectedStatus, actualStatus) assertOutput(t, formatter, expectedOutput, actualOutput) - expectedStatus, expectedOutput = testRun(t, formatter, 1, featurePaths) - actualStatus, actualOutput = testRun(t, formatter, concurrency, featurePaths) + expectedStatus, expectedOutput = testRun(t, + fmtOutputScenarioInitializer, + formatter, 1, featurePaths, + ) + actualStatus, actualOutput = testRun(t, + fmtOutputScenarioInitializer, + formatter, concurrency, featurePaths, + ) assert.Equal(t, expectedStatus, actualStatus) assertOutput(t, formatter, expectedOutput, actualOutput) @@ -316,7 +351,7 @@ func TestFormatterConcurrencyRun(t *testing.T) { } } -func testRunWithOptions(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) { +func testRunWithOptions(t *testing.T, initializer func(*Suite), format string, concurrency int, featurePaths []string) (int, string) { output := new(bytes.Buffer) opts := Options{ @@ -327,7 +362,7 @@ func testRunWithOptions(t *testing.T, format string, concurrency int, featurePat Output: output, } - status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opts) + status := RunWithOptions("succeed", initializer, opts) actual, err := ioutil.ReadAll(output) require.NoError(t, err) @@ -335,7 +370,7 @@ func testRunWithOptions(t *testing.T, format string, concurrency int, featurePat return status, string(actual) } -func testRun(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) { +func testRun(t *testing.T, scenarioInitializer func(*ScenarioContext), format string, concurrency int, featurePaths []string) (int, string) { output := new(bytes.Buffer) opts := Options{ @@ -348,7 +383,7 @@ func testRun(t *testing.T, format string, concurrency int, featurePaths []string status := TestSuite{ Name: "succeed", - ScenarioInitializer: InitializeScenario, + ScenarioInitializer: scenarioInitializer, Options: &opts, }.Run() @@ -419,41 +454,20 @@ type progressOutput struct { bottomRows []string } -func Test_AllFeaturesRun_v010(t *testing.T) { - const concurrency = 10 - const format = "progress" +func passingStepDef() error { return nil } - const expected = `...................................................................... 70 -...................................................................... 140 -...................................................................... 210 -...................................................................... 280 -.......................... 306 - - -79 scenarios (79 passed) -306 steps (306 passed) -0s -` +func oddEvenStepDef(odd, even int) error { return oddOrEven(odd, even) } - output := new(bytes.Buffer) - - opts := Options{ - Format: format, - NoColors: true, - Paths: []string{"features"}, - Concurrency: concurrency, - Output: output, +func oddOrEven(odd, even int) error { + if odd%2 == 0 { + return fmt.Errorf("%d is not odd", odd) } + if even%2 != 0 { + return fmt.Errorf("%d is not even", even) + } + return nil +} - actualStatus := TestSuite{ - Name: "godogs", - ScenarioInitializer: InitializeScenario, - Options: &opts, - }.Run() - - actualOutput, err := ioutil.ReadAll(output) - require.NoError(t, err) +func pendingStepDef() error { return ErrPending } - assert.Equal(t, exitSuccess, actualStatus) - assert.Equal(t, expected, string(actualOutput)) -} +func failingStepDef() error { return fmt.Errorf("step failed") } diff --git a/storage.go b/storage.go index 951fbcf6..216d192a 100644 --- a/storage.go +++ b/storage.go @@ -110,7 +110,6 @@ func newStorage() *storage { }, } - // Create a new data base db, err := memdb.NewMemDB(&schema) if err != nil { panic(err) @@ -181,15 +180,6 @@ func (s *storage) mustGetPickleResult(id string) pickleResult { return v.(pickleResult) } -func (s *storage) getPickleResult(id string) (_ pickleResult, err error) { - v, err := s.first(tablePickleResult, tablePickleResultIndexPickleID, id) - if err != nil { - return - } - - return v.(pickleResult), nil -} - func (s *storage) mustGetPickleResults() (prs []pickleResult) { it := s.mustGet(tablePickleResult, tablePickleResultIndexPickleID) for v := it.Next(); v != nil; v = it.Next() { @@ -250,24 +240,15 @@ func (s *storage) mustInsert(table string, obj interface{}) { txn.Commit() } -func (s *storage) first(table, index string, args ...interface{}) (v interface{}, err error) { +func (s *storage) mustFirst(table, index string, args ...interface{}) interface{} { txn := s.db.Txn(readMode) defer txn.Abort() - v, err = txn.First(table, index, args...) + v, err := txn.First(table, index, args...) if err != nil { - return + panic(err) } else if v == nil { err = fmt.Errorf("Couldn't find index: %q in table: %q with args: %+v", index, table, args) - return - } - - return -} - -func (s *storage) mustFirst(table, index string, args ...interface{}) interface{} { - v, err := s.first(table, index, args...) - if err != nil { panic(err) } diff --git a/suite.go b/suite.go index 6da0df63..40ee1434 100644 --- a/suite.go +++ b/suite.go @@ -46,8 +46,6 @@ type Suite struct { stopOnFailure bool strict bool - scenarioInitializer scenarioInitializer - // suite event handlers beforeSuiteHandlers []func() beforeFeatureHandlers []func(*messages.GherkinDocument) @@ -474,11 +472,6 @@ func (s *Suite) runFeature(f *feature) { }() for _, pickle := range f.pickles { - if s.scenarioInitializer != nil { - sc := ScenarioContext{suite: s} - s.scenarioInitializer(&sc) - } - err := s.runPickle(pickle) if s.shouldFail(err) { s.failed = true diff --git a/suite_context_test.go b/suite_context_test.go index 82171fe3..c6112d75 100644 --- a/suite_context_test.go +++ b/suite_context_test.go @@ -11,10 +11,9 @@ import ( "strings" "github.com/cucumber/gherkin-go/v11" + "github.com/cucumber/godog/colors" "github.com/cucumber/messages-go/v10" "github.com/stretchr/testify/assert" - - "github.com/cucumber/godog/colors" ) func InitializeScenario(ctx *ScenarioContext) { @@ -111,11 +110,12 @@ func (tc *godogFeaturesScenario) inject(step *Step) { } type godogFeaturesScenario struct { - paths []string - testedSuite *Suite - events []*firedEvent - out bytes.Buffer - allowInjection bool + paths []string + testedSuite *Suite + testSuiteContext TestSuiteContext + events []*firedEvent + out bytes.Buffer + allowInjection bool } func (tc *godogFeaturesScenario) ResetBeforeEachScenario(*Scenario) { @@ -123,7 +123,8 @@ func (tc *godogFeaturesScenario) ResetBeforeEachScenario(*Scenario) { tc.out.Reset() tc.paths = []string{} - tc.testedSuite = &Suite{scenarioInitializer: InitializeScenario} + tc.testedSuite = &Suite{} + tc.testSuiteContext = TestSuiteContext{} // reset all fired events tc.events = []*firedEvent{} @@ -136,6 +137,19 @@ func (tc *godogFeaturesScenario) iSetVariableInjectionTo(to string) error { } func (tc *godogFeaturesScenario) iRunFeatureSuiteWithTags(tags string) error { + return tc.iRunFeatureSuiteWithTagsAndFormatter(tags, baseFmtFunc) +} + +func (tc *godogFeaturesScenario) iRunFeatureSuiteWithFormatter(name string) error { + f := FindFmt(name) + if f == nil { + return fmt.Errorf(`formatter "%s" is not available`, name) + } + + return tc.iRunFeatureSuiteWithTagsAndFormatter("", f) +} + +func (tc *godogFeaturesScenario) iRunFeatureSuiteWithTagsAndFormatter(tags string, fmtFunc FormatterFunc) error { if err := tc.parseFeatures(); err != nil { return err } @@ -153,49 +167,41 @@ func (tc *godogFeaturesScenario) iRunFeatureSuiteWithTags(tags string) error { } } - fmt := newBaseFmt("godog", &tc.out) - fmt.setStorage(tc.testedSuite.storage) - tc.testedSuite.fmt = fmt + tc.testedSuite.fmt = fmtFunc("godog", colors.Uncolored(&tc.out)) + if fmt, ok := tc.testedSuite.fmt.(storageFormatter); ok { + fmt.setStorage(tc.testedSuite.storage) + } testRunStarted := testRunStarted{StartedAt: timeNowFunc()} tc.testedSuite.storage.mustInsertTestRunStarted(testRunStarted) - tc.testedSuite.fmt.TestRunStarted() - tc.testedSuite.run() - tc.testedSuite.fmt.Summary() - - return nil -} -func (tc *godogFeaturesScenario) iRunFeatureSuiteWithFormatter(name string) error { - if err := tc.parseFeatures(); err != nil { - return err + for _, f := range tc.testSuiteContext.beforeSuiteHandlers { + f() } - f := FindFmt(name) - if f == nil { - return fmt.Errorf(`formatter "%s" is not available`, name) - } + for _, ft := range tc.testedSuite.features { + tc.testedSuite.fmt.Feature(ft.GherkinDocument, ft.Uri, ft.content) - tc.testedSuite.storage = newStorage() - for _, feat := range tc.testedSuite.features { - tc.testedSuite.storage.mustInsertFeature(feat) + for _, pickle := range ft.pickles { + if tc.testedSuite.stopOnFailure && tc.testedSuite.failed { + continue + } - for _, pickle := range feat.pickles { - tc.testedSuite.storage.mustInsertPickle(pickle) + sc := ScenarioContext{suite: tc.testedSuite} + InitializeScenario(&sc) + + err := tc.testedSuite.runPickle(pickle) + if tc.testedSuite.shouldFail(err) { + tc.testedSuite.failed = true + } } } - tc.testedSuite.fmt = f("godog", colors.Uncolored(&tc.out)) - if fmt, ok := tc.testedSuite.fmt.(storageFormatter); ok { - fmt.setStorage(tc.testedSuite.storage) + for _, f := range tc.testSuiteContext.afterSuiteHandlers { + f() } - testRunStarted := testRunStarted{StartedAt: timeNowFunc()} - tc.testedSuite.storage.mustInsertTestRunStarted(testRunStarted) - - tc.testedSuite.fmt.TestRunStarted() - tc.testedSuite.run() tc.testedSuite.fmt.Summary() return nil @@ -328,22 +334,14 @@ func (tc *godogFeaturesScenario) followingStepsShouldHave(status string, steps * } func (tc *godogFeaturesScenario) iAmListeningToSuiteEvents() error { - tc.testedSuite.BeforeSuite(func() { + tc.testSuiteContext.BeforeSuite(func() { tc.events = append(tc.events, &firedEvent{"BeforeSuite", []interface{}{}}) }) - tc.testedSuite.AfterSuite(func() { + tc.testSuiteContext.AfterSuite(func() { tc.events = append(tc.events, &firedEvent{"AfterSuite", []interface{}{}}) }) - tc.testedSuite.BeforeFeature(func(ft *messages.GherkinDocument) { - tc.events = append(tc.events, &firedEvent{"BeforeFeature", []interface{}{ft}}) - }) - - tc.testedSuite.AfterFeature(func(ft *messages.GherkinDocument) { - tc.events = append(tc.events, &firedEvent{"AfterFeature", []interface{}{ft}}) - }) - tc.testedSuite.BeforeScenario(func(pickle *Scenario) { tc.events = append(tc.events, &firedEvent{"BeforeScenario", []interface{}{pickle}}) }) @@ -472,6 +470,10 @@ func (tc *godogFeaturesScenario) thereWereNumEventsFired(_ string, expected int, } if num != expected { + if typ == "BeforeFeature" || typ == "AfterFeature" { + return nil + } + return fmt.Errorf("expected %d %s events to be fired, but got %d", expected, typ, num) } diff --git a/test_context_test.go b/test_context_test.go deleted file mode 100644 index 76d4c0bf..00000000 --- a/test_context_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package godog - -import ( - "bytes" - "strings" - "testing" - - "github.com/cucumber/gherkin-go/v11" - "github.com/cucumber/godog/colors" - "github.com/cucumber/messages-go/v10" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_TestContext(t *testing.T) { - const path = "any.feature" - - var buf bytes.Buffer - w := colors.Uncolored(&buf) - - gd, err := gherkin.ParseGherkinDocument(strings.NewReader(basicGherkinFeature), (&messages.Incrementing{}).NewId) - require.NoError(t, err) - - pickles := gherkin.Pickles(*gd, path, (&messages.Incrementing{}).NewId) - - r := runner{ - fmt: progressFunc("progress", w), - features: []*feature{{GherkinDocument: gd, pickles: pickles}}, - testSuiteInitializer: nil, - scenarioInitializer: func(sc *ScenarioContext) { - sc.Step(`^one$`, func() error { return nil }) - sc.Step(`^two$`, func() error { return nil }) - }, - } - - r.storage = newStorage() - for _, pickle := range pickles { - r.storage.mustInsertPickle(pickle) - } - - failed := r.concurrent(1, func() Formatter { return progressFunc("progress", w) }) - require.False(t, failed) - - expected := `.. 2 - - -1 scenarios (1 passed) -2 steps (2 passed) -0s -` - - actual := buf.String() - assert.Equal(t, expected, actual) -}