Skip to content

Commit

Permalink
Merge pull request #311 from cucumber/feature/concurrent-scenarios
Browse files Browse the repository at this point in the history 
Added support for concurrent scenarios
  • Loading branch information
@lonnblad
lonnblad authored Jun 15, 2020
2 parents 57422f2 + 39940f5 commit 6e01c51
Show file tree
Hide file tree
Showing 8 changed files with 212 additions and 213 deletions.
4 changes: 4 additions & 0 deletions fmt_base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ import (
"github.com/cucumber/godog/colors"
)

func baseFmtFunc(suite string, out io.Writer) Formatter {
return newBaseFmt(suite, out)
}

func newBaseFmt(suite string, out io.Writer) *basefmt {
return &basefmt{
suiteName: suite,
Expand Down
36 changes: 16 additions & 20 deletions fmt_pretty.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,33 +345,29 @@ func (f *pretty) printStep(pickle *messages.Pickle, pickleStep *messages.Pickle_
astScenario := feature.findScenario(pickle.AstNodeIds[0])
astStep := feature.findStep(pickleStep.AstNodeIds[0])

var astBackgroundStep bool
var firstExecutedBackgroundStep bool
var backgroundSteps int
if astBackground != nil {
backgroundSteps = len(astBackground.Steps)
}

pickleStepResults := f.storage.mustGetPickleStepResultsByPickleID(pickle.Id)
astBackgroundStep := backgroundSteps > 0 && backgroundSteps >= len(pickleStepResults)

if astBackgroundStep {
pickles := f.storage.mustGetPickles(pickle.Uri)

var pickleResults []pickleResult
for _, pickle := range pickles {
pr, err := f.storage.getPickleResult(pickle.Id)
if err == nil {
pickleResults = append(pickleResults, pr)
for idx, step := range astBackground.Steps {
if step.Id == pickleStep.AstNodeIds[0] {
astBackgroundStep = true
firstExecutedBackgroundStep = idx == 0
break
}
}
}

if len(pickleResults) > 1 {
return
}
firstPickle := feature.pickles[0].Id == pickle.Id

firstExecutedBackgroundStep := astBackground != nil && len(pickleStepResults) == 1
if firstExecutedBackgroundStep {
fmt.Fprintln(f.out, "\n"+s(f.indent)+keywordAndName(astBackground.Keyword, astBackground.Name))
}
if astBackgroundStep && !firstPickle {
return
}

if astBackgroundStep && firstExecutedBackgroundStep {
fmt.Fprintln(f.out, "\n"+s(f.indent)+keywordAndName(astBackground.Keyword, astBackground.Name))
}

if !astBackgroundStep && len(astScenario.Examples) > 0 {
Expand All @@ -382,7 +378,7 @@ func (f *pretty) printStep(pickle *messages.Pickle, pickleStep *messages.Pickle_
scenarioHeaderLength, maxLength := f.scenarioLengths(pickle)
stepLength := f.lengthPickleStep(astStep.Keyword, pickleStep.Text)

firstExecutedScenarioStep := len(pickleStepResults) == backgroundSteps+1
firstExecutedScenarioStep := astScenario.Steps[0].Id == pickleStep.AstNodeIds[0]
if !astBackgroundStep && firstExecutedScenarioStep {
f.printScenarioHeader(pickle, astScenario, maxLength-scenarioHeaderLength)
}
Expand Down
99 changes: 81 additions & 18 deletions run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"sync"

"github.com/cucumber/godog/colors"
"github.com/cucumber/messages-go/v10"
)

const (
Expand Down Expand Up @@ -50,20 +51,10 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
fmt.setStorage(r.storage)
}

testSuiteContext := TestSuiteContext{}
if r.testSuiteInitializer != nil {
r.testSuiteInitializer(&testSuiteContext)
}

testRunStarted := testRunStarted{StartedAt: timeNowFunc()}
r.storage.mustInsertTestRunStarted(testRunStarted)
r.fmt.TestRunStarted()

// run before suite handlers
for _, f := range testSuiteContext.beforeSuiteHandlers {
f()
}

queue := make(chan int, rate)
for i, ft := range r.features {
queue <- i // reserve space in queue
Expand Down Expand Up @@ -105,13 +96,7 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
fmt.setStorage(r.storage)
}

if r.initializer != nil {
r.initializer(suite)
}

if r.scenarioInitializer != nil {
suite.scenarioInitializer = r.scenarioInitializer
}
r.initializer(suite)

suite.run()

Expand Down Expand Up @@ -145,6 +130,79 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
}
close(queue)

// print summary
r.fmt.Summary()
return
}

func (r *runner) scenarioConcurrent(rate int) (failed bool) {
var copyLock sync.Mutex

if fmt, ok := r.fmt.(storageFormatter); ok {
fmt.setStorage(r.storage)
}

testSuiteContext := TestSuiteContext{}
if r.testSuiteInitializer != nil {
r.testSuiteInitializer(&testSuiteContext)
}

testRunStarted := testRunStarted{StartedAt: timeNowFunc()}
r.storage.mustInsertTestRunStarted(testRunStarted)
r.fmt.TestRunStarted()

// run before suite handlers
for _, f := range testSuiteContext.beforeSuiteHandlers {
f()
}

queue := make(chan int, rate)
for _, ft := range r.features {
r.fmt.Feature(ft.GherkinDocument, ft.Uri, ft.content)

for i, p := range ft.pickles {
pickle := *p

queue <- i // reserve space in queue

go func(fail *bool, pickle *messages.Pickle) {
defer func() {
<-queue // free a space in queue
}()

if r.stopOnFailure && *fail {
return
}

suite := &Suite{
fmt: r.fmt,
randomSeed: r.randomSeed,
strict: r.strict,
storage: r.storage,
}

if r.scenarioInitializer != nil {
sc := ScenarioContext{suite: suite}
r.scenarioInitializer(&sc)
}

err := suite.runPickle(pickle)
if suite.shouldFail(err) {
copyLock.Lock()
*fail = true
copyLock.Unlock()
}
}(&failed, &pickle)
}
}

// wait until last are processed
for i := 0; i < rate; i++ {
queue <- i
}

close(queue)

// run after suite handlers
for _, f := range testSuiteContext.afterSuiteHandlers {
f()
Expand Down Expand Up @@ -261,7 +319,12 @@ func runWithOptions(suite string, runner runner, opt Options) int {
_, filename, _, _ := runtime.Caller(1)
os.Setenv("GODOG_TESTED_PACKAGE", runsFromPackage(filename))

failed := runner.concurrent(opt.Concurrency, func() Formatter { return formatter(suite, output) })
var failed bool
if runner.initializer != nil {
failed = runner.concurrent(opt.Concurrency, func() Formatter { return formatter(suite, output) })
} else {
failed = runner.scenarioConcurrent(opt.Concurrency)
}

// @TODO: should prevent from having these
os.Setenv("GODOG_SEED", "")
Expand Down
104 changes: 59 additions & 45 deletions run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func TestFeatureFilePathParser(t *testing.T) {
}

func Test_AllFeaturesRun(t *testing.T) {
const concurrency = 10
const concurrency = 100
const format = "progress"

const expected = `...................................................................... 70
Expand All @@ -272,12 +272,21 @@ func Test_AllFeaturesRun(t *testing.T) {
0s
`

actualStatus, actualOutput := testRunWithOptions(t, format, concurrency, []string{"features"})
fmtOutputSuiteInitializer := func(s *Suite) { SuiteContext(s) }
fmtOutputScenarioInitializer := InitializeScenario

actualStatus, actualOutput := testRunWithOptions(t,
fmtOutputSuiteInitializer,
format, concurrency, []string{"features"},
)

assert.Equal(t, exitSuccess, actualStatus)
assert.Equal(t, expected, actualOutput)

actualStatus, actualOutput = testRun(t, format, concurrency, []string{"features"})
actualStatus, actualOutput = testRun(t,
fmtOutputScenarioInitializer,
format, concurrency, []string{"features"},
)

assert.Equal(t, exitSuccess, actualStatus)
assert.Equal(t, expected, actualOutput)
Expand All @@ -294,20 +303,46 @@ func TestFormatterConcurrencyRun(t *testing.T) {

featurePaths := []string{"formatter-tests/features"}

const concurrency = 10
const concurrency = 100

fmtOutputSuiteInitializer := func(s *Suite) {
s.Step(`^(?:a )?failing step`, failingStepDef)
s.Step(`^(?:a )?pending step$`, pendingStepDef)
s.Step(`^(?:a )?passing step$`, passingStepDef)
s.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef)
}

fmtOutputScenarioInitializer := func(ctx *ScenarioContext) {
ctx.Step(`^(?:a )?failing step`, failingStepDef)
ctx.Step(`^(?:a )?pending step$`, pendingStepDef)
ctx.Step(`^(?:a )?passing step$`, passingStepDef)
ctx.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef)
}

for _, formatter := range formatters {
t.Run(
fmt.Sprintf("%s/concurrency/%d", formatter, concurrency),
func(t *testing.T) {
expectedStatus, expectedOutput := testRunWithOptions(t, formatter, 1, featurePaths)
actualStatus, actualOutput := testRunWithOptions(t, formatter, concurrency, featurePaths)
expectedStatus, expectedOutput := testRunWithOptions(t,
fmtOutputSuiteInitializer,
formatter, 1, featurePaths,
)
actualStatus, actualOutput := testRunWithOptions(t,
fmtOutputSuiteInitializer,
formatter, concurrency, featurePaths,
)

assert.Equal(t, expectedStatus, actualStatus)
assertOutput(t, formatter, expectedOutput, actualOutput)

expectedStatus, expectedOutput = testRun(t, formatter, 1, featurePaths)
actualStatus, actualOutput = testRun(t, formatter, concurrency, featurePaths)
expectedStatus, expectedOutput = testRun(t,
fmtOutputScenarioInitializer,
formatter, 1, featurePaths,
)
actualStatus, actualOutput = testRun(t,
fmtOutputScenarioInitializer,
formatter, concurrency, featurePaths,
)

assert.Equal(t, expectedStatus, actualStatus)
assertOutput(t, formatter, expectedOutput, actualOutput)
Expand All @@ -316,7 +351,7 @@ func TestFormatterConcurrencyRun(t *testing.T) {
}
}

func testRunWithOptions(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) {
func testRunWithOptions(t *testing.T, initializer func(*Suite), format string, concurrency int, featurePaths []string) (int, string) {
output := new(bytes.Buffer)

opts := Options{
Expand All @@ -327,15 +362,15 @@ func testRunWithOptions(t *testing.T, format string, concurrency int, featurePat
Output: output,
}

status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opts)
status := RunWithOptions("succeed", initializer, opts)

actual, err := ioutil.ReadAll(output)
require.NoError(t, err)

return status, string(actual)
}

func testRun(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) {
func testRun(t *testing.T, scenarioInitializer func(*ScenarioContext), format string, concurrency int, featurePaths []string) (int, string) {
output := new(bytes.Buffer)

opts := Options{
Expand All @@ -348,7 +383,7 @@ func testRun(t *testing.T, format string, concurrency int, featurePaths []string

status := TestSuite{
Name: "succeed",
ScenarioInitializer: InitializeScenario,
ScenarioInitializer: scenarioInitializer,
Options: &opts,
}.Run()

Expand Down Expand Up @@ -419,41 +454,20 @@ type progressOutput struct {
bottomRows []string
}

func Test_AllFeaturesRun_v010(t *testing.T) {
const concurrency = 10
const format = "progress"
func passingStepDef() error { return nil }

const expected = `...................................................................... 70
...................................................................... 140
...................................................................... 210
...................................................................... 280
.......................... 306
79 scenarios (79 passed)
306 steps (306 passed)
0s
`
func oddEvenStepDef(odd, even int) error { return oddOrEven(odd, even) }

output := new(bytes.Buffer)

opts := Options{
Format: format,
NoColors: true,
Paths: []string{"features"},
Concurrency: concurrency,
Output: output,
func oddOrEven(odd, even int) error {
if odd%2 == 0 {
return fmt.Errorf("%d is not odd", odd)
}
if even%2 != 0 {
return fmt.Errorf("%d is not even", even)
}
return nil
}

actualStatus := TestSuite{
Name: "godogs",
ScenarioInitializer: InitializeScenario,
Options: &opts,
}.Run()

actualOutput, err := ioutil.ReadAll(output)
require.NoError(t, err)
func pendingStepDef() error { return ErrPending }

assert.Equal(t, exitSuccess, actualStatus)
assert.Equal(t, expected, string(actualOutput))
}
func failingStepDef() error { return fmt.Errorf("step failed") }
Loading

0 comments on commit 6e01c51

Please sign in to comment.