Skip to content

Commit

Permalink
Merge pull request #132 from dnephin/improve-rerun
Browse files Browse the repository at this point in the history 
Add re-run number to testname format, and summary
  • Loading branch information
@dnephin
dnephin authored Jun 21, 2020
2 parents b2e1ee6 + a78bf96 commit 07020ae
Show file tree
Hide file tree
Showing 12 changed files with 179 additions and 100 deletions.
8 changes: 6 additions & 2 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,9 +210,13 @@ func run(opts *options) error {
return err
}
goTestExitErr := goTestProc.cmd.Wait()

if goTestExitErr != nil && opts.rerunFailsMaxAttempts > 0 {
cfg := testjson.ScanConfig{Execution: exec, Handler: handler}
goTestExitErr = rerunFailed(ctx, opts, cfg)
goTestExitErr = hasErrors(goTestExitErr, exec)
if goTestExitErr == nil {
cfg := testjson.ScanConfig{Execution: exec, Handler: handler}
goTestExitErr = rerunFailed(ctx, opts, cfg)
}
}

testjson.PrintSummary(opts.stdout, exec, opts.noSummary.value)
Expand Down
23 changes: 10 additions & 13 deletions rerunfails.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"strings"

"github.com/pkg/errors"
"gotest.tools/gotestsum/log"
"gotest.tools/gotestsum/testjson"
)

Expand All @@ -33,9 +32,6 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
failed, opts.rerunFailsMaxInitialFailures)
}
if err := hasErrors(scanConfig.Execution); err != nil {
return err
}

rec := newFailureRecorderFromExecution(scanConfig.Execution)
var lastErr error
Expand All @@ -55,6 +51,7 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
}

cfg := testjson.ScanConfig{
RunID: attempts + 1,
Stdout: goTestProc.stdout,
Stderr: goTestProc.stderr,
Handler: nextRec,
Expand All @@ -64,12 +61,7 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
return err
}
lastErr = goTestProc.cmd.Wait()
// 0 and 1 are expected.
if ExitCodeWithDefault(lastErr) > 1 {
log.Warnf("unexpected go test exit code: %v", lastErr)
// TODO: will 'go test' exit with 2 if it panics? maybe return err here.
}
if err := hasErrors(scanConfig.Execution); err != nil {
if err := hasErrors(lastErr, scanConfig.Execution); err != nil {
return err
}
rec = nextRec
Expand All @@ -78,11 +70,16 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
return lastErr
}

func hasErrors(exec *testjson.Execution) error {
if len(exec.Errors()) > 0 {
func hasErrors(err error, exec *testjson.Execution) error {
switch {
case len(exec.Errors()) > 0:
return fmt.Errorf("rerun aborted because previous run had errors")
// Exit code 0 and 1 are expected.
case ExitCodeWithDefault(err) > 1:
return fmt.Errorf("unexpected go test exit code: %v", err)
default:
return nil
}
return nil
}

type failureRecorder struct {
Expand Down
20 changes: 10 additions & 10 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,27 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
--- FAIL: TestFailsSometimes
flaky_test.go:58: not this time
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures
DONE 2 runs, 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

=== Failed
Expand All @@ -56,17 +56,17 @@ SEED: 0
SEED: 0
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
flaky_test.go:65: not this time


DONE 11 tests, 6 failures
DONE 3 runs, 11 tests, 6 failures
20 changes: 10 additions & 10 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,27 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time
--- FAIL: TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures
DONE 2 runs, 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

=== Failed
Expand All @@ -56,17 +56,17 @@ SEED: 0
SEED: 0
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
TestFailsOften: flaky_test.go:65: not this time


DONE 11 tests, 6 failures
DONE 3 runs, 11 tests, 6 failures
30 changes: 15 additions & 15 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,41 +20,41 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
--- FAIL: TestFailsSometimes
flaky_test.go:58: not this time
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures
DONE 2 runs, 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

DONE 11 tests, 6 failures
DONE 3 runs, 11 tests, 6 failures

=== RUN TestFailsOften
SEED: 3
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
FAIL testdata/e2e/flaky

DONE 12 tests, 7 failures
DONE 4 runs, 12 tests, 7 failures

PASS testdata/e2e/flaky.TestFailsOften
PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
PASS testdata/e2e/flaky

=== Failed
Expand All @@ -70,21 +70,21 @@ SEED: 0
SEED: 0
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
SEED: 3
flaky_test.go:65: not this time


DONE 13 tests, 7 failures
DONE 5 runs, 13 tests, 7 failures
30 changes: 15 additions & 15 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,41 +20,41 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time
--- FAIL: TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures
DONE 2 runs, 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

DONE 11 tests, 6 failures
DONE 3 runs, 11 tests, 6 failures

=== RUN TestFailsOften
SEED: 3
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
FAIL testdata/e2e/flaky

DONE 12 tests, 7 failures
DONE 4 runs, 12 tests, 7 failures

PASS testdata/e2e/flaky.TestFailsOften
PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
PASS testdata/e2e/flaky

=== Failed
Expand All @@ -70,21 +70,21 @@ SEED: 0
SEED: 0
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
SEED: 3
TestFailsOften: flaky_test.go:65: not this time


DONE 13 tests, 7 failures
DONE 5 runs, 13 tests, 7 failures
Loading

0 comments on commit 07020ae

Please sign in to comment.