diff --git a/examples/crds/envtest_test.go b/examples/crds/envtest_test.go index 1153fd2d..ad65f930 100644 --- a/examples/crds/envtest_test.go +++ b/examples/crds/envtest_test.go @@ -64,5 +64,5 @@ func TestCRDSetup(t *testing.T) { return ctx }).Feature() - testEnv.Test(t, feature) + _ = testEnv.Test(t, feature) } diff --git a/examples/dry_run/dry_run_test.go b/examples/dry_run/dry_run_test.go index 49fc7a2e..d274d338 100644 --- a/examples/dry_run/dry_run_test.go +++ b/examples/dry_run/dry_run_test.go @@ -46,7 +46,7 @@ func TestDryRunOne(t *testing.T) { return ctx }).Feature() - testEnv.TestInParallel(t, f1, f2) + _ = testEnv.TestInParallel(t, f1, f2) } func TestDryRunTwo(t *testing.T) { diff --git a/examples/multi_cluster/deployment_test.go b/examples/multi_cluster/deployment_test.go index 278a406f..0cccc1b0 100644 --- a/examples/multi_cluster/deployment_test.go +++ b/examples/multi_cluster/deployment_test.go @@ -92,5 +92,5 @@ func TestScenarioOne(t *testing.T) { }). Feature() - testEnv.Test(t, feature) + _ = testEnv.Test(t, feature) } diff --git a/examples/parallel_features/parallel_features_test.go b/examples/parallel_features/parallel_features_test.go index e3713a3b..22418246 100644 --- a/examples/parallel_features/parallel_features_test.go +++ b/examples/parallel_features/parallel_features_test.go @@ -118,5 +118,5 @@ func TestPodBringUp(t *testing.T) { return ctx }).Feature() - testEnv.TestInParallel(t, featureOne, featureTwo) + _ = testEnv.TestInParallel(t, featureOne, featureTwo) } diff --git a/examples/pod_exec/envtest_test.go b/examples/pod_exec/envtest_test.go index b96faaf5..b2e702ee 100644 --- a/examples/pod_exec/envtest_test.go +++ b/examples/pod_exec/envtest_test.go @@ -79,7 +79,7 @@ func TestExecPod(t *testing.T) { } return ctx }).Feature() - testEnv.Test(t, feature) + _ = testEnv.Test(t, feature) } func newDeployment(namespace string, name string, replicas int32, containerName string) *appsv1.Deployment { diff --git a/examples/third_party_integration/helm/helm_test.go b/examples/third_party_integration/helm/helm_test.go index 558c79ad..ecb596b1 100644 --- a/examples/third_party_integration/helm/helm_test.go +++ b/examples/third_party_integration/helm/helm_test.go @@ -87,7 +87,7 @@ func TestHelmChartRepoWorkflow(t *testing.T) { return ctx }).Feature() - testEnv.Test(t, feature) + _ = testEnv.Test(t, feature) } func TestLocalHelmChartWorkflow(t *testing.T) { @@ -125,5 +125,5 @@ func TestLocalHelmChartWorkflow(t *testing.T) { return ctx }).Feature() - testEnv.Test(t, feature) + _ = testEnv.Test(t, feature) } diff --git a/hack/test-go.sh b/hack/test-go.sh index 3d1362b4..efbe3b63 100755 --- a/hack/test-go.sh +++ b/hack/test-go.sh @@ -38,5 +38,5 @@ cd "${REPO_ROOT}" # Ensure -p=1 to avoid packages running concurrently which may all try and install kind at the same time or race for # use of the kind binary. -GO111MODULE=on go test -v -p=1 -timeout="${TEST_TIMEOUT}s" -count=1 -cover -coverprofile coverage.out $(go list ./...) +GO111MODULE=on go test -race -v -p=1 -timeout="${TEST_TIMEOUT}s" -count=1 -cover -coverprofile coverage.out $(go list ./...) go tool cover -html coverage.out -o coverage.html diff --git a/pkg/env/env.go b/pkg/env/env.go index 7dc63b8e..72e1a051 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -188,38 +188,44 @@ func (e *testEnv) panicOnMissingContext() { // processTestActions is used to run a series of test action that were configured as // BeforeEachTest or AfterEachTest -func (e *testEnv) processTestActions(t *testing.T, actions []action) { +func (e *testEnv) processTestActions(ctx context.Context, t *testing.T, actions []action) context.Context { var err error + out := ctx for _, action := range actions { - if e.ctx, err = action.runWithT(e.ctx, e.cfg, t); err != nil { + out, err = action.runWithT(ctx, e.cfg, t) + if err != nil { t.Fatalf("%s failure: %s", action.role, err) } } + return out } // processTestFeature is used to trigger the execution of the actual feature. This function wraps the entire // workflow of orchestrating the feature execution be running the action configured by BeforeEachFeature / // AfterEachFeature. -func (e *testEnv) processTestFeature(t *testing.T, featureName string, feature types.Feature) { +func (e *testEnv) processTestFeature(ctx context.Context, t *testing.T, featureName string, feature types.Feature) context.Context { // execute beforeEachFeature actions - e.processFeatureActions(t, feature, e.getBeforeFeatureActions()) + ctx = e.processFeatureActions(ctx, t, feature, e.getBeforeFeatureActions()) // execute feature test - e.ctx = e.execFeature(e.ctx, t, featureName, feature) + ctx = e.execFeature(ctx, t, featureName, feature) // execute afterEachFeature actions - e.processFeatureActions(t, feature, e.getAfterFeatureActions()) + return e.processFeatureActions(ctx, t, feature, e.getAfterFeatureActions()) } // processFeatureActions is used to run a series of feature action that were configured as // BeforeEachFeature or AfterEachFeature -func (e *testEnv) processFeatureActions(t *testing.T, feature types.Feature, actions []action) { +func (e *testEnv) processFeatureActions(ctx context.Context, t *testing.T, feature types.Feature, actions []action) context.Context { var err error + out := ctx for _, action := range actions { - if e.ctx, err = action.runWithFeature(e.ctx, e.cfg, t, deepCopyFeature(feature)); err != nil { + out, err = action.runWithFeature(out, e.cfg, t, deepCopyFeature(feature)) + if err != nil { t.Fatalf("%s failure: %s", action.role, err) } } + return out } // processTests is a wrapper function that can be invoked by either Test or TestInParallel methods. @@ -228,26 +234,28 @@ func (e *testEnv) processFeatureActions(t *testing.T, feature types.Feature, act // // In case if the parallel run of test features are enabled, this function will invoke the processTestFeature // as a go-routine to get them to run in parallel -func (e *testEnv) processTests(t *testing.T, enableParallelRun bool, testFeatures ...types.Feature) { +func (e *testEnv) processTests(ctx context.Context, t *testing.T, enableParallelRun bool, testFeatures ...types.Feature) context.Context { if e.cfg.DryRunMode() { klog.V(2).Info("e2e-framework is being run in dry-run mode. This will skip all the before/after step functions configured around your test assessments and features") } - e.panicOnMissingContext() + if ctx == nil { + panic("nil context") // this should never happen + } if len(testFeatures) == 0 { t.Log("No test testFeatures provided, skipping test") - return + return ctx } beforeTestActions := e.getBeforeTestActions() afterTestActions := e.getAfterTestActions() - e.processTestActions(t, beforeTestActions) - runInParallel := e.cfg.ParallelTestEnabled() && enableParallelRun if runInParallel { klog.V(4).Info("Running test features in parallel") } + ctx = e.processTestActions(ctx, t, beforeTestActions) + var wg sync.WaitGroup for i, feature := range testFeatures { featureCopy := feature @@ -257,12 +265,12 @@ func (e *testEnv) processTests(t *testing.T, enableParallelRun bool, testFeature } if runInParallel { wg.Add(1) - go func(w *sync.WaitGroup, featName string, f types.Feature) { + go func(ctx context.Context, w *sync.WaitGroup, featName string, f types.Feature) { defer w.Done() - e.processTestFeature(t, featName, f) - }(&wg, featName, featureCopy) + _ = e.processTestFeature(ctx, t, featName, f) + }(ctx, &wg, featName, featureCopy) } else { - e.processTestFeature(t, featName, featureCopy) + ctx = e.processTestFeature(ctx, t, featName, featureCopy) // In case if the feature under test has failed, skip reset of the features // that are part of the same test if e.cfg.FailFast() && t.Failed() { @@ -273,7 +281,7 @@ func (e *testEnv) processTests(t *testing.T, enableParallelRun bool, testFeature if runInParallel { wg.Wait() } - e.processTestActions(t, afterTestActions) + return e.processTestActions(ctx, t, afterTestActions) } // TestInParallel executes a series a feature tests from within a @@ -294,8 +302,8 @@ func (e *testEnv) processTests(t *testing.T, enableParallelRun bool, testFeature // set of features being passed to this call while the feature themselves // are executed in parallel to avoid duplication of action that might happen // in BeforeTest and AfterTest actions -func (e *testEnv) TestInParallel(t *testing.T, testFeatures ...types.Feature) { - e.processTests(t, true, testFeatures...) +func (e *testEnv) TestInParallel(t *testing.T, testFeatures ...types.Feature) context.Context { + return e.processTests(e.ctx, t, true, testFeatures...) } // Test executes a feature test from within a TestXXX function. @@ -310,8 +318,8 @@ func (e *testEnv) TestInParallel(t *testing.T, testFeatures ...types.Feature) { // // BeforeTest and AfterTest operations are executed before and after // the feature is tested respectively. -func (e *testEnv) Test(t *testing.T, testFeatures ...types.Feature) { - e.processTests(t, false, testFeatures...) +func (e *testEnv) Test(t *testing.T, testFeatures ...types.Feature) context.Context { + return e.processTests(e.ctx, t, false, testFeatures...) } // Finish registers funcs that are executed at the end of the @@ -331,9 +339,8 @@ func (e *testEnv) Finish(funcs ...Func) types.Environment { // starting the tests and run all Env.Finish operations after // before completing the suite. func (e *testEnv) Run(m *testing.M) int { - if e.ctx == nil { - panic("context not set") // something is terribly wrong. - } + e.panicOnMissingContext() + ctx := e.ctx setups := e.getSetupActions() // fail fast on setup, upon err exit @@ -355,18 +362,20 @@ func (e *testEnv) Run(m *testing.M) int { // Upon error, log and continue. for _, fin := range finishes { // context passed down to each finish step - if e.ctx, err = fin.run(e.ctx, e.cfg); err != nil { + if ctx, err = fin.run(ctx, e.cfg); err != nil { klog.V(2).ErrorS(err, "Cleanup failed", "action", fin.role) } } + e.ctx = ctx }() for _, setup := range setups { // context passed down to each setup - if e.ctx, err = setup.run(e.ctx, e.cfg); err != nil { + if ctx, err = setup.run(ctx, e.cfg); err != nil { klog.Fatalf("%s failure: %s", setup.role, err) } } + e.ctx = ctx // Execute the test suite return m.Run() @@ -423,10 +432,10 @@ func (e *testEnv) executeSteps(ctx context.Context, t *testing.T, steps []types. func (e *testEnv) execFeature(ctx context.Context, t *testing.T, featName string, f types.Feature) context.Context { // feature-level subtest - t.Run(featName, func(t *testing.T) { + t.Run(featName, func(newT *testing.T) { skipped, message := e.requireFeatureProcessing(f) if skipped { - t.Skipf(message) + newT.Skipf(message) } if fDescription, ok := f.(types.DescribableFeature); ok && fDescription.Description() != "" { @@ -435,7 +444,7 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, featName string // setups run at feature-level setups := features.GetStepsByLevel(f.Steps(), types.LevelSetup) - ctx = e.executeSteps(ctx, t, setups) + ctx = e.executeSteps(ctx, newT, setups) // assessments run as feature/assessment sub level assessments := features.GetStepsByLevel(f.Steps(), types.LevelAssess) @@ -449,17 +458,17 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, featName string if assessName == "" { assessName = fmt.Sprintf("Assessment-%d", i+1) } - t.Run(assessName, func(t *testing.T) { + newT.Run(assessName, func(internalT *testing.T) { skipped, message := e.requireAssessmentProcessing(assess, i+1) if skipped { - t.Skipf(message) + internalT.Skipf(message) } - ctx = e.executeSteps(ctx, t, []types.Step{assess}) + ctx = e.executeSteps(ctx, internalT, []types.Step{assess}) }) // Check if the Test assessment under question performed a `t.Fail()` or `t.Failed()` invocation. // We need to track that and stop the next set of assessment in the feature under test from getting // executed - if e.cfg.FailFast() && t.Failed() { + if e.cfg.FailFast() && newT.Failed() { failed = true break } @@ -469,12 +478,12 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, featName string // invoked to make sure we leave the traces of the failed test behind to enable better debugging for the // test developers if e.cfg.FailFast() && failed { - t.FailNow() + newT.FailNow() } // teardowns run at feature-level teardowns := features.GetStepsByLevel(f.Steps(), types.LevelTeardown) - ctx = e.executeSteps(ctx, t, teardowns) + ctx = e.executeSteps(ctx, newT, teardowns) }) return ctx diff --git a/pkg/env/env_test.go b/pkg/env/env_test.go index ba42e269..6b693e21 100644 --- a/pkg/env/env_test.go +++ b/pkg/env/env_test.go @@ -18,6 +18,7 @@ package env import ( "context" + "sync/atomic" "testing" "time" @@ -137,7 +138,7 @@ func TestEnv_Test(t *testing.T) { tests := []struct { name string ctx context.Context - setup func(*testing.T, context.Context) []string + setup func(context.Context, *testing.T) []string expected []string }{ { @@ -146,13 +147,13 @@ func TestEnv_Test(t *testing.T) { expected: []string{ "test-feat", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() f := features.New("test-feat").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { val = append(val, "test-feat") return ctx }) - env.Test(t, f.Feature()) + _ = env.Test(t, f.Feature()) return }, }, @@ -162,20 +163,20 @@ func TestEnv_Test(t *testing.T) { expected: []string{ "test-feat", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := NewWithConfig(envconf.New().WithFeatureRegex("test-feat")) f := features.New("test-feat").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { val = append(val, "test-feat") return ctx }) - env.Test(t, f.Feature()) + _ = env.Test(t, f.Feature()) env2 := NewWithConfig(envconf.New().WithFeatureRegex("skip-me")) f2 := features.New("test-feat-2").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { val = append(val, "test-feat-2") return ctx }) - env2.Test(t, f2.Feature()) + _ = env2.Test(t, f2.Feature()) return }, @@ -187,7 +188,7 @@ func TestEnv_Test(t *testing.T) { "before-each-test", "test-feat", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() env.BeforeEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { val = append(val, "before-each-test") @@ -197,7 +198,7 @@ func TestEnv_Test(t *testing.T) { val = append(val, "test-feat") return ctx }) - env.Test(t, f.Feature()) + _ = env.Test(t, f.Feature()) return }, }, @@ -209,7 +210,7 @@ func TestEnv_Test(t *testing.T) { "test-feat", "after-each-test", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() env.AfterEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { val = append(val, "after-each-test") @@ -222,7 +223,7 @@ func TestEnv_Test(t *testing.T) { val = append(val, "test-feat") return ctx }) - env.Test(t, f.Feature()) + _ = env.Test(t, f.Feature()) return }, }, @@ -233,7 +234,7 @@ func TestEnv_Test(t *testing.T) { "test-feat", "after-each-test", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() env.AfterEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { val = append(val, "after-each-test") @@ -243,7 +244,7 @@ func TestEnv_Test(t *testing.T) { val = append(val, "test-feat") return ctx }) - env.Test(t, f.Feature()) + _ = env.Test(t, f.Feature()) return }, }, @@ -254,7 +255,7 @@ func TestEnv_Test(t *testing.T) { "add-1", "add-2", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { val = []string{} env := NewWithConfig(envconf.New().WithAssessmentRegex("add-*")) f := features.New("test-feat"). @@ -270,7 +271,7 @@ func TestEnv_Test(t *testing.T) { val = append(val, "take-1") return ctx }) - env.Test(t, f.Feature()) + _ = env.Test(t, f.Feature()) return }, }, @@ -282,7 +283,7 @@ func TestEnv_Test(t *testing.T) { "test-feat", "after-each-test", }, - setup: func(t *testing.T, ctx context.Context) []string { + setup: func(ctx context.Context, t *testing.T) []string { env, err := NewWithContext(context.WithValue(ctx, &ctxTestKeyString{}, []string{}), envconf.New()) if err != nil { t.Fatal(err) @@ -315,17 +316,122 @@ func TestEnv_Test(t *testing.T) { return context.WithValue(ctx, &ctxTestKeyString{}, val) }) - env.Test(t, f.Feature()) - return env.(*testEnv).ctx.Value(&ctxTestKeyString{}).([]string) + out := env.Test(t, f.Feature()) + return out.Value(&ctxTestKeyString{}).([]string) + }, + }, + { + name: "context value propagation with with multiple features, before, during, and after test", + ctx: context.TODO(), + expected: []string{ + "before-each-test", + "test-feat-1", + "test-feat-2", + "after-each-test", + }, + setup: func(ctx context.Context, t *testing.T) []string { + env, err := NewWithContext(context.WithValue(ctx, &ctxTestKeyString{}, []string{}), envconf.New()) + if err != nil { + t.Fatal(err) + } + env.BeforeEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { + // update before test + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "before-each-test") + return context.WithValue(ctx, &ctxTestKeyString{}, val), nil + }) + env.AfterEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { + // update after the test + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "after-each-test") + return context.WithValue(ctx, &ctxTestKeyString{}, val), nil + }) + f1 := features.New("test-feat").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "test-feat-1") + + return context.WithValue(ctx, &ctxTestKeyString{}, val) + }).Feature() + f2 := features.New("test-feat").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "test-feat-2") + + return context.WithValue(ctx, &ctxTestKeyString{}, val) + }).Feature() + + out := env.Test(t, f1, f2) + return out.Value(&ctxTestKeyString{}).([]string) + }, + }, + { + name: "context value propagation with with multiple features in parallel, before, during, and after test", + ctx: context.WithValue(context.TODO(), &ctxTestKeyString{}, []string{}), + expected: []string{ + "before-each-test", + "after-each-test", + }, + setup: func(ctx context.Context, t *testing.T) []string { + env := NewParallel().WithContext(context.WithValue(ctx, &ctxTestKeyString{}, []string{})) + env.BeforeEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { + // update before test + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "before-each-test") + return context.WithValue(ctx, &ctxTestKeyString{}, val), nil + }) + env.AfterEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { + // update after the test + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "after-each-test") + return context.WithValue(ctx, &ctxTestKeyString{}, val), nil + }) + f1 := features.New("test-feat").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "test-feat-1") + + return context.WithValue(ctx, &ctxTestKeyString{}, val) + }).Feature() + f2 := features.New("test-feat").Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + t.Fatal("context value was not []string") + } + val = append(val, "test-feat-2") + + return context.WithValue(ctx, &ctxTestKeyString{}, val) + }).Feature() + + out := env.TestInParallel(t, f1, f2) + return out.Value(&ctxTestKeyString{}).([]string) }, }, { name: "no features specified", ctx: context.TODO(), expected: []string{}, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() - env.Test(t) + _ = env.Test(t) return }, }, @@ -336,7 +442,7 @@ func TestEnv_Test(t *testing.T) { "test-feature-1", "test-feature-2", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() f1 := features.New("test-feat-1"). Assess("assess", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { @@ -350,7 +456,7 @@ func TestEnv_Test(t *testing.T) { return ctx }) - env.Test(t, f1.Feature(), f2.Feature()) + _ = env.Test(t, f1.Feature(), f2.Feature()) return }, }, @@ -363,7 +469,7 @@ func TestEnv_Test(t *testing.T) { "test-feat-2", "after-each-test", }, - setup: func(t *testing.T, ctx context.Context) (val []string) { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() val = []string{} env.BeforeEachTest(func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) { @@ -382,7 +488,7 @@ func TestEnv_Test(t *testing.T) { val = append(val, "test-feat-2") return ctx }) - env.Test(t, f1.Feature(), f2.Feature()) + _ = env.Test(t, f1.Feature(), f2.Feature()) return }, }, @@ -397,9 +503,8 @@ func TestEnv_Test(t *testing.T) { "test-feat-2", "after-each-feature", }, - setup: func(t *testing.T, ctx context.Context) []string { + setup: func(ctx context.Context, t *testing.T) (val []string) { env := newTestEnv() - val := []string{} env.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, _ *testing.T, info features.Feature) (context.Context, error) { val = append(val, "before-each-feature") return ctx, nil @@ -415,8 +520,8 @@ func TestEnv_Test(t *testing.T) { val = append(val, "test-feat-2") return ctx }) - env.Test(t, f1.Feature(), f2.Feature()) - return val + _ = env.Test(t, f1.Feature(), f2.Feature()) + return }, }, { @@ -430,7 +535,7 @@ func TestEnv_Test(t *testing.T) { "test-feat-2", "after-each-feature", }, - setup: func(t *testing.T, ctx context.Context) []string { + setup: func(ctx context.Context, t *testing.T) []string { env := newTestEnv() val := []string{} env.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, _ *testing.T, info features.Feature) (context.Context, error) { @@ -474,14 +579,14 @@ func TestEnv_Test(t *testing.T) { val = append(val, "test-feat-2") return ctx }) - env.Test(t, f1.Feature(), f2.Feature()) + _ = env.Test(t, f1.Feature(), f2.Feature()) return val }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := test.setup(t, test.ctx) + result := test.setup(test.ctx, t) if len(test.expected) != len(result) { t.Fatalf("Expected:\n%v but got result:\n%v", test.expected, result) } @@ -509,14 +614,9 @@ func TestEnv_Context_Propagation(t *testing.T) { return context.WithValue(ctx, &ctxTestKeyString{}, val) }) - envForTesting.Test(t, f.Feature()) + out := envForTesting.Test(t, f.Feature()) - env, ok := envForTesting.(*testEnv) - if !ok { - t.Fatal("wrong type") - } - - finalVal, ok := env.ctx.Value(&ctxTestKeyString{}).([]string) + finalVal, ok := out.Value(&ctxTestKeyString{}).([]string) if !ok { t.Fatal("wrong type") } @@ -537,8 +637,8 @@ func TestTestEnv_TestInParallel(t *testing.T) { env := NewParallel() beforeEachCallCount := 0 afterEachCallCount := 0 - beforeFeatureCount := 0 - afterFeatureCount := 0 + var beforeFeatureCount, + afterFeatureCount atomic.Int32 env.BeforeEachTest(func(ctx context.Context, config *envconf.Config, t *testing.T) (context.Context, error) { beforeEachCallCount++ return ctx, nil @@ -551,13 +651,13 @@ func TestTestEnv_TestInParallel(t *testing.T) { env.BeforeEachFeature(func(ctx context.Context, config *envconf.Config, _ *testing.T, feature types.Feature) (context.Context, error) { t.Logf("Running before each feature for feature %s", feature.Name()) - beforeFeatureCount++ + beforeFeatureCount.Add(1) return ctx, nil }) env.AfterEachFeature(func(ctx context.Context, config *envconf.Config, _ *testing.T, feature types.Feature) (context.Context, error) { t.Logf("Running after each feature for feature %s", feature.Name()) - afterFeatureCount++ + afterFeatureCount.Add(1) return ctx, nil }) @@ -579,8 +679,131 @@ func TestTestEnv_TestInParallel(t *testing.T) { return ctx }) - env.TestInParallel(t, f1.Feature(), f2.Feature()) - if beforeEachCallCount > 1 { - t.Fatal("BeforeEachTest handler should be invoked only once") + _ = env.TestInParallel(t, f1.Feature(), f2.Feature()) + if beforeEachCallCount != 1 { + t.Fatal("BeforeEachTest handler should be invoked exactly once") + } + if afterEachCallCount != 1 { + t.Fatal("AfterEachTest handler should be invoked exactly once") + } + if beforeFeatureCount.Load() != 2 { + t.Fatal("BeforeEachFeature handler should be invoked exactly twice") + } + if afterFeatureCount.Load() != 2 { + t.Fatal("AfterEachFeature handler should be invoked exactly twice") + } +} + +// TestTParallelMultipleFeaturesInParallel runs multple features in parallel with a dedicated Parallel environment, +// just to check there are no race conditions with this setting +func TestTParallelMultipleFeaturesInParallel(t *testing.T) { + env := NewParallel() + t.Parallel() + f1 := features.New("feature 1"). + Assess("assess", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + time.Sleep(2 * time.Second) + return ctx + }) + f2 := features.New("feature 2"). + Assess("assess", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + time.Sleep(3 * time.Second) + return ctx + }) + _ = env.TestInParallel(t, f1.Feature(), f2.Feature()) +} + +// env with parallel disabled to be used in the two tests below, reusing testEnv could result on a race condition due to +// the Before/AfterEachTest accessing the same array from the context at the same time, which is not thread safe +var envTForParallelTesting = New() + +// TestMultipleAssess runs multiple assessments sequentially, but can run in parallel with other parallel tests in the suite +// just to check there are no race conditions, the resulting context is not defined though as it t.Parallel() is used +// a dedicated Context for each test has to be manually created and injected into the environment before running Test +func TestTParallelMultipleAssess(t *testing.T) { + t.Parallel() + f := features.New("assess"). + Assess("assess one", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Logf("Started at: %s", time.Now().UTC()) + time.Sleep(3 * time.Second) + t.Logf("Terminated at: %s", time.Now().UTC()) + return ctx + }). + Assess("assess two", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Logf("Started at: %s", time.Now().UTC()) + time.Sleep(2 * time.Second) + t.Logf("Terminated at: %s", time.Now().UTC()) + return ctx + }) + _ = envTForParallelTesting.Test(t, f.Feature()) +} + +// TestTParallelOne, TestTParallelTwo are used to test that there is no race condition when running in parallel by using +// t.Parallel() instead of TestInParallel on an environment with parallel disabled and running in parallel with other +// tests in the suite +func TestTParallelOne(t *testing.T) { + t.Parallel() + f := features.New("parallel one"). + Assess("log a message", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Log("Running in parallel") + return ctx + }).Feature() + + _ = envTForParallelTesting.Test(t, f) +} + +// See comment of TestTParallelOne +func TestTParallelTwo(t *testing.T) { + t.Parallel() + f := features.New("parallel two"). + Assess("log a message", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Log("Running in parallel") + return ctx + }).Feature() + + _ = envTForParallelTesting.Test(t, f) +} + +// env with parallel enabled to be used in the two test below, reusing testEnv defined by TestMain would result on a +// race condition due to the Before/AfterEachTest accessing the same array from the context at the same time, for +// multiple parallel tests which is not thread safe +var envForTParallelInParallelTesting = NewParallel() + +type ctxRunsKeyString struct{} + +// TestTParallelInParallelOne, TestTParallelInParallelTwo are used to test that there is no race condition when running in +// parallel both multiple tests using t.Parallel() and multiple features using TestInParallel per test +func TestTParallelInParallelOne(t *testing.T) { + t.Parallel() + out := envForTParallelInParallelTesting.TestInParallel(t, getFeaturesForTest()...) + if i, ok := out.Value(ctxRunsKeyString{}).(int); ok && i != 0 { + t.Fatalf("Runs should be 0, the context should not be shared between features tested in parallel with tests running in parallel, got %v", i) + } +} + +func TestTParallelInParallelTwo(t *testing.T) { + t.Parallel() + out := envForTParallelInParallelTesting.TestInParallel(t, getFeaturesForTest()...) + if i, ok := out.Value(ctxRunsKeyString{}).(int); ok && i != 0 { + t.Fatalf("Runs should be 0, the context should not be shared between features tested in parallel with tests running in parallel, got %v", i) } } + +func getFeaturesForTest() []features.Feature { + f1 := features.New("parallel one"). + Assess("log a message", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Log("Running in parallel 1 1") + if i := ctx.Value(ctxRunsKeyString{}); i != nil { + return context.WithValue(ctx, ctxRunsKeyString{}, i.(int)+1) + } + return context.WithValue(ctx, ctxRunsKeyString{}, 1) + }).Feature() + f2 := features.New("parallel one"). + Assess("log a message", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Log("Running in parallel 1 2") + if i := ctx.Value(ctxRunsKeyString{}); i != nil { + return context.WithValue(ctx, ctxRunsKeyString{}, i.(int)+1) + } + return context.WithValue(ctx, ctxRunsKeyString{}, 1) + }).Feature() + return []features.Feature{f1, f2} +} diff --git a/pkg/env/main_test.go b/pkg/env/main_test.go index 7ae59534..5d7401b4 100644 --- a/pkg/env/main_test.go +++ b/pkg/env/main_test.go @@ -75,6 +75,15 @@ func TestMain(m *testing.M) { } val = append(val, "after-each-test") return context.WithValue(ctx, &ctxTestKeyString{}, val), nil + }).Finish(func(ctx context.Context, _ *envconf.Config) (context.Context, error) { + // update after the test suite + val, ok := ctx.Value(&ctxTestKeyString{}).([]string) + if !ok { + log.Fatal("context value was not of expected type []string] or nil") + } + // this will only be accessible after the whole suite run + val = append(val, "finish") + return context.WithValue(ctx, &ctxTestKeyString{}, val), nil }) os.Exit(envForTesting.Run(m)) diff --git a/pkg/internal/types/types.go b/pkg/internal/types/types.go index bf6c932b..ed4fef4b 100644 --- a/pkg/internal/types/types.go +++ b/pkg/internal/types/types.go @@ -68,12 +68,12 @@ type Environment interface { // Test executes a test feature defined in a TestXXX function // This method surfaces context for further updates. - Test(*testing.T, ...Feature) + Test(*testing.T, ...Feature) context.Context // TestInParallel executes a series of test features defined in a // TestXXX function in parallel. This works the same way Test method // does with the caveat that the features will all be run in parallel - TestInParallel(*testing.T, ...Feature) + TestInParallel(*testing.T, ...Feature) context.Context // AfterEachTest registers environment funcs that are executed // after each Env.Test(...).