From 0dcc3fac357ed32c95cdc25674b67036a7f60db5 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Wed, 13 Nov 2024 17:34:17 -0500 Subject: [PATCH] storage: don't cache benchmark fixtures in metamorphic builds In CI, we sometimes run benchmarks in metamorphic builds to ensure the benchmarks pass. We don't want to cache the fixtures generated by these runs, because we may end up caching fixtures that reflect metamorphic values. This can lead to very non-deterministic behavior in the benchmarks depending on what metamorphic settings were active when the fixture was generated. Epic: none Release note: none Informs #135114. --- pkg/storage/bench_data_test.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/storage/bench_data_test.go b/pkg/storage/bench_data_test.go index 4f8c02a6de85..e91e4f2a4ad5 100644 --- a/pkg/storage/bench_data_test.go +++ b/pkg/storage/bench_data_test.go @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/metamorphic" "github.com/cockroachdb/cockroach/pkg/util/randutil" "github.com/cockroachdb/pebble/vfs" "github.com/stretchr/testify/require" @@ -69,9 +70,23 @@ func getInitialStateEngine( ctx context.Context, b *testing.B, initial initialState, inMemory bool, ) Engine { name := strings.Join(initial.Key(), "-") - dir := testfixtures.ReuseOrGenerate(b, name, func(dir string) { + + // In CI, we sometimes run benchmarks in metamorphic builds to ensure the + // benchmarks pass. We don't want to cache the fixtures generated by these + // runs, because we may end up caching fixtures that reflect metamorphic + // values. This can lead to very non-deterministic behavior in the + // benchmarks depending on what metamorphic settings were active when the + // fixture was generated. + var dir string + if metamorphic.IsMetamorphicBuild() { + b.Log("running in metamorphic build; ignoring existing fixtures and not caching new ones") + dir = b.TempDir() buildInitialState(ctx, b, initial, dir) - }) + } else { + dir = testfixtures.ReuseOrGenerate(b, name, func(dir string) { + buildInitialState(ctx, b, initial, dir) + }) + } dataDir := filepath.Join(dir, "data") opts := append([]ConfigOption{