Skip to content

Commit

Permalink
test: add more sims tests (#23421)
Browse files Browse the repository at this point in the history
(cherry picked from commit b8638f8)

# Conflicts:
#	scripts/build/simulations.mk
  • Loading branch information
alpe authored and mergify[bot] committed Jan 17, 2025
1 parent 0455db9 commit 6bbcbf9
Show file tree
Hide file tree
Showing 8 changed files with 239 additions and 18 deletions.
19 changes: 17 additions & 2 deletions scripts/build/simulations.mk
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ test-sim-multi-seed-short-v2:
@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
-NumBlocks=50

<<<<<<< HEAD
test-v2-sim:
@echo "Running short multi-seed application simulation. This may take awhile!"
@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestSimsAppV2
Expand All @@ -79,6 +80,8 @@ test-sim-benchmark-invariants:
cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
-Enabled=true -NumBlocks=1000 -BlockSize=200 -Commit=true -Seed=57 -v -timeout 24h

=======
>>>>>>> b8638f892 (test: add more sims tests (#23421))
.PHONY: \
test-sim-nondeterminism \
test-sim-nondeterminism-streaming \
Expand All @@ -88,29 +91,41 @@ test-sim-after-import \
test-sim-custom-genesis-multi-seed \
test-sim-multi-seed-short \
test-sim-multi-seed-long \
test-sim-benchmark-invariants

SIM_NUM_BLOCKS ?= 500
SIM_BLOCK_SIZE ?= 200
SIM_COMMIT ?= true

#? test-sim-fuzz: Run fuzz test for simapp
test-sim-fuzz:
@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
#ld flags are a quick fix to make it work on current osx
<<<<<<< HEAD
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
=======
@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -json -tags='sims' -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
>>>>>>> b8638f892 (test: add more sims tests (#23421))

#? test-sim-benchmark: Run benchmark test for simapp
test-sim-benchmark:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
<<<<<<< HEAD
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h
=======
@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE)
>>>>>>> b8638f892 (test: add more sims tests (#23421))


test-sim-profile:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
<<<<<<< HEAD
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
=======
@cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -cpuprofile cpu.out -memprofile mem.out
>>>>>>> b8638f892 (test: add more sims tests (#23421))

.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz

Expand Down
1 change: 1 addition & 0 deletions simapp/v2/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/simapp.test
17 changes: 17 additions & 0 deletions simapp/v2/sim_bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//go:build sims

package simapp

import (
"github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
"testing"
)

func BenchmarkFullAppSimulation(b *testing.B) {
b.ReportAllocs()
cfg := cli.NewConfigFromFlags()
cfg.ChainID = SimAppChainID
for i := 0; i < b.N; i++ {
RunWithSeed[Tx](b, NewSimApp[Tx], AppConfig, cfg, 1)
}
}
23 changes: 23 additions & 0 deletions simapp/v2/sim_fuzz_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
//go:build sims

package simapp

import (
simsxv2 "github.com/cosmos/cosmos-sdk/simsx/v2"
simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
"testing"
)

func FuzzFullAppSimulation(f *testing.F) {
cfg := simcli.NewConfigFromFlags()
cfg.ChainID = SimAppChainID

f.Fuzz(func(t *testing.T, rawSeed []byte) {
if len(rawSeed) < 8 {
t.Skip()
return
}
randSource := simsxv2.NewByteSource(cfg.FuzzSeed, cfg.Seed)
RunWithRandSource[Tx](t, NewSimApp[Tx], AppConfig, cfg, randSource)
})
}
40 changes: 29 additions & 11 deletions simapp/v2/sim_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ type (

// TestInstance system under test
TestInstance[T Tx] struct {
Seed int64
RandSource simsxv2.RandSource
App SimulationApp[T]
TxDecoder transaction.Codec[T]
BankKeeper BankKeeper
Expand All @@ -126,7 +126,7 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](
tb testing.TB,
appFactory AppFactory[T, V],
appConfigFactory AppConfigFactory,
seed int64,
randSource simsxv2.RandSource,
) TestInstance[T] {
tb.Helper()
vp := viper.New()
Expand All @@ -153,7 +153,7 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](
xapp, err := appFactory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings()))))
require.NoError(tb, err)
return TestInstance[T]{
Seed: seed,
RandSource: randSource,
App: xapp,
BankKeeper: bankKeeper,
AuthKeeper: authKeeper,
Expand Down Expand Up @@ -197,7 +197,7 @@ func (ti TestInstance[T]) InitializeChain(
}
}

// RunWithSeeds runs a series of subtests using the default set of random seeds for deterministic simulation testing.
// RunWithSeeds runs a series of subtests for each of the given set of seeds for deterministic simulation testing.
func RunWithSeeds[T Tx, V SimulationApp[T]](
t *testing.T,
appFactory AppFactory[T, V],
Expand All @@ -224,13 +224,26 @@ func RunWithSeed[T Tx, V SimulationApp[T]](
tCfg simtypes.Config,
seed int64,
postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
) {
tb.Helper()
RunWithRandSource(tb, appFactory, appConfigFactory, tCfg, simsxv2.NewSeededRandSource(seed), postRunActions...)
}

// RunWithRandSource initializes and executes a simulation run with the given rand source, generating blocks and transactions.
func RunWithRandSource[T Tx, V SimulationApp[T]](
tb testing.TB,
appFactory AppFactory[T, V],
appConfigFactory AppConfigFactory,
tCfg simtypes.Config,
randSource simsxv2.RandSource,
postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
) {
tb.Helper()
initialBlockHeight := tCfg.InitialBlockHeight
require.NotEmpty(tb, initialBlockHeight, "initial block height must not be 0")

setupFn := func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account) {
testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, seed)
testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, randSource)
accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState(
testInstance.App,
r,
Expand All @@ -249,20 +262,21 @@ func RunWithSeed[T Tx, V SimulationApp[T]](

return testInstance, cs, accounts
}
RunWithSeedX(tb, tCfg, setupFn, seed, postRunActions...)
RunWithRandSourceX(tb, tCfg, setupFn, randSource, postRunActions...)
}

// RunWithSeedX entrypoint for custom chain setups.
// The function runs the full simulation test circle for the specified seed and setup function, followed by optional post-run actions.
func RunWithSeedX[T Tx](
// RunWithRandSourceX entrypoint for custom chain setups.
// The function runs the full simulation test circle for the specified random source and setup function, followed by optional post-run actions.
// when tb implements ResetTimer, the method is called after setup, before jumping into the main loop
func RunWithRandSourceX[T Tx](
tb testing.TB,
tCfg simtypes.Config,
setupChainStateFn func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account),
seed int64,
randSource rand.Source,
postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
) {
tb.Helper()
r := rand.New(rand.NewSource(seed))
r := rand.New(randSource)
rootCtx, done := context.WithCancel(context.Background())
defer done()

Expand All @@ -273,6 +287,10 @@ func RunWithSeedX[T Tx](
modules := testInstance.ModuleManager.Modules()
msgFactoriesFn := prepareSimsMsgFactories(r, modules, simsx.ParamWeightSource(emptySimParams))

if b, ok := tb.(interface{ ResetTimer() }); ok {
b.ResetTimer()
}

doMainLoop(
tb,
rootCtx,
Expand Down
15 changes: 10 additions & 5 deletions simapp/v2/sim_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ func TestFullAppSimulation(t *testing.T) {
RunWithSeeds[Tx](t, NewSimApp[Tx], AppConfig, DefaultSeeds)
}

// Scenario:
//
// Run 3 times a fresh node with the same seed,
// then the app hash should always be the same after n blocks
func TestAppStateDeterminism(t *testing.T) {
var seeds []int64
if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue {
Expand All @@ -44,13 +48,14 @@ func TestAppStateDeterminism(t *testing.T) {
tb.Helper()
mx.Lock()
defer mx.Unlock()
otherHashes, ok := appHashResults[ti.Seed]
seed := ti.RandSource.GetSeed()
otherHashes, ok := appHashResults[seed]
if !ok {
appHashResults[ti.Seed] = cs.AppHash
appHashResults[seed] = cs.AppHash
return
}
if !bytes.Equal(otherHashes, cs.AppHash) {
tb.Fatalf("non-determinism in seed %d", ti.Seed)
tb.Fatalf("non-determinism in seed %d", seed)
}
}
// run simulations
Expand Down Expand Up @@ -85,7 +90,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
chainID := SimAppChainID + "_2"

importGenesisChainStateFactory := func(ctx context.Context, r *rand.Rand) (TestInstance[Tx], ChainState[Tx], []simtypes.Account) {
testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.Seed)
testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.RandSource)
newCs := testInstance.InitializeChain(
tb,
ctx,
Expand All @@ -97,7 +102,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
return testInstance, newCs, accs
}
// run sims with new app setup from exported genesis
RunWithSeedX[Tx](tb, cfg, importGenesisChainStateFactory, ti.Seed)
RunWithRandSourceX[Tx](tb, cfg, importGenesisChainStateFactory, ti.RandSource)
}
RunWithSeeds[Tx, *SimApp[Tx]](t, appFactory, AppConfig, DefaultSeeds, exportAndStartChainFromGenesisPostAction)
}
89 changes: 89 additions & 0 deletions simsx/v2/rand_source.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package v2

import (
"bytes"
"encoding/binary"
"io"
"math/rand"
)

const (
rngMax = 1 << 63
rngMask = rngMax - 1
)

// RandSource defines an interface for random number sources with a method to retrieve the seed.
type RandSource interface {
rand.Source
GetSeed() int64
}

var (
_ RandSource = &SeededRandomSource{}
_ RandSource = &ByteSource{}
)

// SeededRandomSource wraps a random source with an associated seed value for reproducible random number generation.
// It implements the RandSource interface, allowing access to both the random source and seed.
type SeededRandomSource struct {
rand.Source
seed int64
}

// NewSeededRandSource constructor
func NewSeededRandSource(seed int64) *SeededRandomSource {
r := new(SeededRandomSource)
r.Seed(seed)
return r
}

func (r *SeededRandomSource) Seed(seed int64) {
r.seed = seed
r.Source = rand.NewSource(seed)
}

func (r SeededRandomSource) GetSeed() int64 {
return r.seed
}

// ByteSource offers deterministic pseudo-random numbers for math.Rand with fuzzer support.
// The 'seed' data is read in big endian to uint64. When exhausted,
// it falls back to a standard random number generator initialized with a specific 'seed' value.
type ByteSource struct {
seed *bytes.Reader
fallback *rand.Rand
}

// NewByteSource creates a new ByteSource with a specified byte slice and seed. This gives a fixed sequence of pseudo-random numbers.
// Initially, it utilizes the byte slice. Once that's exhausted, it continues generating numbers using the provided seed.
func NewByteSource(fuzzSeed []byte, seed int64) *ByteSource {
return &ByteSource{
seed: bytes.NewReader(fuzzSeed),
fallback: rand.New(rand.NewSource(seed)),
}
}

func (s *ByteSource) Uint64() uint64 {
if s.seed.Len() < 8 {
return s.fallback.Uint64()
}
var b [8]byte
if _, err := s.seed.Read(b[:]); err != nil && err != io.EOF {
panic(err) // Should not happen.
}
return binary.BigEndian.Uint64(b[:])
}

func (s *ByteSource) Int63() int64 {
return int64(s.Uint64() & rngMask)
}

// Seed is not supported and will panic
func (s *ByteSource) Seed(seed int64) {
panic("not supported")
}

// GetSeed is not supported and will panic
func (s ByteSource) GetSeed() int64 {
panic("not supported")
}
53 changes: 53 additions & 0 deletions simsx/v2/rand_source_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package v2

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestSeededRandSource(t *testing.T) {
const (
seed1 int64 = 1
firstValFromSeed1 int64 = 0x4d65822107fcfd52
secondValFromSeed1 int64 = 0x78629a0f5f3f164f
)
src := NewSeededRandSource(seed1)
for _, v := range []int64{firstValFromSeed1, secondValFromSeed1} {
assert.Equal(t, v, src.Int63())
}
assert.Equal(t, seed1, src.GetSeed())
}

func TestByteSource(t *testing.T) {
const (
seed1 = 1
firstValFromSeed1 = 0x4d65822107fcfd52
secondValFromSeed1 = 0x78629a0f5f3f164f
)
specs := map[string]struct {
fuzzSeed []byte
exp []uint64
}{
"fallback fuzz takes over": {
fuzzSeed: []byte{},
exp: []uint64{firstValFromSeed1, secondValFromSeed1},
},
"fuzzSeeds served first": {
fuzzSeed: []byte{
1, 2, 3, 4, 5, 6, 7, 8,
9, 10, 11, 12, 13, 14, 15, 16,
17, 18, // incomplete uin64, should be ignored
},
exp: []uint64{0x102030405060708, 0x90a0b0c0d0e0f10, firstValFromSeed1},
},
}
for name, spec := range specs {
t.Run(name, func(t *testing.T) {
byteSource := NewByteSource(spec.fuzzSeed, seed1)
for _, v := range spec.exp {
assert.Equal(t, v, byteSource.Uint64())
}
})
}
}

0 comments on commit 6bbcbf9

Please sign in to comment.