R4R: Fix governance simulation, more import/export work

Makefile

@@ -178,12 +178,7 @@ test_sim_gaia_fast:
 
 test_sim_gaia_import_export:
 	@echo "Running Gaia import/export simulation. This may take several minutes..."
-	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=4 -v -timeout 24h
-	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=11 -v -timeout 24h
-	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=12 -v -timeout 24h
-	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=13 -v -timeout 24h
-	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=414 -v -timeout 24h
-	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=4142 -v -timeout 24h
+	@bash scripts/import-export-sim.sh 50
 
 test_sim_gaia_multi_seed:
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"

PENDING.md

@@ -58,6 +58,8 @@ BUG FIXES
   * [\#2742](https://github.com/cosmos/cosmos-sdk/issues/2742) Fix time format of TimeoutCommit override
 
 * SDK
+
+  - \#2733 [x/gov, x/mock/simulation] Fix governance simulation, update x/gov import/export
 
 * Tendermint
   * [\#2797](https://github.com/tendermint/tendermint/pull/2797) AddressBook requires addresses to have IDs; Do not crap out immediately after sending pex addrs in seed mode

cmd/gaia/app/app.go

@@ -251,6 +251,8 @@ func (app *GaiaApp) initChainer(ctx sdk.Context, req abci.RequestInitChain) abci
 	gov.InitGenesis(ctx, app.govKeeper, genesisState.GovData)
 	mint.InitGenesis(ctx, app.mintKeeper, genesisState.MintData)
 	distr.InitGenesis(ctx, app.distrKeeper, genesisState.DistrData)
+
+	// validate genesis state
 	err = GaiaValidateGenesisState(genesisState)
 	if err != nil {
 		panic(err) // TODO find a way to do this w/o panics

scripts/import-export-sim.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+seeds=(1 2 4 7 9 20 32 123 124 582 1893 2989 3012 4728 37827 981928 87821 891823782 989182 89182391 \
+11 22 44 77 99 2020 3232 123123 124124 582582 18931893 29892989 30123012 47284728 37827)
+blocks=$1
+
+echo "Running multi-seed import-export simulation with seeds ${seeds[@]}"
+echo "Running $blocks blocks per seed"
+echo "Edit scripts/import-export-sim.sh to add new seeds. Keeping parameters in the file makes failures easy to reproduce."
+echo "This script will kill all sub-simulations on SIGINT/SIGTERM (i.e. Ctrl-C)."
+
+trap 'kill $(jobs -pr)' SIGINT SIGTERM
+
+tmpdir=$(mktemp -d)
+echo "Using temporary log directory: $tmpdir"
+
+sim() {
+  seed=$1
+	echo "Running import/export Gaia simulation with seed $seed. This may take awhile!"
+  file="$tmpdir/gaia-simulation-seed-$seed-date-$(date -Iseconds -u).stdout"
+  echo "Writing stdout to $file..."
+  go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=$blocks \
+    -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=$seed -v -timeout 24h > $file
+}
+
+i=0
+pids=()
+for seed in ${seeds[@]}; do
+  sim $seed &
+  pids[${i}]=$!
+  i=$(($i+1))
+  sleep 10 # start in order, nicer logs
+done
+
+echo "Simulation processes spawned, waiting for completion..."
+
+code=0
+
+i=0
+for pid in ${pids[*]}; do
+  wait $pid
+  last=$?
+  seed=${seeds[${i}]}
+  if [ $last -ne 0 ]
+  then
+    echo "Import/export simulation with seed $seed failed!"
+    code=1
+  else
+    echo "Import/export simulation with seed $seed OK"
+  fi
+  i=$(($i+1))
+done
+
+exit $code

types/store.go

@@ -197,15 +197,16 @@ func DiffKVStores(a KVStore, b KVStore, prefixesToSkip [][]byte) (kvA cmn.KVPair
 			kvB = cmn.KVPair{Key: iterB.Key(), Value: iterB.Value()}
 			iterB.Next()
 		}
+		if !bytes.Equal(kvA.Key, kvB.Key) {
+			return kvA, kvB, count, false
+		}
 		compareValue := true
 		for _, prefix := range prefixesToSkip {
+			// Skip value comparison if we matched a prefix
 			if bytes.Equal(kvA.Key[:len(prefix)], prefix) {
 				compareValue = false
 			}
 		}
-		if !bytes.Equal(kvA.Key, kvB.Key) {
-			return kvA, kvB, count, false
-		}
 		if compareValue && !bytes.Equal(kvA.Value, kvB.Value) {
 			return kvA, kvB, count, false
 		}

x/gov/simulation/msgs.go

@@ -96,7 +96,8 @@ func SimulateMsgSubmitProposal(k gov.Keeper) simulation.Operation {
 
 func simulateHandleMsgSubmitProposal(msg gov.MsgSubmitProposal, handler sdk.Handler, ctx sdk.Context, event func(string)) (action string, ok bool) {
 	ctx, _ = ctx.CacheContext()
-	handler(ctx, msg)
+	result := handler(ctx, msg)
+	ok = result.IsOK()
 	event(fmt.Sprintf("gov/MsgSubmitProposal/%v", ok))
 	action = fmt.Sprintf("TestMsgSubmitProposal: ok %v, msg %s", ok, msg.GetSignBytes())
 	return

x/slashing/genesis.go

@@ -73,13 +73,13 @@ func ExportGenesis(ctx sdk.Context, keeper Keeper) (data GenesisState) {
 	keeper.iterateValidatorSigningInfos(ctx, func(address sdk.ConsAddress, info ValidatorSigningInfo) (stop bool) {
 		bechAddr := address.String()
 		signingInfos[bechAddr] = info
-		array := []MissedBlock{}
+		localMissedBlocks := []MissedBlock{}
 
 		keeper.iterateValidatorMissedBlockBitArray(ctx, address, func(index int64, missed bool) (stop bool) {
-			array = append(array, MissedBlock{index, missed})
+			localMissedBlocks = append(localMissedBlocks, MissedBlock{index, missed})
 			return false
 		})
-		missedBlocks[bechAddr] = array
+		missedBlocks[bechAddr] = localMissedBlocks
 
 		return false
 	})

x/slashing/signing_info.go

@@ -84,10 +84,10 @@ func (k Keeper) setValidatorMissedBlockBitArray(ctx sdk.Context, address sdk.Con
 func (k Keeper) clearValidatorMissedBlockBitArray(ctx sdk.Context, address sdk.ConsAddress) {
 	store := ctx.KVStore(k.storeKey)
 	iter := sdk.KVStorePrefixIterator(store, GetValidatorMissedBlockBitArrayPrefixKey(address))
+	defer iter.Close()
 	for ; iter.Valid(); iter.Next() {
 		store.Delete(iter.Key())
 	}
-	iter.Close()
 }
 
 // Construct a new `ValidatorSigningInfo` struct

x/stake/genesis.go

@@ -29,17 +29,9 @@ func InitGenesis(ctx sdk.Context, keeper Keeper, data types.GenesisState) (res [
 	keeper.SetIntraTxCounter(ctx, data.IntraTxCounter)
 	keeper.SetLastTotalPower(ctx, data.LastTotalPower)
 
-	// We only need to set this if we're starting from a list of validators, not a state export
-	setBondIntraTxCounter := true
-	for _, validator := range data.Validators {
-		if validator.BondIntraTxCounter != 0 {
-			setBondIntraTxCounter = false
-		}
-	}
-
 	for i, validator := range data.Validators {
 		// set the intra-tx counter to the order the validators are presented, if necessary
-		if setBondIntraTxCounter {
+		if !data.Exported {
 			validator.BondIntraTxCounter = int16(i)
 		}
 		keeper.SetValidator(ctx, validator)
@@ -76,7 +68,15 @@ func InitGenesis(ctx sdk.Context, keeper Keeper, data types.GenesisState) (res [
 		keeper.InsertRedelegationQueue(ctx, red)
 	}
 
-	res = keeper.ApplyAndReturnValidatorSetUpdates(ctx)
+	// don't need to run Tendermint updates if we exported
+	if data.Exported {
+		for _, lv := range data.LastValidatorPowers {
+			keeper.SetLastValidatorPower(ctx, lv.Address, lv.Power)
+		}
+	} else {
+		res = keeper.ApplyAndReturnValidatorSetUpdates(ctx)
+	}
+
 	return
 }
 
@@ -100,16 +100,23 @@ func ExportGenesis(ctx sdk.Context, keeper Keeper) types.GenesisState {
 		redelegations = append(redelegations, red)
 		return false
 	})
+	var lastValidatorPowers []types.LastValidatorPower
+	keeper.IterateLastValidatorPowers(ctx, func(addr sdk.ValAddress, power sdk.Int) (stop bool) {
+		lastValidatorPowers = append(lastValidatorPowers, types.LastValidatorPower{addr, power})
+		return false
+	})
 
 	return types.GenesisState{
 		Pool:                 pool,
 		Params:               params,
 		IntraTxCounter:       intraTxCounter,
 		LastTotalPower:       lastTotalPower,
+		LastValidatorPowers:  lastValidatorPowers,
 		Validators:           validators,
 		Bonds:                bonds,
 		UnbondingDelegations: unbondingDelegations,
 		Redelegations:        redelegations,
+		Exported:             true,
 	}
 }
 

x/stake/keeper/keeper.go

@@ -112,6 +112,21 @@ func (k Keeper) SetLastValidatorPower(ctx sdk.Context, operator sdk.ValAddress,
 	store.Set(GetLastValidatorPowerKey(operator), bz)
 }
 
+// Iterate over last validator powers.
+func (k Keeper) IterateLastValidatorPowers(ctx sdk.Context, handler func(operator sdk.ValAddress, power sdk.Int) (stop bool)) {
+	store := ctx.KVStore(k.storeKey)
+	iter := sdk.KVStorePrefixIterator(store, LastValidatorPowerKey)
+	defer iter.Close()
+	for ; iter.Valid(); iter.Next() {
+		addr := sdk.ValAddress(iter.Key()[len(LastValidatorPowerKey):])
+		var power sdk.Int
+		k.cdc.MustUnmarshalBinaryLengthPrefixed(iter.Value(), &power)
+		if handler(addr, power) {
+			break
+		}
+	}
+}
+
 // Delete the last validator power.
 func (k Keeper) DeleteLastValidatorPower(ctx sdk.Context, operator sdk.ValAddress) {
 	store := ctx.KVStore(k.storeKey)

x/stake/types/genesis.go

@@ -10,10 +10,18 @@ type GenesisState struct {
 	Params               Params                `json:"params"`
 	IntraTxCounter       int16                 `json:"intra_tx_counter"`
 	LastTotalPower       sdk.Int               `json:"last_total_power"`
+	LastValidatorPowers  []LastValidatorPower  `json:"last_validator_powers"`
 	Validators           []Validator           `json:"validators"`
 	Bonds                []Delegation          `json:"bonds"`
 	UnbondingDelegations []UnbondingDelegation `json:"unbonding_delegations"`
 	Redelegations        []Redelegation        `json:"redelegations"`
+	Exported             bool                  `json:"exported"`
+}
+
+// Last validator power, needed for validator set update logic
+type LastValidatorPower struct {
+	Address sdk.ValAddress
+	Power   sdk.Int
 }
 
 func NewGenesisState(pool Pool, params Params, validators []Validator, bonds []Delegation) GenesisState {