Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add in-place store migrations #8485

Merged
merged 33 commits into from
Feb 10, 2021
Merged
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
10d72d7
Add 1st version of migrate
amaury1093 Feb 1, 2021
8036fca
Merge branch 'master' of ssh://github.com/cosmos/cosmos-sdk into am-8…
amaury1093 Feb 2, 2021
a9a406e
Put migration logic into Configurator
amaury1093 Feb 2, 2021
9ed3987
add test to bank store migration
amaury1093 Feb 2, 2021
a095d3a
add test for configurator
amaury1093 Feb 2, 2021
d550bff
Merge branch 'master' into am-8345-migration
amaury1093 Feb 2, 2021
b7141f9
Error if no migration found
amaury1093 Feb 3, 2021
2d554ce
Remove RunMigrations from Configurator interface
amaury1093 Feb 3, 2021
3df3f44
Update spec
amaury1093 Feb 3, 2021
ba6bd44
Rename folders
amaury1093 Feb 3, 2021
197b7ce
Merge branch 'master' into am-8345-migration
amaury1093 Feb 3, 2021
424932c
copy-paste from keys.go
amaury1093 Feb 3, 2021
254a71f
Merge branch 'am-8345-migration' of ssh://github.com/cosmos/cosmos-sd…
amaury1093 Feb 3, 2021
b6c0714
Fix nil map
amaury1093 Feb 3, 2021
9ab7f13
rename function
amaury1093 Feb 3, 2021
fc076f5
Update simapp/app.go
amaury1093 Feb 5, 2021
ae7b7dc
Update simapp/app_test.go
amaury1093 Feb 5, 2021
2c26734
Adderss reviews
amaury1093 Feb 8, 2021
eb15047
Merge branch 'am-8345-migration' of ssh://github.com/cosmos/cosmos-sd…
amaury1093 Feb 8, 2021
291a9e6
Fix tests
amaury1093 Feb 8, 2021
41d29ed
Update testutil/context.go
amaury1093 Feb 8, 2021
33494dc
Update docs for ConsensusVersion
amaury1093 Feb 8, 2021
e3a2412
Rename to forVersion
amaury1093 Feb 8, 2021
6db31d4
Merge branch 'am-8345-migration' of ssh://github.com/cosmos/cosmos-sd…
amaury1093 Feb 8, 2021
55f547e
Fix tests
amaury1093 Feb 8, 2021
8f3b9d4
Check error early
amaury1093 Feb 8, 2021
29b85b7
Return 1 for intiial version
amaury1093 Feb 8, 2021
41e0339
Use MigrationKeeper
amaury1093 Feb 10, 2021
d922003
Fix test
amaury1093 Feb 10, 2021
f422b93
Revert adding marshaler to Configurator
amaury1093 Feb 10, 2021
8cf88fe
Godoc updates
amaury1093 Feb 10, 2021
5add13a
Update docs
amaury1093 Feb 10, 2021
2cef291
Merge branch 'master' into am-8345-migration
aaronc Feb 10, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add test to bank store migration
  • Loading branch information
amaury1093 committed Feb 2, 2021
commit 9ed398766574f0fb0ce87d36c2cf40828e7ce134
25 changes: 25 additions & 0 deletions testutil/context.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package testutil

import (
"github.com/tendermint/tendermint/libs/log"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
dbm "github.com/tendermint/tm-db"

"github.com/cosmos/cosmos-sdk/store"
sdk "github.com/cosmos/cosmos-sdk/types"
)

// DefaultContext creates a sdk.Context that can be used in tests.
amaury1093 marked this conversation as resolved.
Show resolved Hide resolved
func DefaultContext(key sdk.StoreKey, tkey sdk.StoreKey) sdk.Context {
db := dbm.NewMemDB()
cms := store.NewCommitMultiStore(db)
cms.MountStoreWithDB(key, sdk.StoreTypeIAVL, db)
cms.MountStoreWithDB(tkey, sdk.StoreTypeTransient, db)
err := cms.LoadLatestVersion()
if err != nil {
panic(err)
}
ctx := sdk.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger())

return ctx
}
17 changes: 3 additions & 14 deletions types/context_test.go
Original file line number Diff line number Diff line change
@@ -8,13 +8,11 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/suite"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
dbm "github.com/tendermint/tm-db"

"github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1"
"github.com/cosmos/cosmos-sdk/store"
"github.com/cosmos/cosmos-sdk/tests/mocks"
"github.com/cosmos/cosmos-sdk/testutil"
"github.com/cosmos/cosmos-sdk/types"
)

@@ -26,23 +24,14 @@ func TestContextTestSuite(t *testing.T) {
suite.Run(t, new(contextTestSuite))
}

func (s *contextTestSuite) defaultContext(key types.StoreKey) types.Context {
db := dbm.NewMemDB()
cms := store.NewCommitMultiStore(db)
cms.MountStoreWithDB(key, types.StoreTypeIAVL, db)
s.Require().NoError(cms.LoadLatestVersion())
ctx := types.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger())
return ctx
}

func (s *contextTestSuite) TestCacheContext() {
key := types.NewKVStoreKey(s.T().Name() + "_TestCacheContext")
k1 := []byte("hello")
v1 := []byte("world")
k2 := []byte("key")
v2 := []byte("value")

ctx := s.defaultContext(key)
ctx := testutil.DefaultContext(key, types.NewTransientStoreKey("transient_"+s.T().Name()))
store := ctx.KVStore(key)
store.Set(k1, v1)
s.Require().Equal(v1, store.Get(k1))
@@ -64,7 +53,7 @@ func (s *contextTestSuite) TestCacheContext() {

func (s *contextTestSuite) TestLogContext() {
key := types.NewKVStoreKey(s.T().Name())
ctx := s.defaultContext(key)
ctx := testutil.DefaultContext(key, types.NewTransientStoreKey("transient_"+s.T().Name()))
ctrl := gomock.NewController(s.T())
s.T().Cleanup(ctrl.Finish)

6 changes: 3 additions & 3 deletions types/module/configurator.go
Original file line number Diff line number Diff line change
@@ -25,9 +25,9 @@ type Configurator interface {
// `fromVersion` to version `fromVersion+1`.
RegisterMigration(moduleName string, fromVersion uint64, handler func(store sdk.KVStore) error)

// RunMigration runs all in-place store migrations for a module from a
// RunMigrations runs all in-place store migrations for a module from a
// given version to another version.
RunMigration(ctx sdk.Context, moduleName string, fromVersion, toVersion uint64) error
RunMigrations(ctx sdk.Context, moduleName string, fromVersion, toVersion uint64) error
amaury1093 marked this conversation as resolved.
Show resolved Hide resolved
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
}
Marshaler() codec.Marshaler
}


type configurator struct {
@@ -63,7 +63,7 @@ func (c configurator) RegisterMigration(moduleName string, fromVersion uint64, h
}

// RunMigration implements the Configurator.RunMigration method
func (c configurator) RunMigration(ctx sdk.Context, moduleName string, fromVersion, toVersion uint64) error {
func (c configurator) RunMigrations(ctx sdk.Context, moduleName string, fromVersion, toVersion uint64) error {
storeKey, found := c.storeKeys[moduleName]
if !found {
return sdkerrors.Wrapf(sdkerrors.ErrNotFound, "store key for module %s not found", moduleName)
8 changes: 4 additions & 4 deletions types/module/module.go
Original file line number Diff line number Diff line change
@@ -334,25 +334,25 @@ func (m *Manager) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) map[st
return genesisData
}

// MigrateStore performs in-place store migrations. This function is not called
// RunMigrations performs in-place store migrations. This function is not called
// automatically, it is meant to be called from an x/upgrade UpgradeHandler.
// `migrationsMap` is a map of moduleName to fromVersion (unit64), where
// fromVersion denotes the version from which we should migrate the module.
//
// Example:
// cfg := module.NewConfigurator(...)
// app.UpgradeKeeper.SetUpgradeHandler("store-migration", func(ctx sdk.Context, plan upgradetypes.Plan) {
// err := app.mm.MigrateStore(ctx, cfg, map[string]unint64{
// err := app.mm.RunMigrations(ctx, cfg, map[string]unint64{
// "bank": 1, // Migrate x/bank from v1 to current x/bank's ConsensusVersion
// "staking": 8, // Migrate x/staking from v8 to current x/staking's ConsensusVersion
// })
// if err != nil {
// panic(err)
// }
// })
func (m Manager) MigrateStore(ctx sdk.Context, cfg Configurator, migrationsMap map[string]uint64) error {
func (m Manager) RunMigrations(ctx sdk.Context, cfg Configurator, migrationsMap map[string]uint64) error {
for moduleName, module := range m.Modules {
err := cfg.RunMigration(ctx, moduleName, migrationsMap[moduleName], module.ConsensusVersion())
err := cfg.RunMigrations(ctx, moduleName, migrationsMap[moduleName], module.ConsensusVersion())
if err != nil {
return err
}
6 changes: 3 additions & 3 deletions x/bank/legacy/v042/store.go
Original file line number Diff line number Diff line change
@@ -41,17 +41,17 @@ func StoreMigration(store sdk.KVStore) error {
// new key is of format
// prefix (0x02) || addrLen (1 byte) || addrBytes || denomBytes
oldStore := prefix.NewStore(store, v040bank.BalancesPrefix)
newStore := prefix.NewStore(store, BalancesPrefix)

oldStoreIter := oldStore.Iterator(nil, nil)
defer oldStoreIter.Close()

for ; oldStoreIter.Valid(); oldStoreIter.Next() {
addr := v040bank.AddressFromBalancesStore(oldStoreIter.Key())
denom := oldStoreIter.Key()[1+v040auth.AddrLen:]
denom := oldStoreIter.Key()[v040auth.AddrLen:]
newStoreKey := append(CreateAccountBalancesPrefix(addr), denom...)

newStore.Set(newStoreKey, oldStoreIter.Value()) // Values don't change.
// Set new key on store. Values don't change.
store.Set(newStoreKey, oldStoreIter.Value())
oldStore.Delete(oldStoreIter.Key())
}

33 changes: 31 additions & 2 deletions x/bank/legacy/v042/store_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,36 @@
package v042_test

import "testing"
import (
"testing"

"github.com/stretchr/testify/require"

"github.com/cosmos/cosmos-sdk/testutil"
"github.com/cosmos/cosmos-sdk/testutil/testdata"
sdk "github.com/cosmos/cosmos-sdk/types"
v040bank "github.com/cosmos/cosmos-sdk/x/bank/legacy/v040"
v042bank "github.com/cosmos/cosmos-sdk/x/bank/legacy/v042"
)

func TestStoreMigration(t *testing.T) {
// TODO
bankKey := sdk.NewKVStoreKey("bank")
ctx := testutil.DefaultContext(bankKey, sdk.NewTransientStoreKey("transient_test"))
store := ctx.KVStore(bankKey)

_, _, addr := testdata.KeyTestPubAddr()
denom := []byte("foo")
value := []byte("bar")

oldKey := append(append(v040bank.BalancesPrefix, addr...), denom...)
store.Set(oldKey, value)

err := v042bank.StoreMigration(store)
require.NoError(t, err)

newKey := append(v042bank.CreateAccountBalancesPrefix(addr), denom...)
// -7 because we replaced "balances" with 0x02,
// +1 because we added length-prefix to address.
require.Equal(t, len(oldKey)-7+1, len(newKey))
require.Nil(t, store.Get(oldKey))
require.Equal(t, value, store.Get(newKey))
}
24 changes: 3 additions & 21 deletions x/params/keeper/common_test.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
package keeper_test

import (
"github.com/tendermint/tendermint/libs/log"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
dbm "github.com/tendermint/tm-db"

"github.com/cosmos/cosmos-sdk/simapp"

"github.com/cosmos/cosmos-sdk/codec"
"github.com/cosmos/cosmos-sdk/store"
"github.com/cosmos/cosmos-sdk/simapp"
"github.com/cosmos/cosmos-sdk/testutil"
sdk "github.com/cosmos/cosmos-sdk/types"
paramskeeper "github.com/cosmos/cosmos-sdk/x/params/keeper"
)
@@ -18,7 +13,7 @@ func testComponents() (*codec.LegacyAmino, sdk.Context, sdk.StoreKey, sdk.StoreK
legacyAmino := createTestCodec()
mkey := sdk.NewKVStoreKey("test")
tkey := sdk.NewTransientStoreKey("transient_test")
ctx := defaultContext(mkey, tkey)
ctx := testutil.DefaultContext(mkey, tkey)
keeper := paramskeeper.NewKeeper(marshaler, legacyAmino, mkey, tkey)

return legacyAmino, ctx, mkey, tkey, keeper
@@ -37,16 +32,3 @@ func createTestCodec() *codec.LegacyAmino {
cdc.RegisterConcrete(invalid{}, "test/invalid", nil)
return cdc
}

func defaultContext(key sdk.StoreKey, tkey sdk.StoreKey) sdk.Context {
db := dbm.NewMemDB()
cms := store.NewCommitMultiStore(db)
cms.MountStoreWithDB(key, sdk.StoreTypeIAVL, db)
cms.MountStoreWithDB(tkey, sdk.StoreTypeTransient, db)
err := cms.LoadLatestVersion()
if err != nil {
panic(err)
}
ctx := sdk.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger())
return ctx
}