From 2a8306be9147a4858dfbbce023af05be58bb710c Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 3 Jul 2023 11:24:44 +0800 Subject: [PATCH 1/8] Problem: memiavl don't support rollback Closes: #1089 Update CHANGELOG.md Signed-off-by: yihuang Update memiavl/db.go Signed-off-by: yihuang revert int type changes --- CHANGELOG.md | 1 + integration_tests/configs/rollback.jsonnet | 8 +- memiavl/db.go | 225 +++++++++++++-------- memiavl/export.go | 3 +- memiavl/import.go | 2 +- store/rootmulti/store.go | 33 ++- 6 files changed, 182 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0bd6ccee..2b301a3ad1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - [#1042](https://github.com/crypto-org-chain/cronos/pull/1042) call Close method on app to cleanup resource on graceful shutdown ([ethermint commit](https://github.com/crypto-org-chain/ethermint/commit/0ea7b86532a1144f229961f94b4524d5889e874d)). - [#1083](https://github.com/crypto-org-chain/cronos/pull/1083) memiavl support both sdk 46 and 47 root hash rules. +- [#1091](https://github.com/crypto-org-chain/cronos/pull/1091) memiavl support rollback. ### Improvements diff --git a/integration_tests/configs/rollback.jsonnet b/integration_tests/configs/rollback.jsonnet index 64ea07a84a..b01ff47bc3 100644 --- a/integration_tests/configs/rollback.jsonnet +++ b/integration_tests/configs/rollback.jsonnet @@ -2,11 +2,13 @@ local config = import 'default.jsonnet'; config { 'cronos_777-1'+: { - 'app-config'+: { - 'iavl-disable-fastnode': true, - }, validators: super.validators + [{ name: 'fullnode', + 'app-config': { + memiavl: { + enable: true, + }, + }, }], }, } diff --git a/memiavl/db.go b/memiavl/db.go index d232656dea..177fd8ab1d 100644 --- a/memiavl/db.go +++ b/memiavl/db.go @@ -88,6 +88,10 @@ type Options struct { ZeroCopy bool // CacheSize defines the cache's max entry size for each memiavl store. CacheSize int + // LoadForOverwriting if true rollbacks the state, specifically the Load method will + // truncate the versions after the `TargetVersion`, the `TargetVersion` becomes the latest version. + // it do nothing if the target version is `0`. + LoadForOverwriting bool } const ( @@ -96,27 +100,21 @@ const ( ) func Load(dir string, opts Options) (*DB, error) { - snapshotName := "current" + currentVersion, err := currentVersion(dir) + if err != nil { + return nil, fmt.Errorf("fail to read current version: %w", err) + } + + version := currentVersion if opts.TargetVersion > 0 { - version, err := currentVersion(dir) + // find the biggest snapshot version that's less than or equal to the target version + version, err = seekSnapshot(dir, opts.TargetVersion) if err != nil { - return nil, fmt.Errorf("failed to load current version: %w", err) - } - - if int64(opts.TargetVersion) < version { - // try to load historical snapshots - snapshotName, err = seekSnapshot(dir, opts.TargetVersion) - if err != nil { - return nil, fmt.Errorf("failed to find snapshot: %w", err) - } - - if snapshotName == "" { - return nil, fmt.Errorf("target version is pruned: %d", opts.TargetVersion) - } + return nil, fmt.Errorf("fail to seek snapshot: %w", err) } } - path := filepath.Join(dir, snapshotName) + path := filepath.Join(dir, snapshotName(version)) mtree, err := LoadMultiTree(path, opts.ZeroCopy, opts.CacheSize) if err != nil { if opts.CreateIfMissing && os.IsNotExist(err) { @@ -141,6 +139,38 @@ func Load(dir string, opts Options) (*DB, error) { } } + if opts.LoadForOverwriting && opts.TargetVersion > 0 { + if version != currentVersion { + // downgrade `"current"` link first + opts.Logger.Info("downgrade current link, version: %d", version) + if err := updateCurrentSymlink(dir, snapshotName(version)); err != nil { + return nil, fmt.Errorf("fail to update current snapshot link: %w", err) + } + } + + // truncate the WAL + opts.Logger.Info("truncate WAL from back, version: %d", opts.TargetVersion) + if err := wal.TruncateBack(walIndex(int64(opts.TargetVersion), mtree.initialVersion)); err != nil { + return nil, fmt.Errorf("fail to truncate wal logs: %w", err) + } + + // prune snapshots that's larger than the target version + if err := traverseSnapshots(dir, false, func(version int64) (bool, error) { + if version <= int64(opts.TargetVersion) { + return true, nil + } + + if err := atomicRemoveDir(filepath.Join(dir, snapshotName(version))); err != nil { + opts.Logger.Error("fail to prune snapshot, version: %d", version) + } else { + opts.Logger.Info("prune snapshot, version: %d", version) + } + return false, nil + }); err != nil { + return nil, fmt.Errorf("fail to prune snapshots: %w", err) + } + } + db := &DB{ MultiTree: *mtree, logger: opts.Logger, @@ -270,45 +300,35 @@ func (db *DB) pruneSnapshots() { go func() { defer db.pruneSnapshotLock.Unlock() - currentName, err := os.Readlink(currentPath(db.dir)) - if err != nil { - db.logger.Error("failed to read current snapshot name", "err", err) - return - } - - entries, err := os.ReadDir(db.dir) + currentVersion, err := currentVersion(db.dir) if err != nil { - db.logger.Error("failed to read db dir", "err", err) + db.logger.Error("failed to read current snapshot version", "err", err) return } counter := db.snapshotKeepRecent - for i := len(entries) - 1; i >= 0; i-- { - name := entries[i].Name() - if !entries[i].IsDir() || !isSnapshotName(name) { - continue - } - - if name >= currentName { + if err := traverseSnapshots(db.dir, false, func(version int64) (bool, error) { + if version >= currentVersion { // ignore any newer snapshot directories, there could be ongoning snapshot rewrite. - continue + return false, nil } if counter > 0 { counter-- - continue + return false, nil } + name := snapshotName(version) db.logger.Info("prune snapshot", "name", name) - tmpPath := filepath.Join(db.dir, name+"-tmp") - if err := os.Rename(filepath.Join(db.dir, name), tmpPath); err != nil { - db.logger.Error("failed to move the snapshot to tmp file", "err", err) - continue - } - if err := os.RemoveAll(tmpPath); err != nil { + if err := atomicRemoveDir(filepath.Join(db.dir, name)); err != nil { db.logger.Error("failed to prune snapshot", "err", err) } + + return false, nil + }); err != nil { + db.logger.Error("fail to prune snapshots", "err", err) + return } // truncate WAL until the earliest remaining snapshot @@ -365,7 +385,7 @@ func (db *DB) Commit(changeSets []*NamedChangeSet) ([]byte, int64, error) { db.pendingUpgrades = db.pendingUpgrades[:0] - db.rewriteIfApplicable(v) + db.rewriteIfApplicable(uint32(v)) return hash, v, nil } @@ -436,8 +456,7 @@ func (db *DB) RewriteSnapshot() error { db.mtx.Lock() defer db.mtx.Unlock() - version := uint32(db.lastCommitInfo.Version) - snapshotDir := snapshotName(version) + snapshotDir := snapshotName(db.lastCommitInfo.Version) tmpDir := snapshotDir + "-tmp" path := filepath.Join(db.dir, tmpDir) if err := os.MkdirAll(path, os.ModePerm); err != nil { @@ -484,8 +503,8 @@ func (db *DB) reloadMultiTree(mtree *MultiTree) error { } // rewriteIfApplicable execute the snapshot rewrite strategy according to current height -func (db *DB) rewriteIfApplicable(height int64) { - if height%int64(db.snapshotInterval) != 0 { +func (db *DB) rewriteIfApplicable(height uint32) { + if height%db.snapshotInterval != 0 { return } @@ -610,7 +629,7 @@ func (db *DB) WriteSnapshot(dir string) error { return db.MultiTree.WriteSnapshot(dir) } -func snapshotName(version uint32) string { +func snapshotName(version int64) string { return fmt.Sprintf("%s%020d", SnapshotPrefix, version) } @@ -618,10 +637,6 @@ func currentPath(root string) string { return filepath.Join(root, "current") } -func snapshotPath(root string, version uint32) string { - return filepath.Join(root, snapshotName(version)) -} - func currentTmpPath(root string) string { return filepath.Join(root, "current-tmp") } @@ -645,53 +660,54 @@ func parseVersion(name string) (int64, error) { return 0, fmt.Errorf("invalid snapshot name %s", name) } - return strconv.ParseInt(name[len(SnapshotPrefix):], 10, 64) -} - -// seekSnapshot find the biggest snapshot that's smaller than or equal to target version, -// returns the directory name, if not found, returns empty string. -func seekSnapshot(root string, version uint32) (string, error) { - entries, err := os.ReadDir(root) + v, err := strconv.ParseInt(name[len(SnapshotPrefix):], 10, 32) if err != nil { - return "", err + return 0, fmt.Errorf("snapshot version overflows: %d", err) } - targetName := snapshotName(version) - for i := len(entries) - 1; i >= 0; i-- { - name := entries[i].Name() - if !entries[i].IsDir() || !isSnapshotName(name) { - continue - } + return v, nil +} - if name <= targetName { - return name, nil +// seekSnapshot find the biggest snapshot version that's smaller than or equal to the target version, +// returns 0 if not found. +func seekSnapshot(root string, targetVersion uint32) (int64, error) { + var ( + snapshotVersion int64 + found bool + ) + if err := traverseSnapshots(root, false, func(version int64) (bool, error) { + if version <= int64(targetVersion) { + found = true + snapshotVersion = version + return true, nil } + return false, nil + }); err != nil { + return 0, err } - return "", nil + if !found { + return 0, fmt.Errorf("target version is pruned: %d", targetVersion) + } + + return snapshotVersion, nil } // firstSnapshotVersion returns the earliest snapshot name in the db func firstSnapshotVersion(root string) (int64, error) { - entries, err := os.ReadDir(root) - if err != nil { + var found int64 + if err := traverseSnapshots(root, true, func(version int64) (bool, error) { + found = version + return true, nil + }); err != nil { return 0, err } - for _, entry := range entries { - if !entry.IsDir() || !isSnapshotName(entry.Name()) { - continue - } - - version, err := parseVersion(entry.Name()) - if err != nil { - return 0, err - } - - return version, nil + if found == 0 { + return 0, errors.New("empty memiavl db") } - return 0, errors.New("empty memiavl db") + return found, nil } func walPath(root string) string { @@ -727,6 +743,55 @@ func updateCurrentSymlink(dir, snapshot string) error { return os.Rename(tmpPath, currentPath(dir)) } +// traverseSnapshots traverse the snapshot list in specified order. +func traverseSnapshots(dir string, ascending bool, callback func(int64) (bool, error)) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + + process := func(entry os.DirEntry) (bool, error) { + if !entry.IsDir() || !isSnapshotName(entry.Name()) { + return false, nil + } + + version, err := parseVersion(entry.Name()) + if err != nil { + return true, fmt.Errorf("invalid snapshot name: %w", err) + } + + return callback(version) + } + + if ascending { + for i := 0; i < len(entries); i++ { + stop, err := process(entries[i]) + if stop || err != nil { + return err + } + } + } else { + for i := len(entries) - 1; i >= 0; i-- { + stop, err := process(entries[i]) + if stop || err != nil { + return err + } + } + } + + return nil +} + +// atomicRemoveDir is equavalent to `mv snapshot snapshot-tmp && rm -r snapshot-tmp` +func atomicRemoveDir(path string) error { + tmpPath := path + "-tmp" + if err := os.Rename(path, tmpPath); err != nil { + return err + } + + return os.RemoveAll(tmpPath) +} + type walEntry struct { index uint64 data *WALEntry diff --git a/memiavl/export.go b/memiavl/export.go index f2ce2144f0..daa9df34bb 100644 --- a/memiavl/export.go +++ b/memiavl/export.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "path/filepath" "cosmossdk.io/errors" snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types" @@ -46,7 +47,7 @@ func (db *DB) Snapshot(height uint64, protoWriter protoio.Writer) (returnErr err if int64(version) > curVersion { return fmt.Errorf("snapshot is not created yet: height: %d", version) } - mtree, err = LoadMultiTree(snapshotPath(db.dir, version), true, 0) + mtree, err = LoadMultiTree(filepath.Join(db.dir, snapshotName(int64(version))), true, 0) if err != nil { return errors.Wrapf(err, "snapshot don't exists: height: %d", version) } diff --git a/memiavl/import.go b/memiavl/import.go index 4872438cf3..76ab6ee225 100644 --- a/memiavl/import.go +++ b/memiavl/import.go @@ -24,7 +24,7 @@ func Import( if height > math.MaxUint32 { return snapshottypes.SnapshotItem{}, fmt.Errorf("version overflows uint32: %d", height) } - snapshotDir := snapshotName(uint32(height)) + snapshotDir := snapshotName(int64(height)) tmpDir := snapshotDir + "-tmp" // Import nodes into stores. The first item is expected to be a SnapshotItem containing diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index d0fe73c5d4..f04b713120 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -1,7 +1,6 @@ package rootmulti import ( - stderrors "errors" "fmt" "io" "math" @@ -286,7 +285,6 @@ func (rs *Store) LoadVersionAndUpgrade(version int64, upgrades *types.StoreUpgra } opts := rs.opts - opts.Logger = rs.logger.With("module", "memiavl") opts.CreateIfMissing = true opts.InitialStores = initialStores opts.TargetVersion = uint32(version) @@ -393,12 +391,37 @@ func (rs *Store) SetLazyLoading(lazyLoading bool) { } func (rs *Store) SetMemIAVLOptions(opts memiavl.Options) { + if opts.Logger == nil { + opts.Logger = rs.logger.With("module", "memiavl") + } rs.opts = opts } -// Implements interface CommitMultiStore -func (rs *Store) RollbackToVersion(version int64) error { - return stderrors.New("rootmulti store don't support rollback") +// RollbackToVersion delete the versions after `target` and update the latest version. +// it should only be called in standalone cli commands. +func (rs *Store) RollbackToVersion(target int64) error { + if target <= 0 { + return fmt.Errorf("invalid rollback height target: %d", target) + } + + if target > math.MaxUint32 { + return fmt.Errorf("rollback height target %d exceeds max uint32", target) + } + + if rs.db != nil { + if err := rs.db.Close(); err != nil { + return err + } + } + + opts := rs.opts + opts.TargetVersion = uint32(target) + opts.LoadForOverwriting = true + + var err error + rs.db, err = memiavl.Load(rs.dir, opts) + + return err } // Implements interface CommitMultiStore From 8d48988d99f41d2fcc1b9bdd6fdfb4b355510405 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 3 Jul 2023 11:42:42 +0800 Subject: [PATCH 2/8] int type --- memiavl/db.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/memiavl/db.go b/memiavl/db.go index 177fd8ab1d..cb26dc5bfa 100644 --- a/memiavl/db.go +++ b/memiavl/db.go @@ -385,7 +385,7 @@ func (db *DB) Commit(changeSets []*NamedChangeSet) ([]byte, int64, error) { db.pendingUpgrades = db.pendingUpgrades[:0] - db.rewriteIfApplicable(uint32(v)) + db.rewriteIfApplicable(v) return hash, v, nil } @@ -503,8 +503,8 @@ func (db *DB) reloadMultiTree(mtree *MultiTree) error { } // rewriteIfApplicable execute the snapshot rewrite strategy according to current height -func (db *DB) rewriteIfApplicable(height uint32) { - if height%db.snapshotInterval != 0 { +func (db *DB) rewriteIfApplicable(height int64) { + if height%int64(db.snapshotInterval) != 0 { return } From 4ac394f94d4028e7d2716a2ea17a31b0d3ef3287 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 3 Jul 2023 11:48:00 +0800 Subject: [PATCH 3/8] remove TODO --- store/rootmulti/store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index f04b713120..b10d05164a 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -216,13 +216,13 @@ func (rs *Store) Restore(height uint64, format uint32, protoReader protoio.Reade } // Implements interface Snapshotter +// not needed, memiavl manage it's own snapshot/pruning strategy func (rs *Store) PruneSnapshotHeight(height int64) { - // TODO } // Implements interface Snapshotter +// not needed, memiavl manage it's own snapshot/pruning strategy func (rs *Store) SetSnapshotInterval(snapshotInterval uint64) { - // TODO } // Implements interface CommitMultiStore From b58e15f1618a6dbbc93db5e85af18c617083c805 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 3 Jul 2023 11:51:07 +0800 Subject: [PATCH 4/8] naming --- memiavl/db.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/memiavl/db.go b/memiavl/db.go index cb26dc5bfa..12e9add54c 100644 --- a/memiavl/db.go +++ b/memiavl/db.go @@ -105,16 +105,16 @@ func Load(dir string, opts Options) (*DB, error) { return nil, fmt.Errorf("fail to read current version: %w", err) } - version := currentVersion + snapshotVersion := currentVersion if opts.TargetVersion > 0 { // find the biggest snapshot version that's less than or equal to the target version - version, err = seekSnapshot(dir, opts.TargetVersion) + snapshotVersion, err = seekSnapshot(dir, opts.TargetVersion) if err != nil { return nil, fmt.Errorf("fail to seek snapshot: %w", err) } } - path := filepath.Join(dir, snapshotName(version)) + path := filepath.Join(dir, snapshotName(snapshotVersion)) mtree, err := LoadMultiTree(path, opts.ZeroCopy, opts.CacheSize) if err != nil { if opts.CreateIfMissing && os.IsNotExist(err) { @@ -140,10 +140,10 @@ func Load(dir string, opts Options) (*DB, error) { } if opts.LoadForOverwriting && opts.TargetVersion > 0 { - if version != currentVersion { + if snapshotVersion != currentVersion { // downgrade `"current"` link first - opts.Logger.Info("downgrade current link, version: %d", version) - if err := updateCurrentSymlink(dir, snapshotName(version)); err != nil { + opts.Logger.Info("downgrade current link, version: %d", snapshotVersion) + if err := updateCurrentSymlink(dir, snapshotName(snapshotVersion)); err != nil { return nil, fmt.Errorf("fail to update current snapshot link: %w", err) } } From fc883e9d9c3a36463395161a9451d0415a4283b9 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 3 Jul 2023 13:05:03 +0800 Subject: [PATCH 5/8] cleanup --- memiavl/db.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/memiavl/db.go b/memiavl/db.go index 12e9add54c..4269ae1318 100644 --- a/memiavl/db.go +++ b/memiavl/db.go @@ -100,21 +100,17 @@ const ( ) func Load(dir string, opts Options) (*DB, error) { - currentVersion, err := currentVersion(dir) - if err != nil { - return nil, fmt.Errorf("fail to read current version: %w", err) - } - - snapshotVersion := currentVersion + snapshot := "current" if opts.TargetVersion > 0 { // find the biggest snapshot version that's less than or equal to the target version - snapshotVersion, err = seekSnapshot(dir, opts.TargetVersion) + snapshotVersion, err := seekSnapshot(dir, opts.TargetVersion) if err != nil { return nil, fmt.Errorf("fail to seek snapshot: %w", err) } + snapshot = snapshotName(snapshotVersion) } - path := filepath.Join(dir, snapshotName(snapshotVersion)) + path := filepath.Join(dir, snapshot) mtree, err := LoadMultiTree(path, opts.ZeroCopy, opts.CacheSize) if err != nil { if opts.CreateIfMissing && os.IsNotExist(err) { @@ -140,10 +136,15 @@ func Load(dir string, opts Options) (*DB, error) { } if opts.LoadForOverwriting && opts.TargetVersion > 0 { - if snapshotVersion != currentVersion { + currentSnapshot, err := os.Readlink(currentPath(dir)) + if err != nil { + return nil, fmt.Errorf("fail to read current version: %w", err) + } + + if snapshot != currentSnapshot { // downgrade `"current"` link first - opts.Logger.Info("downgrade current link, version: %d", snapshotVersion) - if err := updateCurrentSymlink(dir, snapshotName(snapshotVersion)); err != nil { + opts.Logger.Info("downgrade current link to %d", snapshot) + if err := updateCurrentSymlink(dir, snapshot); err != nil { return nil, fmt.Errorf("fail to update current snapshot link: %w", err) } } From 4998009f30a56c4ad9b8700fbd5f05dff6d66647 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 3 Jul 2023 17:25:50 +0800 Subject: [PATCH 6/8] Update memiavl/db.go Co-authored-by: mmsqe Signed-off-by: yihuang --- memiavl/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memiavl/db.go b/memiavl/db.go index 4269ae1318..63baad14e0 100644 --- a/memiavl/db.go +++ b/memiavl/db.go @@ -143,7 +143,7 @@ func Load(dir string, opts Options) (*DB, error) { if snapshot != currentSnapshot { // downgrade `"current"` link first - opts.Logger.Info("downgrade current link to %d", snapshot) + opts.Logger.Info("downgrade current link to %s", snapshot) if err := updateCurrentSymlink(dir, snapshot); err != nil { return nil, fmt.Errorf("fail to update current snapshot link: %w", err) } From fff326a8507a9eb8d8346d64c1f8565991475dd1 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 3 Jul 2023 17:26:07 +0800 Subject: [PATCH 7/8] Update store/rootmulti/store.go Co-authored-by: mmsqe Signed-off-by: yihuang --- store/rootmulti/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index b10d05164a..775075bfa0 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -221,7 +221,7 @@ func (rs *Store) PruneSnapshotHeight(height int64) { } // Implements interface Snapshotter -// not needed, memiavl manage it's own snapshot/pruning strategy +// not needed, memiavl manage its own snapshot/pruning strategy func (rs *Store) SetSnapshotInterval(snapshotInterval uint64) { } From 2750120b4878e43ca45defc03abfa2961707e967 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 3 Jul 2023 17:26:14 +0800 Subject: [PATCH 8/8] Update store/rootmulti/store.go Co-authored-by: mmsqe Signed-off-by: yihuang --- store/rootmulti/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 775075bfa0..b529aeb32a 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -216,7 +216,7 @@ func (rs *Store) Restore(height uint64, format uint32, protoReader protoio.Reade } // Implements interface Snapshotter -// not needed, memiavl manage it's own snapshot/pruning strategy +// not needed, memiavl manage its own snapshot/pruning strategy func (rs *Store) PruneSnapshotHeight(height int64) { }