From 5198f3649174c282b224f04cd355ef426b899dd1 Mon Sep 17 00:00:00 2001 From: Chill Validation <92176880+chillyvee@users.noreply.github.com> Date: Tue, 7 Mar 2023 01:44:26 +0900 Subject: [PATCH] fix: snapshot recover from exporter error (#13935) Co-authored-by: Marko Co-authored-by: Emmanuel T Odeke Co-authored-by: Marko Co-authored-by: Aleksandr Bezobchuk (cherry picked from commit 183dde8d007b5a3c971c743a80ae8e451cf81cd0) # Conflicts: # store/rootmulti/store.go --- store/rootmulti/store.go | 57 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 1847649adcfe..ee9032955132 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -202,6 +202,7 @@ func (rs *Store) LoadVersion(ver int64) error { func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error { infos := make(map[string]types.StoreInfo) + rs.logger.Debug("loadVersion", "ver", ver) cInfo := &types.CommitInfo{} // load old data if we are not version 0 @@ -238,12 +239,13 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error { for _, key := range storesKeys { storeParams := rs.storesParams[key] commitID := rs.getCommitID(infos, key.Name()) + rs.logger.Debug("loadVersion commitID", "key", key, "ver", ver, "hash", fmt.Sprintf("%x", commitID.Hash)) // If it has been added, set the initial version if upgrades.IsAdded(key.Name()) || upgrades.RenamedFrom(key.Name()) != "" { storeParams.initialVersion = uint64(ver) + 1 } else if commitID.Version != ver && storeParams.typ == types.StoreTypeIAVL { - return fmt.Errorf("version of store %s mismatch root store's version; expected %d got %d", key.Name(), ver, commitID.Version) + return fmt.Errorf("version of store %s mismatch root store's version; expected %d got %d; new stores should be added using StoreUpgrades", key.Name(), ver, commitID.Version) } store, err := rs.loadCommitStoreFromParams(key, commitID, storeParams) @@ -595,9 +597,11 @@ func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) ( return nil } - rs.logger.Debug("pruning heights", "heights", pruningHeights) + rs.logger.Debug("pruning store", "heights", pruningHeights) for key, store := range rs.stores { + rs.logger.Debug("pruning store", "key", key) // Also log store.name (a private variable)? + // If the store is wrapped with an inter-block cache, we must first unwrap // it to get the underlying IAVL store. if store.GetStoreType() != types.StoreTypeIAVL { @@ -763,8 +767,14 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error { // and the following messages contain a SnapshotNode (i.e. an ExportNode). Store changes // are demarcated by new SnapshotStore items. for _, store := range stores { + rs.logger.Debug("starting snapshot", "store", store.name, "height", height) exporter, err := store.Export(int64(height)) + if exporter == nil { + rs.logger.Error("snapshot failed; exporter is nil", "store", store.name) + return err + } if err != nil { + rs.logger.Error("snapshot failed; exporter error", "store", store.name, "err", err) return err } defer exporter.Close() @@ -797,8 +807,44 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error { }, }) if err != nil { + rs.logger.Error("snapshot failed; item store write failed", "store", store.name, "err", err) return err } +<<<<<<< HEAD +======= + + nodeCount := 0 + for { + node, err := exporter.Next() + if err == iavltree.ErrorExportDone { + rs.logger.Debug("Snapshot Done", "store", store.name, "nodeCount", nodeCount) + nodeCount = 0 + break + } else if err != nil { + return err + } + err = protoWriter.WriteMsg(&snapshottypes.SnapshotItem{ + Item: &snapshottypes.SnapshotItem_IAVL{ + IAVL: &snapshottypes.SnapshotIAVLItem{ + Key: node.Key, + Value: node.Value, + Height: int32(node.Height), + Version: node.Version, + }, + }, + }) + if err != nil { + return err + } + nodeCount++ + } + + return nil + }() + + if err != nil { + return err +>>>>>>> 183dde8d0 (fix: snapshot recover from exporter error (#13935)) } exporter.Close() } @@ -844,10 +890,17 @@ loop: return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "import failed") } defer importer.Close() + // Importer height must reflect the node height (which usually matches the block height, but not always) + rs.logger.Debug("restoring snapshot", "store", item.Store.Name) case *snapshottypes.SnapshotItem_IAVL: if importer == nil { +<<<<<<< HEAD return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(sdkerrors.ErrLogic, "received IAVL node item before store item") +======= + rs.logger.Error("failed to restore; received IAVL node item before store item") + return snapshottypes.SnapshotItem{}, errorsmod.Wrap(types.ErrLogic, "received IAVL node item before store item") +>>>>>>> 183dde8d0 (fix: snapshot recover from exporter error (#13935)) } if item.IAVL.Height > math.MaxInt8 { return snapshottypes.SnapshotItem{}, sdkerrors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v",