From aade791635656a36a322e50ec8f862e2f459fc5f Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Sat, 29 Jul 2023 10:46:40 -0300 Subject: [PATCH 1/5] export overlay preimages tool Signed-off-by: Ignacio Hagopian --- cmd/geth/chaincmd.go | 29 +++++++++++++++++++ cmd/geth/main.go | 1 + cmd/utils/cmd.go | 68 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c89f736169ef..add1f6b763ce 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -143,6 +143,17 @@ It's deprecated, please use "geth db import" instead. Description: ` The export-preimages command exports hash preimages to an RLP encoded stream. It's deprecated, please use "geth db export" instead. +`, + } + exportOverlayPreimagesCommand = &cli.Command{ + Action: exportOverlayPreimages, + Name: "export-overlay-preimages", + Usage: "Export the preimage in overlay tree migration order", + ArgsUsage: "", + Flags: flags.Merge([]cli.Flag{}, utils.DatabasePathFlags), + Description: ` +The export-overlay-preimages command exports hash preimages to a flat file, in exactly +the expected order for the overlay tree migration. `, } dumpCommand = &cli.Command{ @@ -394,6 +405,24 @@ func exportPreimages(ctx *cli.Context) error { return nil } +// exportOverlayPreimages dumps the preimage data to a flat file. +func exportOverlayPreimages(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + utils.Fatalf("This command requires an argument.") + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, _ := utils.MakeChain(ctx, stack) + + start := time.Now() + if err := utils.ExportOverlayPreimages(chain, ctx.Args().First()); err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} + func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { db := utils.MakeChainDatabase(ctx, stack, true) var header *types.Header diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 5d54ee41ca2f..25a32e250a70 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -213,6 +213,7 @@ func init() { exportCommand, importPreimagesCommand, exportPreimagesCommand, + exportOverlayPreimagesCommand, removedbCommand, dumpCommand, dumpGenesisCommand, diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 15a601da7dbc..a54891a362cf 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -379,6 +380,73 @@ func ExportPreimages(db ethdb.Database, fn string) error { return nil } +// ExportOverlayPreimages exports all known hash preimages into the specified file, +// in the same order as expected by the overlay tree migration. +func ExportOverlayPreimages(chain *core.BlockChain, fn string) error { + log.Info("Exporting preimages", "file", fn) + + fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer fh.Close() + + writer := bufio.NewWriter(fh) + defer writer.Flush() + + statedb, err := chain.State() + if err != nil { + return fmt.Errorf("failed to open statedb: %w", err) + } + + mptRoot := chain.CurrentBlock().Root() + + accIt, err := statedb.Snaps().AccountIterator(mptRoot, common.Hash{}) + if err != nil { + return err + } + defer accIt.Release() + + count := 0 + for accIt.Next() { + acc, err := snapshot.FullAccount(accIt.Account()) + if err != nil { + return fmt.Errorf("invalid account encountered during traversal: %s", err) + } + addr := rawdb.ReadPreimage(statedb.Database().DiskDB(), accIt.Hash()) + if len(addr) != 20 { + return fmt.Errorf("addr len is zero is not 32: %d", len(addr)) + } + if _, err := writer.Write(addr); err != nil { + return fmt.Errorf("failed to write addr preimage: %w", err) + } + + if acc.HasStorage() { + stIt, err := statedb.Snaps().StorageIterator(mptRoot, accIt.Hash(), common.Hash{}) + if err != nil { + return fmt.Errorf("failed to create storage iterator: %w", err) + } + for stIt.Next() { + slotnr := rawdb.ReadPreimage(statedb.Database().DiskDB(), stIt.Hash()) + if len(slotnr) != 32 { + return fmt.Errorf("slotnr not 32 len") + } + if _, err := writer.Write(slotnr); err != nil { + return fmt.Errorf("failed to write slotnr preimage: %w", err) + } + } + stIt.Release() + } + count++ + if count%100000 == 0 { + log.Info("Last exported account", "account", accIt.Hash()) + } + } + + log.Info("Exported preimages", "file", fn) + return nil +} + // exportHeader is used in the export/import flow. When we do an export, // the first element we output is the exportHeader. // Whenever a backwards-incompatible change is made, the Version header From 148086b3589d7eafdd171cfb83adda02f8e8421a Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Mon, 31 Jul 2023 15:38:31 -0300 Subject: [PATCH 2/5] use preimages flat file in overlay tree migration logic Signed-off-by: Ignacio Hagopian --- cmd/utils/cmd.go | 4 ++ core/state/database.go | 42 ++++++++++++++----- core/state_processor.go | 89 ++++++++++++++++++++++++++++++----------- light/trie.go | 16 +++++++- 4 files changed, 117 insertions(+), 34 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index a54891a362cf..6464999ea49b 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -170,6 +170,10 @@ func ImportChain(chain *core.BlockChain, fn string) error { } defer fh.Close() + if _, err := fh.Seek(18224628422, 0); err != nil { + panic(err) + } + var reader io.Reader = fh if strings.HasSuffix(fn, ".gz") { if reader, err = gzip.NewReader(reader); err != nil { diff --git a/core/state/database.go b/core/state/database.go index ec60dad81029..c965ed996c66 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -73,11 +74,13 @@ type Database interface { Transitioned() bool - SetCurrentAccountHash(hash common.Hash) + SetCurrentSlotHash(hash common.Hash) - GetCurrentAccountHash() common.Hash + GetCurrentAccountAddress() common.Address - SetCurrentSlotHash(hash common.Hash) + SetCurrentAccountAddress(common.Address) + + GetCurrentAccountHash() common.Hash GetCurrentSlotHash() common.Hash @@ -85,6 +88,10 @@ type Database interface { GetStorageProcessed() bool + GetCurrentPreimageOffset() int64 + + SetCurrentPreimageOffset(int64) + AddRootTranslation(originalRoot, translatedRoot common.Hash) } @@ -246,9 +253,10 @@ type cachingDB struct { addrToPoint *utils.PointCache - baseRoot common.Hash // hash of the read-only base tree - CurrentAccountHash common.Hash // hash of the last translated account - CurrentSlotHash common.Hash // hash of the last translated storage slot + baseRoot common.Hash // hash of the read-only base tree + CurrentAccountAddress common.Address // addresss of the last translated account + CurrentSlotHash common.Hash // hash of the last translated storage slot + CurrentPreimageOffset int64 // next byte to read from the preimage file // Mark whether the storage for an account has been processed. This is useful if the // maximum number of leaves of the conversion is reached before the whole storage is @@ -409,12 +417,28 @@ func (db *cachingDB) GetTreeKeyHeader(addr []byte) *verkle.Point { return db.addrToPoint.GetTreeKeyHeader(addr) } -func (db *cachingDB) SetCurrentAccountHash(hash common.Hash) { - db.CurrentAccountHash = hash +func (db *cachingDB) SetCurrentAccountAddress(addr common.Address) { + db.CurrentAccountAddress = addr } func (db *cachingDB) GetCurrentAccountHash() common.Hash { - return db.CurrentAccountHash + var addrHash common.Hash + if db.CurrentAccountAddress != (common.Address{}) { + addrHash = crypto.Keccak256Hash(db.CurrentAccountAddress[:]) + } + return addrHash +} + +func (db *cachingDB) GetCurrentAccountAddress() common.Address { + return db.CurrentAccountAddress +} + +func (db *cachingDB) GetCurrentPreimageOffset() int64 { + return db.CurrentPreimageOffset +} + +func (db *cachingDB) SetCurrentPreimageOffset(offset int64) { + db.CurrentPreimageOffset = offset } func (db *cachingDB) SetCurrentSlotHash(hash common.Hash) { diff --git a/core/state_processor.go b/core/state_processor.go index 6a01460327f7..2fd6fa1d2f63 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,10 +17,13 @@ package core import ( + "bufio" "bytes" "encoding/binary" "fmt" + "io" "math/big" + "os" "time" "github.com/ethereum/go-ethereum/common" @@ -98,9 +101,22 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg allLogs = append(allLogs, receipt.Logs...) } + // Overlay tree migration logic + migrdb := statedb.Database() + filePreimages, err := os.Open("preimages.bin") + if err != nil { + return nil, nil, 0, fmt.Errorf("opening preimage file: %s", err) + } + defer filePreimages.Close() + preimageSeek := migrdb.GetCurrentPreimageOffset() + if _, err := filePreimages.Seek(migrdb.GetCurrentPreimageOffset(), io.SeekStart); err != nil { + return nil, nil, 0, fmt.Errorf("seeking preimage file: %s", err) + } + fpreimages := bufio.NewReader(filePreimages) + // verkle transition: if the conversion process is in progress, move // N values from the MPT into the verkle tree. - if statedb.Database().InTransition() { + if migrdb.InTransition() { var ( now = time.Now() tt = statedb.GetTrie().(*trie.TransitionTrie) @@ -108,13 +124,27 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vkt = tt.Overlay() ) - accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), statedb.Database().GetCurrentAccountHash()) + accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash()) if err != nil { return nil, nil, 0, err } defer accIt.Release() accIt.Next() + // If we're about to start with the migration process, we have to read the first account hash preimage. + if migrdb.GetCurrentAccountAddress() == (common.Address{}) { + var addr common.Address + if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { + return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) + } + // fmt.Printf("first account: %s != %s\n", crypto.Keccak256Hash(addr[:]), accIt.Hash()) + if crypto.Keccak256Hash(addr[:]) != accIt.Hash() { + return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) + } + preimageSeek += int64(len(addr)) + migrdb.SetCurrentAccountAddress(addr) + } + const maxMovedCount = 10000 // mkv will be assiting in the collection of up to maxMovedCount key values to be migrated to the VKT. // It has internal caches to do efficient MPT->VKT key calculations, which will be discarded after @@ -126,18 +156,12 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // if less than maxCount slots were moved, move to the next account for count < maxMovedCount { - statedb.Database().SetCurrentAccountHash(accIt.Hash()) - acc, err := snapshot.FullAccount(accIt.Account()) if err != nil { log.Error("Invalid account encountered during traversal", "error", err) return nil, nil, 0, err } - addr := rawdb.ReadPreimage(statedb.Database().DiskDB(), accIt.Hash()) - if len(addr) == 0 { - panic(fmt.Sprintf("%x %x %v", addr, accIt.Hash(), acc)) - } - vkt.SetStorageRootConversion(addr, common.BytesToHash(acc.Root)) + vkt.SetStorageRootConversion(migrdb.GetCurrentAccountAddress().Bytes(), common.BytesToHash(acc.Root)) // Start with processing the storage, because once the account is // converted, the `stateRoot` field loses its meaning. Which means @@ -149,7 +173,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // to during normal block execution. A mitigation strategy has been // introduced with the `*StorageRootConversion` fields in VerkleDB. if acc.HasStorage() { - stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), statedb.Database().GetCurrentSlotHash()) + stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash()) if err != nil { return nil, nil, 0, err } @@ -161,7 +185,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // processing the storage for that account where we left off. // If the entire storage was processed, then the iterator was // created in vain, but it's ok as this will not happen often. - for ; !statedb.Database().GetStorageProcessed() && count < maxMovedCount; count++ { + for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ { var ( value []byte // slot value after RLP decoding safeValue [32]byte // 32-byte aligned value @@ -170,14 +194,23 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, fmt.Errorf("error decoding bytes %x: %w", stIt.Slot(), err) } copy(safeValue[32-len(value):], value) - slotnr := rawdb.ReadPreimage(statedb.Database().DiskDB(), stIt.Hash()) - mkv.addStorageSlot(addr, slotnr, safeValue[:]) + var slotnr [32]byte + if _, err := io.ReadFull(fpreimages, slotnr[:]); err != nil { + return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) + } + // fmt.Printf("slot: %s != %s\n", crypto.Keccak256Hash(slotnr[:]), stIt.Hash()) + if crypto.Keccak256Hash(slotnr[:]) != stIt.Hash() { + return nil, nil, 0, fmt.Errorf("preimage file does not match storage hash: %s!=%s", crypto.Keccak256Hash(slotnr[:]), stIt.Hash()) + } + preimageSeek += int64(len(slotnr)) + + mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr[:], safeValue[:]) // advance the storage iterator - statedb.Database().SetStorageProcessed(!stIt.Next()) - if !statedb.Database().GetStorageProcessed() { - statedb.Database().SetCurrentSlotHash(stIt.Hash()) + migrdb.SetStorageProcessed(!stIt.Next()) + if !migrdb.GetStorageProcessed() { + migrdb.SetCurrentSlotHash(stIt.Hash()) } } stIt.Release() @@ -190,33 +223,43 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if count < maxMovedCount { count++ // count increase for the account itself - mkv.addAccount(addr, acc) - vkt.ClearStrorageRootConversion(addr) + mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc) + vkt.ClearStrorageRootConversion(migrdb.GetCurrentAccountAddress().Bytes()) // Store the account code if present if !bytes.Equal(acc.CodeHash, emptyCodeHash[:]) { code := rawdb.ReadCode(statedb.Database().DiskDB(), common.BytesToHash(acc.CodeHash)) chunks := trie.ChunkifyCode(code) - mkv.addAccountCode(addr, uint64(len(code)), chunks) + mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks) } // reset storage iterator marker for next account - statedb.Database().SetStorageProcessed(false) - statedb.Database().SetCurrentSlotHash(common.Hash{}) + migrdb.SetStorageProcessed(false) + migrdb.SetCurrentSlotHash(common.Hash{}) // Move to the next account, if available - or end // the transition otherwise. if accIt.Next() { - statedb.Database().SetCurrentAccountHash(accIt.Hash()) + var addr common.Address + if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { + return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) + } + // fmt.Printf("account switch: %s != %s\n", crypto.Keccak256Hash(addr[:]), accIt.Hash()) + if crypto.Keccak256Hash(addr[:]) != accIt.Hash() { + return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) + } + preimageSeek += int64(len(addr)) + migrdb.SetCurrentAccountAddress(addr) } else { // case when the account iterator has // reached the end but count < maxCount - statedb.Database().EndVerkleTransition() + migrdb.EndVerkleTransition() break } } } + migrdb.SetCurrentPreimageOffset(preimageSeek) log.Info("Collected and prepared key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash()) diff --git a/light/trie.go b/light/trie.go index f5270358f7dd..40b416282055 100644 --- a/light/trie.go +++ b/light/trie.go @@ -117,14 +117,18 @@ func (db *odrDatabase) Transitioned() bool { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) SetCurrentAccountHash(hash common.Hash) { +func (db *odrDatabase) SetCurrentAccountAddress(common.Address) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetCurrentAccountHash() common.Hash { +func (db *odrDatabase) GetCurrentAccountAddress() common.Address { panic("not implemented") // TODO: Implement } +func (*odrDatabase) GetCurrentAccountHash() common.Hash { + panic("unimplemented") +} + func (db *odrDatabase) SetCurrentSlotHash(hash common.Hash) { panic("not implemented") // TODO: Implement } @@ -141,6 +145,14 @@ func (db *odrDatabase) GetStorageProcessed() bool { panic("not implemented") // TODO: Implement } +func (db *odrDatabase) GetCurrentPreimageOffset() int64 { + panic("not implemented") // TODO: Implement +} + +func (db *odrDatabase) SetCurrentPreimageOffset(int64) { + panic("not implemented") // TODO: Implement +} + func (db *odrDatabase) AddRootTranslation(originalRoot common.Hash, translatedRoot common.Hash) { panic("not implemented") // TODO: Implement } From 08b731b181f2bc5d163d0c058843b729ab0f7fd7 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Mon, 31 Jul 2023 15:38:31 -0300 Subject: [PATCH 3/5] cmd/geth: add --roothash to overlay tree preimage exporting command Signed-off-by: Ignacio Hagopian --- cmd/geth/chaincmd.go | 13 +++++++++++-- cmd/utils/cmd.go | 10 ++++++---- cmd/utils/flags.go | 5 +++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index add1f6b763ce..f00e1bbfe243 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -150,7 +150,7 @@ It's deprecated, please use "geth db export" instead. Name: "export-overlay-preimages", Usage: "Export the preimage in overlay tree migration order", ArgsUsage: "", - Flags: flags.Merge([]cli.Flag{}, utils.DatabasePathFlags), + Flags: flags.Merge([]cli.Flag{utils.TreeRootFlag}, utils.DatabasePathFlags), Description: ` The export-overlay-preimages command exports hash preimages to a flat file, in exactly the expected order for the overlay tree migration. @@ -415,8 +415,17 @@ func exportOverlayPreimages(ctx *cli.Context) error { chain, _ := utils.MakeChain(ctx, stack) + var root common.Hash + if ctx.String(utils.TreeRootFlag.Name) != "" { + rootBytes := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) + if len(rootBytes) != common.HashLength { + return fmt.Errorf("invalid root hash length") + } + root = common.BytesToHash(rootBytes) + } + start := time.Now() - if err := utils.ExportOverlayPreimages(chain, ctx.Args().First()); err != nil { + if err := utils.ExportOverlayPreimages(chain, ctx.Args().First(), root); err != nil { utils.Fatalf("Export error: %v\n", err) } fmt.Printf("Export done in %v\n", time.Since(start)) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 6464999ea49b..3f8099e8cd6b 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -386,7 +386,7 @@ func ExportPreimages(db ethdb.Database, fn string) error { // ExportOverlayPreimages exports all known hash preimages into the specified file, // in the same order as expected by the overlay tree migration. -func ExportOverlayPreimages(chain *core.BlockChain, fn string) error { +func ExportOverlayPreimages(chain *core.BlockChain, fn string, root common.Hash) error { log.Info("Exporting preimages", "file", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) @@ -403,9 +403,11 @@ func ExportOverlayPreimages(chain *core.BlockChain, fn string) error { return fmt.Errorf("failed to open statedb: %w", err) } - mptRoot := chain.CurrentBlock().Root() + if root == (common.Hash{}) { + root = chain.CurrentBlock().Root() + } - accIt, err := statedb.Snaps().AccountIterator(mptRoot, common.Hash{}) + accIt, err := statedb.Snaps().AccountIterator(root, common.Hash{}) if err != nil { return err } @@ -426,7 +428,7 @@ func ExportOverlayPreimages(chain *core.BlockChain, fn string) error { } if acc.HasStorage() { - stIt, err := statedb.Snaps().StorageIterator(mptRoot, accIt.Hash(), common.Hash{}) + stIt, err := statedb.Snaps().StorageIterator(root, accIt.Hash(), common.Hash{}) if err != nil { return fmt.Errorf("failed to create storage iterator: %w", err) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ca6ded475668..1d583b69cbbd 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -219,6 +219,11 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } + TreeRootFlag = &cli.StringFlag{ + Name: "roothash", + Usage: "Root hash of the tree (if empty, use the latest)", + Value: "", + } defaultSyncMode = ethconfig.Defaults.SyncMode SyncModeFlag = &flags.TextMarshalerFlag{ From 5dc1a63a397c8db795a0d9c84b97e1ef41c5ca27 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Mon, 31 Jul 2023 15:55:19 -0300 Subject: [PATCH 4/5] cleanup Signed-off-by: Ignacio Hagopian --- cmd/utils/cmd.go | 4 ---- core/state_processor.go | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 3f8099e8cd6b..d85a141783b0 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -170,10 +170,6 @@ func ImportChain(chain *core.BlockChain, fn string) error { } defer fh.Close() - if _, err := fh.Seek(18224628422, 0); err != nil { - panic(err) - } - var reader io.Reader = fh if strings.HasSuffix(fn, ".gz") { if reader, err = gzip.NewReader(reader); err != nil { diff --git a/core/state_processor.go b/core/state_processor.go index 2fd6fa1d2f63..604941029b18 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -103,6 +103,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // Overlay tree migration logic migrdb := statedb.Database() + // TODO: avoid opening the preimages file here and make it part of, potentially, statedb.Database(). filePreimages, err := os.Open("preimages.bin") if err != nil { return nil, nil, 0, fmt.Errorf("opening preimage file: %s", err) @@ -137,7 +138,6 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) } - // fmt.Printf("first account: %s != %s\n", crypto.Keccak256Hash(addr[:]), accIt.Hash()) if crypto.Keccak256Hash(addr[:]) != accIt.Hash() { return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) } @@ -199,7 +199,6 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if _, err := io.ReadFull(fpreimages, slotnr[:]); err != nil { return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) } - // fmt.Printf("slot: %s != %s\n", crypto.Keccak256Hash(slotnr[:]), stIt.Hash()) if crypto.Keccak256Hash(slotnr[:]) != stIt.Hash() { return nil, nil, 0, fmt.Errorf("preimage file does not match storage hash: %s!=%s", crypto.Keccak256Hash(slotnr[:]), stIt.Hash()) } From c99f5355b2aa3eecc728a3680b8969c7441e8fb2 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 1 Aug 2023 08:50:29 -0300 Subject: [PATCH 5/5] review feedback Signed-off-by: Ignacio Hagopian --- core/state/database.go | 16 ++++++++-------- core/state_processor.go | 6 +++--- light/trie.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index c965ed996c66..2147c1aee1ca 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -76,7 +76,7 @@ type Database interface { SetCurrentSlotHash(hash common.Hash) - GetCurrentAccountAddress() common.Address + GetCurrentAccountAddress() *common.Address SetCurrentAccountAddress(common.Address) @@ -253,10 +253,10 @@ type cachingDB struct { addrToPoint *utils.PointCache - baseRoot common.Hash // hash of the read-only base tree - CurrentAccountAddress common.Address // addresss of the last translated account - CurrentSlotHash common.Hash // hash of the last translated storage slot - CurrentPreimageOffset int64 // next byte to read from the preimage file + baseRoot common.Hash // hash of the read-only base tree + CurrentAccountAddress *common.Address // addresss of the last translated account + CurrentSlotHash common.Hash // hash of the last translated storage slot + CurrentPreimageOffset int64 // next byte to read from the preimage file // Mark whether the storage for an account has been processed. This is useful if the // maximum number of leaves of the conversion is reached before the whole storage is @@ -418,18 +418,18 @@ func (db *cachingDB) GetTreeKeyHeader(addr []byte) *verkle.Point { } func (db *cachingDB) SetCurrentAccountAddress(addr common.Address) { - db.CurrentAccountAddress = addr + db.CurrentAccountAddress = &addr } func (db *cachingDB) GetCurrentAccountHash() common.Hash { var addrHash common.Hash - if db.CurrentAccountAddress != (common.Address{}) { + if db.CurrentAccountAddress != nil { addrHash = crypto.Keccak256Hash(db.CurrentAccountAddress[:]) } return addrHash } -func (db *cachingDB) GetCurrentAccountAddress() common.Address { +func (db *cachingDB) GetCurrentAccountAddress() *common.Address { return db.CurrentAccountAddress } diff --git a/core/state_processor.go b/core/state_processor.go index 604941029b18..eaec0630d46b 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -133,16 +133,16 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg accIt.Next() // If we're about to start with the migration process, we have to read the first account hash preimage. - if migrdb.GetCurrentAccountAddress() == (common.Address{}) { + if migrdb.GetCurrentAccountAddress() == nil { var addr common.Address if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) } - if crypto.Keccak256Hash(addr[:]) != accIt.Hash() { + migrdb.SetCurrentAccountAddress(addr) + if migrdb.GetCurrentAccountHash() != accIt.Hash() { return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) } preimageSeek += int64(len(addr)) - migrdb.SetCurrentAccountAddress(addr) } const maxMovedCount = 10000 diff --git a/light/trie.go b/light/trie.go index 40b416282055..e944637008e0 100644 --- a/light/trie.go +++ b/light/trie.go @@ -121,7 +121,7 @@ func (db *odrDatabase) SetCurrentAccountAddress(common.Address) { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) GetCurrentAccountAddress() common.Address { +func (db *odrDatabase) GetCurrentAccountAddress() *common.Address { panic("not implemented") // TODO: Implement }