From 86f50855d0d32334c8024acab0b653cde13aed7a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 10 Jul 2023 16:39:53 +0200 Subject: [PATCH] use a PBSS-like scheme for internal nodes (#221) * use a PBSS-like scheme for internal nodes * a couple of fixes coming from debugging replay * fix: use an error to notify the transition tree that a deleted account was found in the overlay tree (#222) * fixes for pbss replay (#227) * fixes for pbss replay * trie/verkle: use capped batch size (#229) * trie/verkle: use capped batch size Signed-off-by: Ignacio Hagopian * trie/verkle: avoid path variable allocation per db.Put Signed-off-by: Ignacio Hagopian * don't keep more than 32 state root conversions in RAM (#230) --------- Signed-off-by: Ignacio Hagopian Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> * cleanup some code * mod: update go-verkle Signed-off-by: Ignacio Hagopian * re-enable snapshot (#231) * re-enable cancun block / snapshot (#226) * clear storage conversion key upon translating account (#234) * clear storage conversion key upon translating account * mod: use latest go-verkle Signed-off-by: Ignacio Hagopian --------- Signed-off-by: Ignacio Hagopian Co-authored-by: Ignacio Hagopian * fix: self-deadlock with translated root map mutex (#236) * return compressed commitment as root commitment (#237) --------- Signed-off-by: Ignacio Hagopian Co-authored-by: Ignacio Hagopian --------- Signed-off-by: Ignacio Hagopian Co-authored-by: Ignacio Hagopian --- cmd/geth/verkle.go | 13 +++--- cmd/utils/cmd.go | 12 ++++++ core/blockchain.go | 6 +-- core/state/access_witness.go | 2 +- core/state/database.go | 47 +++++++++++++--------- core/state/statedb.go | 11 ++++++ core/state_processor.go | 13 ++++-- go.mod | 8 ++-- go.sum | 25 ++++-------- params/protocol_params.go | 10 ++--- trie/database.go | 7 ++++ trie/transition.go | 9 ++++- trie/verkle.go | 77 ++++++++++++++++++++++++++---------- trie/verkle_iterator.go | 2 +- 14 files changed, 160 insertions(+), 82 deletions(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 57a6fd3096ee..f16895955b47 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -130,14 +130,13 @@ func convertToVerkle(ctx *cli.Context) error { vRoot = verkle.New().(*verkle.InternalNode) ) - saveverkle := func(node verkle.VerkleNode) { - comm := node.Commit() + saveverkle := func(path []byte, node verkle.VerkleNode) { + node.Commit() s, err := node.Serialize() if err != nil { panic(err) } - commB := comm.Bytes() - if err := chaindb.Put(commB[:], s); err != nil { + if err := chaindb.Put(path, s); err != nil { panic(err) } } @@ -330,7 +329,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error return fmt.Errorf("could not find child %x in db: %w", childC, err) } // depth is set to 0, the tree isn't rebuilt so it's not a problem - childN, err := verkle.ParseNode(childS, 0, childC[:]) + childN, err := verkle.ParseNode(childS, 0) if err != nil { return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err) } @@ -390,7 +389,7 @@ func verifyVerkle(ctx *cli.Context) error { if err != nil { return err } - root, err := verkle.ParseNode(serializedRoot, 0, rootC[:]) + root, err := verkle.ParseNode(serializedRoot, 0) if err != nil { return err } @@ -439,7 +438,7 @@ func expandVerkle(ctx *cli.Context) error { if err != nil { return err } - root, err := verkle.ParseNode(serializedRoot, 0, rootC[:]) + root, err := verkle.ParseNode(serializedRoot, 0) if err != nil { return err } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 90f009041477..e42120e85ade 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -26,6 +26,7 @@ import ( "os" "os/signal" "runtime" + "runtime/pprof" "strings" "syscall" "time" @@ -173,6 +174,17 @@ func ImportChain(chain *core.BlockChain, fn string) error { return err } } + cpuProfile, err := os.Create("cpu.out") + if err != nil { + return fmt.Errorf("Error creating CPU profile: %v", err) + } + defer cpuProfile.Close() + err = pprof.StartCPUProfile(cpuProfile) + if err != nil { + return fmt.Errorf("Error starting CPU profile: %v", err) + } + defer pprof.StopCPUProfile() + stream := rlp.NewStream(reader, 0) // Run actual the import. diff --git a/core/blockchain.go b/core/blockchain.go index 84c92a528738..b32548b9eb57 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1688,7 +1688,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) } if parent.Number.Uint64() == conversionBlock { - bc.StartVerkleTransition(parent.Root, emptyVerkleRoot) + bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), parent.Number) } statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) if err != nil { @@ -2459,8 +2459,8 @@ func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Pro bc.processor = p } -func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash) { - bc.stateCache.(*state.ForkingDB).StartTransition(originalRoot, translatedRoot) +func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunBlock *big.Int) { + bc.stateCache.(*state.ForkingDB).StartTransition(originalRoot, translatedRoot, chainConfig, cancunBlock) } func (bc *BlockChain) AddRootTranslation(originalRoot, translatedRoot common.Hash) { diff --git a/core/state/access_witness.go b/core/state/access_witness.go index 789bbbc27e3a..05970270dac1 100644 --- a/core/state/access_witness.go +++ b/core/state/access_witness.go @@ -251,7 +251,7 @@ func (aw *AccessWitness) Copy() *AccessWitness { } func (aw *AccessWitness) GetTreeKeyVersionCached(addr []byte) []byte { - return aw.statedb.db.(*VerkleDB).addrToPoint.GetTreeKeyVersionCached(addr) + return aw.statedb.db.(*ForkingDB).addrToPoint.GetTreeKeyVersionCached(addr) } func (aw *AccessWitness) TouchAndChargeProofOfAbsence(addr []byte) uint64 { diff --git a/core/state/database.go b/core/state/database.go index c093863f3fc9..1bdca6fe057b 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -19,6 +19,7 @@ package state import ( "errors" "fmt" + "math/big" "sync" "github.com/VictoriaMetrics/fastcache" @@ -26,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/utils" "github.com/gballet/go-verkle" @@ -164,7 +166,9 @@ type ForkingDB struct { // TODO ensure that this info is in the DB started, ended bool - translatedRoots map[common.Hash]common.Hash // hash of the translated root, for opening + translatedRoots [32]common.Hash // hash of the translated root, for opening + origRoots [32]common.Hash + translationIndex int translatedRootsLock sync.RWMutex baseRoot common.Hash // hash of the read-only base tree @@ -213,9 +217,7 @@ func (fdb *ForkingDB) OpenStorageTrie(stateRoot, addrHash, root common.Hash, sel if fdb.started && err == nil { // Return a "storage trie" that is an adapter between the storge MPT // and the unique verkle tree. - fdb.translatedRootsLock.RLock() - vkt, err := fdb.VerkleDB.OpenStorageTrie(stateRoot, addrHash, fdb.translatedRoots[root], self.(*trie.TransitionTrie).Overlay()) - fdb.translatedRootsLock.RUnlock() + vkt, err := fdb.VerkleDB.OpenStorageTrie(stateRoot, addrHash, fdb.getTranslation(root), self.(*trie.TransitionTrie).Overlay()) if err != nil { return nil, err } @@ -237,9 +239,7 @@ func (fdb *ForkingDB) OpenTrie(root common.Hash) (Trie, error) { if err != nil { return nil, err } - fdb.translatedRootsLock.RLock() - vkt, err := fdb.VerkleDB.OpenTrie(fdb.translatedRoots[root]) - fdb.translatedRootsLock.RUnlock() + vkt, err := fdb.VerkleDB.OpenTrie(fdb.getTranslation(root)) if err != nil { return nil, err } @@ -281,7 +281,7 @@ func (fdg *ForkingDB) Transitionned() bool { } // Fork implements the fork -func (fdb *ForkingDB) StartTransition(originalRoot, translatedRoot common.Hash) { +func (fdb *ForkingDB) StartTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunBlock *big.Int) { fmt.Println(` __________.__ .__ .__ __ .__ .__ ____ \__ ___| |__ ____ ____ | | ____ ______ | |__ _____ _____/ |_ | |__ _____ ______ __ _ _|__| ____ / ___\ ______ @@ -290,10 +290,11 @@ func (fdb *ForkingDB) StartTransition(originalRoot, translatedRoot common.Hash) |____| |___| /\___ \___ |____/\___ | __/|___| (____ |___| |__| |___| (____ /_____/ \/\_/ |__|___| /_____//_____/ |__|`) fdb.started = true - fdb.translatedRoots = map[common.Hash]common.Hash{originalRoot: translatedRoot} + fdb.AddTranslation(originalRoot, translatedRoot) fdb.baseRoot = originalRoot // initialize so that the first storage-less accounts are processed fdb.StorageProcessed = true + chainConfig.CancunBlock = cancunBlock } func (fdb *ForkingDB) EndTransition() { @@ -310,8 +311,21 @@ func (fdb *ForkingDB) EndTransition() { func (fdb *ForkingDB) AddTranslation(orig, trans common.Hash) { // TODO make this persistent fdb.translatedRootsLock.Lock() - fdb.translatedRoots[orig] = trans - fdb.translatedRootsLock.Unlock() + defer fdb.translatedRootsLock.Unlock() + fdb.translatedRoots[fdb.translationIndex] = trans + fdb.origRoots[fdb.translationIndex] = orig + fdb.translationIndex = (fdb.translationIndex + 1) % len(fdb.translatedRoots) +} + +func (fdb *ForkingDB) getTranslation(orig common.Hash) common.Hash { + fdb.translatedRootsLock.RLock() + defer fdb.translatedRootsLock.RUnlock() + for i, o := range fdb.origRoots { + if o == orig { + return fdb.translatedRoots[i] + } + } + return common.Hash{} } type cachingDB struct { @@ -415,16 +429,13 @@ func (db *VerkleDB) GetTreeKeyHeader(addr []byte) *verkle.Point { } // OpenTrie opens the main account trie. -func (db *VerkleDB) OpenTrie(root common.Hash) (Trie, error) { - if root == (common.Hash{}) || root == emptyRoot { - return trie.NewVerkleTrie(verkle.New(), db.db, db.addrToPoint), nil - } - payload, err := db.DiskDB().Get(root[:]) +func (db *VerkleDB) OpenTrie(_ common.Hash) (Trie, error) { + payload, err := db.DiskDB().Get([]byte("flat-")) if err != nil { - return nil, err + return trie.NewVerkleTrie(verkle.New(), db.db, db.addrToPoint), nil } - r, err := verkle.ParseNode(payload, 0, root[:]) + r, err := verkle.ParseNode(payload, 0) if err != nil { panic(err) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 4c8255440821..19bc42de7219 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -175,6 +175,17 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) sdb.snapDestructs = make(map[common.Hash]struct{}) sdb.snapAccounts = make(map[common.Hash][]byte) sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + } else { + if fdb, ok := db.(*ForkingDB); ok { + trans := fdb.getTranslation(root) + if trans != (common.Hash{}) { + if sdb.snap = sdb.snaps.Snapshot(trans); sdb.snap != nil { + sdb.snapDestructs = make(map[common.Hash]struct{}) + sdb.snapAccounts = make(map[common.Hash][]byte) + sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + } + } + } } } return sdb, nil diff --git a/core/state_processor.go b/core/state_processor.go index ff161c65749c..8a9bb0852b30 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -102,9 +102,12 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // N values from the MPT into the verkle tree. if fdb, ok := statedb.Database().(*state.ForkingDB); ok { if fdb.InTransition() { - now := time.Now() - tt := statedb.GetTrie().(*trie.TransitionTrie) - mpt := tt.Base() + var ( + now = time.Now() + tt = statedb.GetTrie().(*trie.TransitionTrie) + mpt = tt.Base() + vkt = tt.Overlay() + ) accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), fdb.CurrentAccountHash) if err != nil { @@ -113,7 +116,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg defer accIt.Release() accIt.Next() - const maxMovedCount = 1000 + const maxMovedCount = 10000 // mkv will be assiting in the collection of up to maxMovedCount key values to be migrated to the VKT. // It has internal caches to do efficient MPT->VKT key calculations, which will be discarded after // this function. @@ -135,6 +138,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if len(addr) == 0 { panic(fmt.Sprintf("%x %x %v", addr, accIt.Hash(), acc)) } + vkt.SetStorageRootConversion(addr, common.BytesToHash(acc.Root)) // Start with processing the storage, because once the account is // converted, the `stateRoot` field loses its meaning. Which means @@ -188,6 +192,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg count++ // count increase for the account itself mkv.addAccount(addr, acc) + vkt.ClearStrorageRootConversion(addr) // Store the account code if present if !bytes.Equal(acc.CodeHash, emptyCodeHash[:]) { diff --git a/go.mod b/go.mod index e3c5b95665b1..cd4f452860a7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.14.0 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f - github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 + github.com/crate-crypto/go-ipa v0.0.0-20230626131944-6a9b06cf26df github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/docker/docker v1.6.2 @@ -23,7 +23,7 @@ require ( github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20230605151430-5b4132d06cc8 + github.com/gballet/go-verkle v0.0.0-20230630071859-628f400139a5 github.com/go-stack/stack v1.8.0 github.com/golang-jwt/jwt/v4 v4.3.0 github.com/golang/protobuf v1.5.2 @@ -60,8 +60,8 @@ require ( github.com/urfave/cli/v2 v2.10.2 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 - golang.org/x/sync v0.2.0 - golang.org/x/sys v0.8.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.9.0 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba diff --git a/go.sum b/go.sum index 11e432054ce5..1a1df8764529 100644 --- a/go.sum +++ b/go.sum @@ -86,10 +86,8 @@ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20230410135559-ce4a96995014 h1:bbyTlFQ12wkFA6aVL+9HrBZwVl85AN0VS/Bwam7o93U= -github.com/crate-crypto/go-ipa v0.0.0-20230410135559-ce4a96995014/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= -github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0= -github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= +github.com/crate-crypto/go-ipa v0.0.0-20230626131944-6a9b06cf26df h1:MV8zGEpmkuAocFqFclIJ3zUz1+6rv0T5QYWD9UDDRVE= +github.com/crate-crypto/go-ipa v0.0.0-20230626131944-6a9b06cf26df/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -137,12 +135,8 @@ github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgx github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-verkle v0.0.0-20230424151626-de802a6b19f8 h1:UHRmPcIjYxqcS070yG9OVbr9aPfC/ToIBwRakFxQC9Y= -github.com/gballet/go-verkle v0.0.0-20230424151626-de802a6b19f8/go.mod h1:P3bwGrLhsUNIsUDlq2yzMPvO1c/15oiB3JS85P+hNfw= -github.com/gballet/go-verkle v0.0.0-20230601173322-16892a00eb0e h1:ckx5f8cHbukN4av6fbkH6Hy7wyOxMfDxGmlh7O2y0dE= -github.com/gballet/go-verkle v0.0.0-20230601173322-16892a00eb0e/go.mod h1:CDncRYVRSDqwakm282WEkjfaAj1hxU/v5RXxk5nXOiI= -github.com/gballet/go-verkle v0.0.0-20230605151430-5b4132d06cc8 h1:Dex9w3DLy2UEoTIwFt+vz9nFM/zFLqNRqnocn+cLW5I= -github.com/gballet/go-verkle v0.0.0-20230605151430-5b4132d06cc8/go.mod h1:CDncRYVRSDqwakm282WEkjfaAj1hxU/v5RXxk5nXOiI= +github.com/gballet/go-verkle v0.0.0-20230630071859-628f400139a5 h1:RIGEmSYptolFDoGItkdbyYLKw/JRtdWSHRRTOc1blCU= +github.com/gballet/go-verkle v0.0.0-20230630071859-628f400139a5/go.mod h1:QGqBhrws74t7/y/74D4nLvWuB0FH0p5rKwKmSxeeO1Q= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -517,10 +511,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -562,10 +555,8 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= diff --git a/params/protocol_params.go b/params/protocol_params.go index 70cd72a0916f..884fdd9f25d5 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -159,11 +159,11 @@ const ( RefundQuotientEIP3529 uint64 = 5 // Verkle tree EIP: costs associated to witness accesses - WitnessBranchReadCost = uint64(1900) - WitnessChunkReadCost = uint64(200) - WitnessBranchWriteCost = uint64(3000) - WitnessChunkWriteCost = uint64(500) - WitnessChunkFillCost = uint64(6200) + WitnessBranchReadCost = uint64(0) + WitnessChunkReadCost = uint64(0) + WitnessBranchWriteCost = uint64(0) + WitnessChunkWriteCost = uint64(0) + WitnessChunkFillCost = uint64(0) ) // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations diff --git a/trie/database.go b/trie/database.go index 72733a2354af..70d9ae6f299e 100644 --- a/trie/database.go +++ b/trie/database.go @@ -87,6 +87,7 @@ type Database struct { childrenSize common.StorageSize // Storage size of the external children tracking preimages *preimageStore // The store for caching preimages + // XXX after the shapella rebase, use common.Address as a key type addrToRoot map[string]common.Hash addrToRootLock sync.RWMutex @@ -121,6 +122,12 @@ func (db *Database) StorageRootConversion(addr []byte) common.Hash { return db.addrToRoot[string(addr)] } +func (db *Database) ClearStorageRootConversion(addr []byte) { + db.addrToRootLock.Lock() + defer db.addrToRootLock.Unlock() + delete(db.addrToRoot, string(addr)) +} + // rawNode is a simple binary blob used to differentiate between collapsed trie // nodes and already encoded RLP binary blobs (while at the same time store them // in the same cache fields). diff --git a/trie/transition.go b/trie/transition.go index a839df18e0bb..a08f8191dda3 100644 --- a/trie/transition.go +++ b/trie/transition.go @@ -88,6 +88,11 @@ func (t *TransitionTrie) TryGet(addr, key []byte) ([]byte, error) { func (t *TransitionTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { data, err := t.overlay.TryGetAccount(key) if err != nil { + // WORKAROUND, see the definition of errDeletedAccount + // for an explainer of why this if is needed. + if err == errDeletedAccount { + return nil, nil + } return nil, err } if data != nil { @@ -110,7 +115,7 @@ func (t *TransitionTrie) TryUpdate(address, key []byte, value []byte) error { // TryUpdateAccount abstract an account write to the trie. func (t *TransitionTrie) TryUpdateAccount(key []byte, account *types.StateAccount) error { - if account.Root != (common.Hash{}) || account.Root != emptyRoot { + if account.Root != (common.Hash{}) && account.Root != emptyRoot { t.overlay.db.SetStorageRootConversion(key, account.Root) } return t.overlay.TryUpdateAccount(key, account) @@ -169,7 +174,7 @@ func (t *TransitionTrie) IsVerkle() bool { func (t *TransitionTrie) TryUpdateStem(key []byte, values [][]byte) error { trie := t.overlay resolver := func(h []byte) ([]byte, error) { - return trie.db.diskdb.Get(h) + return trie.db.diskdb.Get(append([]byte("flat-"), h...)) } switch root := trie.root.(type) { case *verkle.InternalNode: diff --git a/trie/verkle.go b/trie/verkle.go index 85121ba3f197..64c067a2bcc2 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -51,8 +51,8 @@ func NewVerkleTrie(root verkle.VerkleNode, db *Database, pointCache *utils.Point } func (trie *VerkleTrie) InsertMigratedLeaves(leaves []verkle.LeafNode) error { - return trie.root.(*verkle.InternalNode).InsertMigratedLeaves(leaves, func(hash []byte) ([]byte, error) { - return trie.db.diskdb.Get(hash) + return trie.root.(*verkle.InternalNode).InsertMigratedLeaves(leaves, func(path []byte) ([]byte, error) { + return trie.db.diskdb.Get(append([]byte("flat-"), path...)) }) } @@ -68,22 +68,37 @@ func (trie *VerkleTrie) GetKey(key []byte) []byte { // not be modified by the caller. If a node was not found in the database, a // trie.MissingNodeError is returned. func (trie *VerkleTrie) TryGet(addr, key []byte) ([]byte, error) { + resolver := func(path []byte) ([]byte, error) { + return trie.db.diskdb.Get(append([]byte("flat-"), path...)) + } pointEval := trie.pointCache.GetTreeKeyHeader(addr) k := utils.GetTreeKeyStorageSlotWithEvaluatedAddress(pointEval, key) - return trie.root.Get(k, trie.db.diskdb.Get) + return trie.root.Get(k, resolver) } // GetWithHashedKey returns the value, assuming that the key has already // been hashed. func (trie *VerkleTrie) GetWithHashedKey(key []byte) ([]byte, error) { - return trie.root.Get(key, trie.db.diskdb.Get) + resolver := func(path []byte) ([]byte, error) { + return trie.db.diskdb.Get(append([]byte("flat-"), path...)) + } + return trie.root.Get(key, resolver) } +// WORKAROUND: this special error is returned if it has been +// detected that the account was deleted in the verkle tree. +// This is needed in case an account was translated while it +// was in the MPT, and was selfdestructed in verkle mode. +// +// This is only a problem for replays, and this code is not +// needed after SELFDESTRUCT has been removed. +var errDeletedAccount = errors.New("account deleted in VKT") + func (t *VerkleTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { var ( acc *types.StateAccount = &types.StateAccount{} - resolver = func(hash []byte) ([]byte, error) { - return t.db.diskdb.Get(hash) + resolver = func(path []byte) ([]byte, error) { + return t.db.diskdb.Get(append([]byte("flat-"), path...)) } ) versionkey := t.pointCache.GetTreeKeyVersionCached(key) @@ -102,7 +117,7 @@ func (t *VerkleTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { // been recreated after that, then its code keccak will NOT be 0. So return `nil` if // the nonce, and values[10], and code keccak is 0. if acc.Nonce == 0 && len(values) > 10 && len(values[10]) > 0 && bytes.Equal(values[utils.CodeKeccakLeafKey], zero[:]) { - return nil, nil + return nil, errDeletedAccount } var balance [32]byte copy(balance[:], values[utils.BalanceLeafKey]) @@ -140,13 +155,13 @@ func (t *VerkleTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error } } - flusher := func(hash []byte) ([]byte, error) { - return t.db.diskdb.Get(hash) + resolver := func(path []byte) ([]byte, error) { + return t.db.diskdb.Get(append([]byte("flat-"), path...)) } switch root := t.root.(type) { case *verkle.InternalNode: - err = root.InsertStem(stem, values, flusher) + err = root.InsertStem(stem, values, resolver) } if err != nil { return fmt.Errorf("TryUpdateAccount (%x) error: %v", key, err) @@ -157,8 +172,8 @@ func (t *VerkleTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error } func (trie *VerkleTrie) TryUpdateStem(key []byte, values [][]byte) error { - resolver := func(h []byte) ([]byte, error) { - return trie.db.diskdb.Get(h) + resolver := func(path []byte) ([]byte, error) { + return trie.db.diskdb.Get(append([]byte("flat-"), path...)) } switch root := trie.root.(type) { case *verkle.InternalNode: @@ -176,8 +191,8 @@ func (trie *VerkleTrie) TryUpdate(address, key, value []byte) error { k := utils.GetTreeKeyStorageSlotWithEvaluatedAddress(trie.pointCache.GetTreeKeyHeader(address), key) var v [32]byte copy(v[:], value[:]) - return trie.root.Insert(k, v[:], func(h []byte) ([]byte, error) { - return trie.db.diskdb.Get(h) + return trie.root.Insert(k, v[:], func(path []byte) ([]byte, error) { + return trie.db.diskdb.Get(append([]byte("flat-"), path...)) }) } @@ -192,8 +207,8 @@ func (t *VerkleTrie) TryDeleteAccount(key []byte) error { values[i] = zero[:] } - resolver := func(hash []byte) ([]byte, error) { - return t.db.diskdb.Get(hash) + resolver := func(path []byte) ([]byte, error) { + return t.db.diskdb.Get(append([]byte("flat-"), path...)) } switch root := t.root.(type) { @@ -214,8 +229,8 @@ func (trie *VerkleTrie) TryDelete(addr, key []byte) error { pointEval := trie.pointCache.GetTreeKeyHeader(addr) k := utils.GetTreeKeyStorageSlotWithEvaluatedAddress(pointEval, key) var zero [32]byte - return trie.root.Insert(k, zero[:], func(h []byte) ([]byte, error) { - return trie.db.diskdb.Get(h) + return trie.root.Insert(k, zero[:], func(path []byte) ([]byte, error) { + return trie.db.diskdb.Get(append([]byte("flat-"), path...)) }) } @@ -242,13 +257,27 @@ func (trie *VerkleTrie) Commit(_ bool) (common.Hash, *NodeSet, error) { return common.Hash{}, nil, fmt.Errorf("serializing tree nodes: %s", err) } + batch := trie.db.diskdb.NewBatch() + const keyPrefix = "flat-" + path := make([]byte, 0, len(keyPrefix)+32) + path = append(path, []byte(keyPrefix)...) for _, node := range nodes { - if err := trie.db.diskdb.Put(node.CommitmentBytes[:], node.SerializedBytes); err != nil { + path := append(path[:len(keyPrefix)], node.Path...) + + if err := batch.Put(path, node.SerializedBytes); err != nil { return common.Hash{}, nil, fmt.Errorf("put node to disk: %s", err) } + + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + } } + batch.Write() - return nodes[0].CommitmentBytes, nil, nil + // Serialize root commitment form + rootH := root.Hash().BytesLE() + return common.BytesToHash(rootH[:]), nil, nil } // NodeIterator returns an iterator that returns nodes of the trie. Iteration @@ -426,3 +455,11 @@ func ChunkifyCode(code []byte) ChunkedCode { return chunks } + +func (t *VerkleTrie) SetStorageRootConversion(key []byte, root common.Hash) { + t.db.SetStorageRootConversion(key, root) +} + +func (t *VerkleTrie) ClearStrorageRootConversion(addr []byte) { + t.db.ClearStorageRootConversion(addr) +} diff --git a/trie/verkle_iterator.go b/trie/verkle_iterator.go index 9c57e5da1956..fef5b0242b11 100644 --- a/trie/verkle_iterator.go +++ b/trie/verkle_iterator.go @@ -104,7 +104,7 @@ func (it *verkleNodeIterator) Next(descend bool) bool { if err != nil { panic(err) } - it.current, err = verkle.ParseNode(data, byte(len(it.stack)-1), nodeToDBKey(node)) + it.current, err = verkle.ParseNode(data, byte(len(it.stack)-1)) if err != nil { panic(err) }