diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e0f29bce77..db00cf19e1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -115,6 +115,7 @@ var ( utils.WhitelistFlag, utils.BloomFilterSizeFlag, utils.TriesInMemoryFlag, + utils.AllowInsecureNoTriesFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index c00334ee9e..8192f091dc 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/json" "errors" "fmt" "os" @@ -29,13 +30,16 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -103,7 +107,7 @@ the trie clean cache with default directory will be deleted. geth offline prune-block for block data in ancientdb. The amount of blocks expected for remaining after prune can be specified via block-amount-reserved in this command, will prune and only remain the specified amount of old block data in ancientdb. -the brief workflow is to backup the the number of this specified amount blocks backward in original ancientdb +the brief workflow is to backup the the number of this specified amount blocks backward in original ancientdb into new ancient_backup, then delete the original ancientdb dir and rename the ancient_backup to original one for replacement, finally assemble the statedb and new ancientDb together. The purpose of doing it is because the block data will be moved into the ancient store when it @@ -129,6 +133,32 @@ geth snapshot verify-state will traverse the whole accounts and storages set based on the specified snapshot and recalculate the root hash of state for verification. In other words, this command does the snapshot to trie conversion. +`, + }, + { + Name: "insecure-prune-all", + Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " + + "who do not need trie storage at all", + ArgsUsage: "", + Action: utils.MigrateFlags(pruneAllState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` +will prune all historical trie state data except genesis block. +All trie nodes will be deleted from the database. + +It expects the genesis file as argument. + +WARNING: It's necessary to delete the trie clean cache after the pruning. +If you specify another directory for the trie clean cache via "--cache.trie.journal" +during the use of Geth, please also specify it here for correct deletion. Otherwise +the trie clean cache with default directory will be deleted. `, }, { @@ -171,7 +201,7 @@ geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced trie node or contract code is missing. This command can be used for state integrity verification. The default checking target is the HEAD state. It's basically identical -to traverse-state, but the check granularity is smaller. +to traverse-state, but the check granularity is smaller. It's also usable without snapshot enabled. `, @@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) { } headHeader := headBlock.Header() //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false) if err != nil { log.Error("snaptree error", "err", err) return nil, err // The relevant snapshot(s) might not exist @@ -363,6 +393,48 @@ func pruneState(ctx *cli.Context) error { return nil } +func pruneAllState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + genesisPath := ctx.Args().First() + if len(genesisPath) == 0 { + utils.Fatalf("Must supply path to genesis JSON file") + } + file, err := os.Open(genesisPath) + if err != nil { + utils.Fatalf("Failed to read genesis file: %v", err) + } + defer file.Close() + + g := new(core.Genesis) + if err := json.NewDecoder(file).Decode(g); err != nil { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + Metrics: metrics.DefaultConfig, + } + + // Load config file. + if err := loadConfig(genesisPath, &cfg); err != nil { + utils.Fatalf("%v", err) + } + g = cfg.Eth.Genesis + } + + chaindb := utils.MakeChainDatabase(ctx, stack, false, false) + pruner, err := pruner.NewAllPruner(chaindb) + if err != nil { + log.Error("Failed to open snapshot tree", "err", err) + return err + } + if err = pruner.PruneAll(g); err != nil { + log.Error("Failed to prune state", "err", err) + return err + } + return nil +} + func verifyState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -373,7 +445,7 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 85d1cef887..066141fe39 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -264,6 +264,10 @@ var ( Usage: "The layer of tries trees that keep in memory", Value: 128, } + AllowInsecureNoTriesFlag = cli.BoolTFlag{ + Name: "allow-insecure-no-tries", + Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`, + } OverrideBerlinFlag = cli.Uint64Flag{ Name: "override.berlin", Usage: "Manually specify Berlin fork-block, overriding the bundled setting", @@ -1666,6 +1670,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(TriesInMemoryFlag.Name) { cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name) } + if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) { + cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name) + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 } diff --git a/core/block_validator.go b/core/block_validator.go index c6a35f1fdf..1c6fa4c077 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -70,12 +70,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } validateFuns := []func() error{ - func() error { - if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { - return ErrKnownBlock - } - return nil - }, func() error { if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) diff --git a/core/blockchain.go b/core/blockchain.go index eeaf1b7e0a..4d5594b7ab 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -141,6 +141,7 @@ type CacheConfig struct { SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk TriesInMemory uint64 // How many tries keeps in memory + NoTries bool // Insecure settings. Do not have any tries in databases if enabled. SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -284,6 +285,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Cache: cacheConfig.TrieCleanLimit, Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, + NoTries: cacheConfig.NoTries, }), triesInMemory: cacheConfig.TriesInMemory, quit: make(chan struct{}), @@ -439,7 +441,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer) recover = true } - bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) + bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries()) } // do options before start any routine for _, option := range options { @@ -1096,6 +1098,15 @@ func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { + if bc.stateCache.NoTries() { + if bc.pipeCommit && bc.snaps != nil { + // If parent snap is pending on verification, treat it as state exist + if s := bc.snaps.Snapshot(hash); s != nil && !s.Verified() { + return true + } + } + return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil + } if bc.pipeCommit && bc.snaps != nil { // If parent snap is pending on verification, treat it as state exist if s := bc.snaps.Snapshot(hash); s != nil && !s.Verified() { @@ -1114,6 +1125,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { if block == nil { return false } + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil + } return bc.HasState(block.Root()) } @@ -2110,6 +2124,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } + if statedb.NoTrie() { + statedb.SetCurrentRoot(block.Root()) + } bc.updateHighestVerifiedHeader(block.Header()) // Enable prefetching to pull in trie node paths while processing transactions diff --git a/core/state/database.go b/core/state/database.go index 487589324c..dd114dc6ad 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -74,6 +74,9 @@ type Database interface { // Purge cache Purge() + + // NoTries returns whether the database has tries storage. + NoTries() bool } // Trie is a Ethereum Merkle Patricia trie. @@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database { func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) cc, _ := lru.New(codeCacheSize) + noTries := config != nil && config.NoTries return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: cc, + noTries: noTries, } } @@ -146,6 +151,7 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab cc, _ := lru.New(codeCacheSize) atc, _ := lru.New(accountTrieCacheSize) stc, _ := lru.New(storageTrieCacheSize) + noTries := config != nil && config.NoTries database := &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), @@ -153,8 +159,11 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab codeCache: cc, accountTrieCache: atc, storageTrieCache: stc, + noTries: noTries, + } + if !noTries { + go database.purgeLoop() } - go database.purgeLoop() return database } @@ -164,6 +173,7 @@ type cachingDB struct { codeCache *lru.Cache accountTrieCache *lru.Cache storageTrieCache *lru.Cache + noTries bool } type triePair struct { @@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.accountTrieCache != nil { if tr, exist := db.accountTrieCache.Get(root); exist { return tr.(Trie).(*trie.SecureTrie).Copy(), nil @@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.storageTrieCache != nil { if tries, exist := db.storageTrieCache.Get(addrHash); exist { triesPairs := tries.([3]*triePair) @@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie } } +func (db *cachingDB) NoTries() bool { + return db.noTries +} + func (db *cachingDB) Purge() { if db.storageTrieCache != nil { db.storageTrieCache.Purge() @@ -263,6 +283,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.SecureTrie: return t.Copy() + case *trie.EmptyTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 5b070f3afa..e68ea48523 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -103,7 +104,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize, trie if headBlock == nil { return nil, errors.New("Failed to load head block") } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false, false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -128,6 +129,105 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize, trie }, nil } +func NewAllPruner(db ethdb.Database) (*Pruner, error) { + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return nil, errors.New("Failed to load head block") + } + return &Pruner{ + db: db, + }, nil +} + +func (p *Pruner) PruneAll(genesis *core.Genesis) error { + deleteCleanTrieCache(p.trieCachePath) + return pruneAll(p.db, genesis) +} + +func pruneAll(maindb ethdb.Database, g *core.Genesis) error { + var ( + count int + size common.StorageSize + pstart = time.Now() + logged = time.Now() + batch = maindb.NewBatch() + iter = maindb.NewIterator(nil, nil) + ) + start := time.Now() + for iter.Next() { + key := iter.Key() + if len(key) == common.HashLength { + count += 1 + size += common.StorageSize(len(key) + len(iter.Value())) + batch.Delete(key) + + var eta time.Duration // Realistically will never remain uninited + if done := binary.BigEndian.Uint64(key[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) + speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + eta = time.Duration(left/speed) * time.Millisecond + } + if time.Since(logged) > 8*time.Second { + log.Info("Pruning state data", "nodes", count, "size", size, + "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta)) + logged = time.Now() + } + // Recreate the iterator after every batch commit in order + // to allow the underlying compactor to delete the entries. + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + + iter.Release() + iter = maindb.NewIterator(nil, key) + } + } + } + if batch.ValueSize() > 0 { + batch.Write() + batch.Reset() + } + iter.Release() + log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + + // Start compactions, will remove the deleted data from the disk immediately. + // Note for small pruning, the compaction is skipped. + if count >= rangeCompactionThreshold { + cstart := time.Now() + for b := 0x00; b <= 0xf0; b += 0x10 { + var ( + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} + ) + if b == 0xf0 { + end = nil + } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := maindb.Compact(start, end); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(maindb), nil) + for addr, account := range g.Alloc { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } + root := statedb.IntermediateRoot(false) + statedb.Commit(nil) + statedb.Database().TrieDB().Commit(root, true, nil) + log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientPath string, BlockAmountReserved uint64) *BlockPruner { return &BlockPruner{ db: db, @@ -137,7 +237,6 @@ func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientP BlockAmountReserved: BlockAmountReserved, } } - func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error { // Delete all stale trie nodes in the disk. With the help of state bloom // the trie nodes(and codes) belong to the active state will be filtered @@ -585,7 +684,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string, tri // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true, false) if err != nil { return err // The relevant snapshot(s) might not exist } @@ -712,7 +811,7 @@ const warningLog = ` WARNING! -The clean trie cache is not found. Please delete it by yourself after the +The clean trie cache is not found. Please delete it by yourself after the pruning. Remember don't start the Geth without deleting the clean trie cache otherwise the entire database may be damaged! diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 587f78a474..aabbfd7e1b 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -126,7 +126,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery, withoutTrie bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { @@ -145,6 +145,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root: baseRoot, } snapshot, generator, err := loadAndParseJournal(diskdb, base) + if err != nil { log.Warn("Failed to load new-format journal", "error", err) return nil, false, err @@ -158,6 +159,11 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // which is below the snapshot. In this case the snapshot can be recovered // by re-executing blocks but right now it's unavailable. if head := snapshot.Root(); head != root { + log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) + + if withoutTrie { + return snapshot, false, nil + } // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. @@ -168,7 +174,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // the disk layer is always higher than chain head. It can // be eventually recovered when the chain head beyonds the // disk layer. - log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) } // Everything loaded correctly, resume any suspended operations if !generator.Done { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 7ad4bcc91b..d28d2e295e 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -194,7 +194,7 @@ type Tree struct { // store, on a background thread. If the memory layers from the journal is not // continuous with disk layer or the journal is missing, all diffs will be discarded // iff it's in "recovery" mode, otherwise rebuild is mandatory. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { +func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery, withoutTrie bool) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ diskdb: diskdb, @@ -207,7 +207,7 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root defer snap.waitBuild() } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery, withoutTrie) if disabled { log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil diff --git a/core/state/statedb.go b/core/state/statedb.go index 76983b2a9c..7b4aa23b50 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -74,14 +74,17 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database + db Database + prefetcher *triePrefetcher + originalRoot common.Hash // The pre-state root, before any changes were made + currentRoot common.Hash // only used when noTrie is true + prefetcherLock sync.Mutex - prefetcher *triePrefetcher - originalRoot common.Hash // The pre-state root, before any changes were made expectedRoot common.Hash // The state root in the block header stateRoot common.Hash // The calculation result of IntermediateRoot trie Trie + noTrie bool hasher crypto.KeccakState diffLayer *types.DiffLayer diffTries map[common.Address]Trie @@ -181,12 +184,12 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, } } - snapVerified := sdb.snap != nil && sdb.snap.Verified() tr, err := db.OpenTrie(root) // return error when 1. failed to open trie and 2. the snap is nil or the snap is not nil and done verification - if err != nil && (sdb.snap == nil || snapVerified) { + if snapVerified := sdb.snap != nil && sdb.snap.Verified(); err != nil && (sdb.snap == nil || snapVerified) { return nil, err } + _, sdb.noTrie = tr.(*trie.EmptyTrie) sdb.trie = tr return sdb, nil } @@ -199,6 +202,9 @@ func (s *StateDB) EnableWriteOnSharedStorage() { // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. func (s *StateDB) StartPrefetcher(namespace string) { + if s.noTrie { + return + } s.prefetcherLock.Lock() defer s.prefetcherLock.Unlock() if s.prefetcher != nil { @@ -213,6 +219,9 @@ func (s *StateDB) StartPrefetcher(namespace string) { // StopPrefetcher terminates a running prefetcher and reports any leftover stats // from the gathered metrics. func (s *StateDB) StopPrefetcher() { + if s.noTrie { + return + } s.prefetcherLock.Lock() defer s.prefetcherLock.Unlock() if s.prefetcher != nil { @@ -259,6 +268,14 @@ func (s *StateDB) setError(err error) { } } +func (s *StateDB) NoTrie() bool { + return s.noTrie +} + +func (s *StateDB) SetCurrentRoot(root common.Hash) { + s.currentRoot = root +} + func (s *StateDB) Error() error { return s.dbErr } @@ -571,6 +588,9 @@ func (s *StateDB) Suicide(addr common.Address) bool { // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on updating the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -592,6 +612,9 @@ func (s *StateDB) updateStateObject(obj *StateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on deleting the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -964,6 +987,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // It is called in between transactions to get the root hash that // goes into transaction receipts. func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { + // light process is not allowed when there is no trie if s.lightProcessed { s.StopPrefetcher() return s.trie.Hash() @@ -1120,16 +1144,18 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { } usedAddrs := make([][]byte, 0, len(s.stateObjectsPending)) - for addr := range s.stateObjectsPending { - if obj := s.stateObjects[addr]; obj.deleted { - s.deleteStateObject(obj) - } else { - s.updateStateObject(obj) + if !s.noTrie { + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; obj.deleted { + s.deleteStateObject(obj) + } else { + s.updateStateObject(obj) + } + usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure + } + if prefetcher != nil { + prefetcher.used(s.originalRoot, usedAddrs) } - usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure - } - if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { @@ -1139,8 +1165,11 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } - root := s.trie.Hash() - return root + if s.noTrie { + return s.currentRoot + } else { + return s.trie.Hash() + } } // Prepare sets the current transaction hash and index and block hash which is @@ -1366,8 +1395,13 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // Write any contract code associated with the state object tasks <- func() { // Write any storage changes in the state object to its storage trie - err := obj.CommitTrie(s.db) - taskResults <- err + if !s.noTrie { + if err := obj.CommitTrie(s.db); err != nil { + taskResults <- err + } + } + taskResults <- nil + } tasksNum++ } @@ -1382,29 +1416,39 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er } close(finishCh) + var start time.Time + if metrics.EnabledExpensive { + start = time.Now() + } + // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. - var account Account - root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { + if !s.noTrie { + var account Account + root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + if err := rlp.DecodeBytes(leaf, &account); err != nil { + return nil + } + if account.Root != emptyRoot { + s.db.TrieDB().Reference(account.Root, parent) + } return nil - } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) - } - return nil - }) - if err != nil { - return err - } - if root != emptyRoot { - s.db.CacheAccount(root, s.trie) - } - for _, postFunc := range postCommitFuncs { - err = postFunc() + }) if err != nil { return err } + if metrics.EnabledExpensive { + s.AccountCommits += time.Since(start) + } + if root != emptyRoot { + s.db.CacheAccount(root, s.trie) + } + for _, postFunc := range postCommitFuncs { + err = postFunc() + if err != nil { + return err + } + } } wg.Wait() return nil diff --git a/core/state_processor.go b/core/state_processor.go index 8e9422a8e8..ff5c8326c7 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -407,6 +407,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // usually do have two tx, one for validator set contract, another for system reward contract. systemTxs := make([]*types.Transaction, 0, 2) + for i, tx := range block.Transactions() { if isPoSA { if isSystemTx, err := posa.IsSystemTransaction(tx, block.Header()); err != nil { @@ -422,11 +423,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return statedb, nil, nil, 0, err } statedb.Prepare(tx.Hash(), block.Hash(), i) + receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv, bloomProcessors) if err != nil { return statedb, nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } - commonTxs = append(commonTxs, tx) receipts = append(receipts, receipt) } diff --git a/eth/backend.go b/eth/backend.go index 3f782ff6a8..d5d76e66e8 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -194,14 +194,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, + NoTries: config.NoTries, SnapshotLimit: config.SnapshotCache, TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, } ) bcOps := make([]core.BlockChainOption, 0) - // TODO diffsync performance is not as expected, disable it when pipecommit is enabled for now - if config.DiffSync && !config.PipeCommit { + if config.DiffSync && !config.NoTries && !config.PipeCommit { bcOps = append(bcOps, core.EnableLightProcessor) } if config.PipeCommit { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 09baad1e1c..4f2c87c52e 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -79,6 +79,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, TriesInMemory: 128, + NoTries: false, SnapshotCache: 102, DiffBlock: uint64(86400), Miner: miner.Config{ @@ -175,6 +176,7 @@ type Config struct { TrieTimeout time.Duration SnapshotCache int TriesInMemory uint64 + NoTries bool Preimages bool // Mining options diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index f192a1aace..ba2996279d 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -3,6 +3,7 @@ package ethconfig import ( + "math/big" "time" "github.com/ethereum/go-ethereum/common" @@ -25,6 +26,10 @@ func (c Config) MarshalTOML() (interface{}, error) { SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool + DirectBroadcast bool + DisableSnapProtocol bool + DiffSync bool + RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` @@ -42,28 +47,30 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseCache int DatabaseFreezer string DatabaseDiff string + PersistDiff bool + DiffBlock uint64 TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` TrieCleanCacheRejournal time.Duration `toml:",omitempty"` TrieDirtyCache int TrieTimeout time.Duration - TriesInMemory uint64 `toml:",omitempty"` SnapshotCache int + TriesInMemory uint64 + NoTries bool Preimages bool - PersistDiff bool - DiffBlock uint64 `toml:",omitempty"` Miner miner.Config - Ethash ethash.Config + Ethash ethash.Config `toml:",omitempty"` TxPool core.TxPoolConfig GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` EWASMInterpreter string EVMInterpreter string - RPCGasCap uint64 `toml:",omitempty"` - RPCTxFeeCap float64 `toml:",omitempty"` + RPCGasCap uint64 + RPCTxFeeCap float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -73,6 +80,10 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs enc.NoPruning = c.NoPruning + enc.DirectBroadcast = c.DirectBroadcast + enc.DisableSnapProtocol = c.DisableSnapProtocol + enc.DiffSync = c.DiffSync + enc.RangeLimit = c.RangeLimit enc.TxLookupLimit = c.TxLookupLimit enc.Whitelist = c.Whitelist enc.LightServ = c.LightServ @@ -90,16 +101,17 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseCache = c.DatabaseCache enc.DatabaseFreezer = c.DatabaseFreezer enc.DatabaseDiff = c.DatabaseDiff + enc.PersistDiff = c.PersistDiff + enc.DiffBlock = c.DiffBlock enc.TrieCleanCache = c.TrieCleanCache enc.TrieCleanCacheJournal = c.TrieCleanCacheJournal enc.TrieCleanCacheRejournal = c.TrieCleanCacheRejournal enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout - enc.TriesInMemory = c.TriesInMemory enc.SnapshotCache = c.SnapshotCache + enc.TriesInMemory = c.TriesInMemory + enc.NoTries = c.NoTries enc.Preimages = c.Preimages - enc.PersistDiff = c.PersistDiff - enc.DiffBlock = c.DiffBlock enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -112,6 +124,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle + enc.OverrideBerlin = c.OverrideBerlin return &enc, nil } @@ -126,6 +139,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool + DirectBroadcast *bool + DisableSnapProtocol *bool + DiffSync *bool + RangeLimit *bool TxLookupLimit *uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` @@ -144,27 +161,29 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseFreezer *string DatabaseDiff *string PersistDiff *bool - DiffBlock *uint64 `toml:",omitempty"` + DiffBlock *uint64 TrieCleanCache *int TrieCleanCacheJournal *string `toml:",omitempty"` TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` TrieDirtyCache *int TrieTimeout *time.Duration - TriesInMemory *uint64 `toml:",omitempty"` SnapshotCache *int + TriesInMemory *uint64 + NoTries *bool Preimages *bool Miner *miner.Config - Ethash *ethash.Config + Ethash *ethash.Config `toml:",omitempty"` TxPool *core.TxPoolConfig GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` EWASMInterpreter *string EVMInterpreter *string - RPCGasCap *uint64 `toml:",omitempty"` - RPCTxFeeCap *float64 `toml:",omitempty"` + RPCGasCap *uint64 + RPCTxFeeCap *float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -191,6 +210,18 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } + if dec.DirectBroadcast != nil { + c.DirectBroadcast = *dec.DirectBroadcast + } + if dec.DisableSnapProtocol != nil { + c.DisableSnapProtocol = *dec.DisableSnapProtocol + } + if dec.DiffSync != nil { + c.DiffSync = *dec.DiffSync + } + if dec.RangeLimit != nil { + c.RangeLimit = *dec.RangeLimit + } if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } @@ -263,11 +294,14 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout } + if dec.SnapshotCache != nil { + c.SnapshotCache = *dec.SnapshotCache + } if dec.TriesInMemory != nil { c.TriesInMemory = *dec.TriesInMemory } - if dec.SnapshotCache != nil { - c.SnapshotCache = *dec.SnapshotCache + if dec.NoTries != nil { + c.NoTries = *dec.NoTries } if dec.Preimages != nil { c.Preimages = *dec.Preimages @@ -308,5 +342,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } + if dec.OverrideBerlin != nil { + c.OverrideBerlin = dec.OverrideBerlin + } return nil } diff --git a/light/trie.go b/light/trie.go index 3896b73c4d..3f942f3607 100644 --- a/light/trie.go +++ b/light/trie.go @@ -49,6 +49,10 @@ type odrDatabase struct { backend OdrBackend } +func (db *odrDatabase) NoTries() bool { + return false +} + func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } @@ -178,6 +182,10 @@ func (t *odrTrie) do(key []byte, fn func() error) error { } } +func (db *odrTrie) NoTries() bool { + return false +} + type nodeIterator struct { trie.NodeIterator t *odrTrie diff --git a/tests/state_test_util.go b/tests/state_test_util.go index a688254a20..1de4a787dd 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -234,7 +234,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false) + snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false, false) } statedb, _ = state.New(root, sdb, snaps) return snaps, statedb diff --git a/trie/database.go b/trie/database.go index 649af6dbf9..85797cc2e4 100644 --- a/trie/database.go +++ b/trie/database.go @@ -282,6 +282,7 @@ type Config struct { Cache int // Memory allowance (MB) to use for caching trie nodes in memory Journal string // Journal of clean cache to survive node restarts Preimages bool // Flag whether the preimage of trie key is recorded + NoTries bool } // NewDatabase creates a new trie database to store ephemeral trie content before diff --git a/trie/dummy_trie.go b/trie/dummy_trie.go new file mode 100644 index 0000000000..99eb79fbd4 --- /dev/null +++ b/trie/dummy_trie.go @@ -0,0 +1,96 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +type EmptyTrie struct{} + +func (t *EmptyTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { + return nil +} + +// NewSecure creates a dummy trie +func NewEmptyTrie() *EmptyTrie { + return &EmptyTrie{} +} + +func (t *EmptyTrie) Get(key []byte) []byte { + return nil +} + +func (t *EmptyTrie) TryGet(key []byte) ([]byte, error) { + return nil, nil +} + +func (t *EmptyTrie) TryGetNode(path []byte) ([]byte, int, error) { + return nil, 0, nil +} +func (t *EmptyTrie) Update(key, value []byte) {} + +func (t *EmptyTrie) TryUpdate(key, value []byte) error { + return nil +} + +// Delete removes any existing value for key from the trie. +func (t *EmptyTrie) Delete(key []byte) { + if err := t.TryDelete(key); err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } +} + +func (t *EmptyTrie) TryDelete(key []byte) error { + + return nil +} + +func (t *EmptyTrie) GetKey(shaKey []byte) []byte { + return nil +} + +func (t *EmptyTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { + + return common.Hash{}, nil +} + +func (t *EmptyTrie) Hash() common.Hash { + return common.Hash{} +} + +// Copy returns a copy of SecureTrie. +func (t *EmptyTrie) Copy() *EmptyTrie { + cpy := *t + return &cpy +} + +func (t *EmptyTrie) ResetCopy() *EmptyTrie { + cpy := *t + return &cpy +} + +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *EmptyTrie) NodeIterator(start []byte) NodeIterator { + return nil +}