[R4R] Resolve conflicts between fastnode and develop branch

cmd/geth/main.go

@@ -115,6 +115,7 @@ var (
 		utils.WhitelistFlag,
 		utils.BloomFilterSizeFlag,
 		utils.TriesInMemoryFlag,
+		utils.AllowInsecureNoTriesFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
 		utils.CacheTrieFlag,

cmd/geth/snapshot.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -29,13 +30,16 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/pruner"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
@@ -103,7 +107,7 @@ the trie clean cache with default directory will be deleted.
 geth offline prune-block for block data in ancientdb.
 The amount of blocks expected for remaining after prune can be specified via block-amount-reserved in this command,
 will prune and only remain the specified amount of old block data in ancientdb.
-the brief workflow is to backup the the number of this specified amount blocks backward in original ancientdb 
+the brief workflow is to backup the the number of this specified amount blocks backward in original ancientdb
 into new ancient_backup, then delete the original ancientdb dir and rename the ancient_backup to original one for replacement,
 finally assemble the statedb and new ancientDb together.
 The purpose of doing it is because the block data will be moved into the ancient store when it
@@ -129,6 +133,32 @@ geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
 snapshot and recalculate the root hash of state for verification.
 In other words, this command does the snapshot to trie conversion.
+`,
+			},
+			{
+				Name: "insecure-prune-all",
+				Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " +
+					"who do not need trie storage at all",
+				ArgsUsage: "<genesisPath>",
+				Action:    utils.MigrateFlags(pruneAllState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.AncientFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
+will prune all historical trie state data except genesis block.
+All trie nodes will be deleted from the database.
+
+It expects the genesis file as argument.
+
+WARNING: It's necessary to delete the trie clean cache after the pruning.
+If you specify another directory for the trie clean cache via "--cache.trie.journal"
+during the use of Geth, please also specify it here for correct deletion. Otherwise
+the trie clean cache with default directory will be deleted.
 `,
 			},
 			{
@@ -171,7 +201,7 @@ geth snapshot traverse-rawstate <state-root>
 will traverse the whole state from the given root and will abort if any referenced
 trie node or contract code is missing. This command can be used for state integrity
 verification. The default checking target is the HEAD state. It's basically identical
-to traverse-state, but the check granularity is smaller. 
+to traverse-state, but the check granularity is smaller.
 
 It's also usable without snapshot enabled.
 `,
@@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) {
 	}
 	headHeader := headBlock.Header()
 	//Make sure the MPT and snapshot matches before pruning, otherwise the node can not start.
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("snaptree error", "err", err)
 		return nil, err // The relevant snapshot(s) might not exist
@@ -363,6 +393,48 @@ func pruneState(ctx *cli.Context) error {
 	return nil
 }
 
+func pruneAllState(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	genesisPath := ctx.Args().First()
+	if len(genesisPath) == 0 {
+		utils.Fatalf("Must supply path to genesis JSON file")
+	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
+
+	g := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(g); err != nil {
+		cfg := gethConfig{
+			Eth:     ethconfig.Defaults,
+			Node:    defaultNodeConfig(),
+			Metrics: metrics.DefaultConfig,
+		}
+
+		// Load config file.
+		if err := loadConfig(genesisPath, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+		g = cfg.Eth.Genesis
+	}
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, false, false)
+	pruner, err := pruner.NewAllPruner(chaindb)
+	if err != nil {
+		log.Error("Failed to open snapshot tree", "err", err)
+		return err
+	}
+	if err = pruner.PruneAll(g); err != nil {
+		log.Error("Failed to prune state", "err", err)
+		return err
+	}
+	return nil
+}
+
 func verifyState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
@@ -373,7 +445,7 @@ func verifyState(ctx *cli.Context) error {
 		log.Error("Failed to load head block")
 		return errors.New("no head block")
 	}
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)
 		return err

cmd/utils/flags.go

@@ -264,6 +264,10 @@ var (
 		Usage: "The layer of tries trees that keep in memory",
 		Value: 128,
 	}
+	AllowInsecureNoTriesFlag = cli.BoolTFlag{
+		Name:  "allow-insecure-no-tries",
+		Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`,
+	}
 	OverrideBerlinFlag = cli.Uint64Flag{
 		Name:  "override.berlin",
 		Usage: "Manually specify Berlin fork-block, overriding the bundled setting",
@@ -1666,6 +1670,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(TriesInMemoryFlag.Name) {
 		cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name)
 	}
+	if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) {
+		cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name)
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) {
 		cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
 	}

core/block_validator.go

@@ -70,12 +70,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	}
 
 	validateFuns := []func() error{
-		func() error {
-			if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
-				return ErrKnownBlock
-			}
-			return nil
-		},
 		func() error {
 			if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
 				return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)

core/blockchain.go

@@ -141,6 +141,7 @@ type CacheConfig struct {
 	SnapshotLimit      int           // Memory allowance (MB) to use for caching snapshot entries in memory
 	Preimages          bool          // Whether to store preimage of trie key to the disk
 	TriesInMemory      uint64        // How many tries keeps in memory
+	NoTries            bool          // Insecure settings. Do not have any tries in databases if enabled.
 
 	SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
 }
@@ -284,6 +285,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 			Cache:     cacheConfig.TrieCleanLimit,
 			Journal:   cacheConfig.TrieCleanJournal,
 			Preimages: cacheConfig.Preimages,
+			NoTries:   cacheConfig.NoTries,
 		}),
 		triesInMemory:         cacheConfig.TriesInMemory,
 		quit:                  make(chan struct{}),
@@ -439,7 +441,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 			log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer)
 			recover = true
 		}
-		bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover)
+		bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries())
 	}
 	// do options before start any routine
 	for _, option := range options {
@@ -1096,6 +1098,15 @@ func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
 
 // HasState checks if state trie is fully present in the database or not.
 func (bc *BlockChain) HasState(hash common.Hash) bool {
+	if bc.stateCache.NoTries() {
+		if bc.pipeCommit && bc.snaps != nil {
+			// If parent snap is pending on verification, treat it as state exist
+			if s := bc.snaps.Snapshot(hash); s != nil && !s.Verified() {
+				return true
+			}
+		}
+		return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil
+	}
 	if bc.pipeCommit && bc.snaps != nil {
 		// If parent snap is pending on verification, treat it as state exist
 		if s := bc.snaps.Snapshot(hash); s != nil && !s.Verified() {
@@ -1114,6 +1125,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
 	if block == nil {
 		return false
 	}
+	if bc.stateCache.NoTries() {
+		return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil
+	}
 	return bc.HasState(block.Root())
 }
 
@@ -2110,6 +2124,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
 		if err != nil {
 			return it.index, err
 		}
+		if statedb.NoTrie() {
+			statedb.SetCurrentRoot(block.Root())
+		}
 		bc.updateHighestVerifiedHeader(block.Header())
 
 		// Enable prefetching to pull in trie node paths while processing transactions

core/state/database.go

@@ -74,6 +74,9 @@ type Database interface {
 
 	// Purge cache
 	Purge()
+
+	// NoTries returns whether the database has tries storage.
+	NoTries() bool
 }
 
 // Trie is a Ethereum Merkle Patricia trie.
@@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database {
 func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	cc, _ := lru.New(codeCacheSize)
+	noTries := config != nil && config.NoTries
 	return &cachingDB{
 		db:            trie.NewDatabaseWithConfig(db, config),
 		codeSizeCache: csc,
 		codeCache:     cc,
+		noTries:       noTries,
 	}
 }
 
@@ -146,15 +151,19 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab
 	cc, _ := lru.New(codeCacheSize)
 	atc, _ := lru.New(accountTrieCacheSize)
 	stc, _ := lru.New(storageTrieCacheSize)
+	noTries := config != nil && config.NoTries
 
 	database := &cachingDB{
 		db:               trie.NewDatabaseWithConfig(db, config),
 		codeSizeCache:    csc,
 		codeCache:        cc,
 		accountTrieCache: atc,
 		storageTrieCache: stc,
+		noTries:          noTries,
+	}
+	if !noTries {
+		go database.purgeLoop()
 	}
-	go database.purgeLoop()
 	return database
 }
 
@@ -164,6 +173,7 @@ type cachingDB struct {
 	codeCache        *lru.Cache
 	accountTrieCache *lru.Cache
 	storageTrieCache *lru.Cache
+	noTries          bool
 }
 
 type triePair struct {
@@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() {
 
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
+	if db.noTries {
+		return trie.NewEmptyTrie(), nil
+	}
 	if db.accountTrieCache != nil {
 		if tr, exist := db.accountTrieCache.Get(root); exist {
 			return tr.(Trie).(*trie.SecureTrie).Copy(), nil
@@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
+	if db.noTries {
+		return trie.NewEmptyTrie(), nil
+	}
 	if db.storageTrieCache != nil {
 		if tries, exist := db.storageTrieCache.Get(addrHash); exist {
 			triesPairs := tries.([3]*triePair)
@@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie
 	}
 }
 
+func (db *cachingDB) NoTries() bool {
+	return db.noTries
+}
+
 func (db *cachingDB) Purge() {
 	if db.storageTrieCache != nil {
 		db.storageTrieCache.Purge()
@@ -263,6 +283,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 	switch t := t.(type) {
 	case *trie.SecureTrie:
 		return t.Copy()
+	case *trie.EmptyTrie:
+		return t.Copy()
 	default:
 		panic(fmt.Errorf("unknown trie type %T", t))
 	}