Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core, trie: intermediate mempool between trie and database #15857

Merged
merged 33 commits into from
Feb 5, 2018
Merged
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
33cca2b
core, trie: intermediate mempool between trie and database
karalabe Jan 11, 2018
0e91a17
core, trie: commit trie mempool on close, fix dao test
karalabe Jan 11, 2018
d2ec96e
core, trie: minor cleanups, test fixes
karalabe Jan 12, 2018
34d766a
core, trie: finalize the new trie node-caching db layer
karalabe Jan 17, 2018
80eb5be
core, trie: thread-safe triedb commit, configurable caches
karalabe Jan 17, 2018
e065755
core, trie: fix some dereferencing issues
karalabe Jan 18, 2018
bb40af2
trie: make databse more Go GC friendly
karalabe Jan 19, 2018
be674fa
cmd, core, eth: support restarting with missing states
karalabe Jan 19, 2018
c03e94c
core: garbage collect side fork tries too
karalabe Jan 22, 2018
b46b291
core: only store pruned sidechains until they beat canonical
karalabe Jan 24, 2018
4315480
core, trie: fix duplicate references, sanity check on shutdown
karalabe Jan 24, 2018
b2d8f14
trie: fix the double reference fix (i.e. roots *can* be double refed)
karalabe Jan 24, 2018
507f20d
core: make trie critical periods verbose, add failsafes
karalabe Jan 25, 2018
f0db662
eth: switch hacky tracer pruning to triedb garbage collector
karalabe Jan 25, 2018
a3cd57e
core, trie: fix all the linters
karalabe Jan 25, 2018
7dc1f18
core: add current cache size to import logs
karalabe Jan 25, 2018
615145e
trie: count preimages in memuse, commit in KB batches
karalabe Jan 26, 2018
cfb510c
core: only count canonical blocks for GC processing time
karalabe Jan 26, 2018
abd9983
eth/dwonloader: moving pivot and capped memory usage
karalabe Jan 30, 2018
a7bae8c
trie: use batch reset instead of recreation
karalabe Jan 30, 2018
a2b7866
core, les, light, trie: correctly use the memdb for tries
karalabe Jan 31, 2018
20369b0
eth/downloader: reduce a log level back to debug
karalabe Jan 31, 2018
a14bfec
light: make the trie mempool play nice with CHTs
karalabe Jan 31, 2018
a4cff48
core: warn the user that we're saving state on shutdown
karalabe Feb 1, 2018
4aeed07
core: report pretty storage sizes for receipt inserts
karalabe Feb 1, 2018
92716dd
eth/downloader: termiante state sync cleanly
karalabe Feb 1, 2018
87a37c3
core: make receipt and headers have the same log format
karalabe Feb 1, 2018
104ccf5
core, eth, les, light: hide triedb behind state database
karalabe Feb 2, 2018
3c30b28
core: remote a debugging warning log
karalabe Feb 2, 2018
5f253f6
eth/downloader: fix statesync fast restart race, hang test
karalabe Feb 2, 2018
30ddc90
cmd: drop trie cache timeout flag (available via config)
karalabe Feb 2, 2018
49bdc8b
core, eth: avoid RLP serialization in downloader queue
karalabe Feb 2, 2018
54b8ded
cmd, core, eth, light, trie: gcmode flag + supress archive logs
karalabe Feb 2, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
core: only count canonical blocks for GC processing time
  • Loading branch information
karalabe committed Feb 5, 2018
commit cfb510c20d58ba23702a51f96ae5f7d9e2eb41ab
24 changes: 13 additions & 11 deletions core/blockchain.go
Original file line number Diff line number Diff line change
@@ -91,6 +91,7 @@ type BlockChain struct {
diskdb ethdb.Database // Low level persistent database to store final content in
triedb *trie.Database // High level ephemeral database to store non-final tries in
triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping

hc *HeaderChain
rmLogsFeed event.Feed
@@ -115,8 +116,6 @@ type BlockChain struct {
blockCache *lru.Cache // Cache for the most recent entire blocks
futureBlocks *lru.Cache // future blocks are blocks added for later processing

procTime time.Duration // Accumulator for measuring total block processing for the trie node dumping

quit chan struct{} // blockchain quit channel
running int32 // running must be called atomically
// procInterrupt must be atomically called
@@ -911,26 +910,26 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
size = bc.triedb.Size()
limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
)
if size > limit || bc.procTime > bc.cacheConfig.TrieTimeLimit {
if size > limit || bc.gcproc > bc.cacheConfig.TrieTimeLimit {
// If we're exceeding limits but haven't reached a large enough memory gap,
// warn the user that the system is becoming unstable.
if chosen < lastWrite+triesInMemory {
switch {
case size >= 2*limit:
log.Error("Trie memory critical, forcing to disk", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
case bc.procTime >= 2*bc.cacheConfig.TrieTimeLimit:
log.Error("Trie timing critical, forcing to disk", "time", bc.procTime, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit:
log.Error("Trie timing critical, forcing to disk", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
case size > limit:
log.Warn("Trie memory at dangerous levels", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
case bc.procTime > bc.cacheConfig.TrieTimeLimit:
log.Warn("Trie timing at dangerous levels", "time", bc.procTime, "limit", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
case bc.gcproc > bc.cacheConfig.TrieTimeLimit:
log.Warn("Trie timing at dangerous levels", "time", bc.gcproc, "limit", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
}
}
// If optimum or critical limits reached, write to disk
if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.procTime >= 2*bc.cacheConfig.TrieTimeLimit {
if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
bc.triedb.Commit(header.Root)
lastWrite = chosen
bc.procTime = 0
bc.gcproc = 0
}
}
// Garbage collect anything below our required write retention
@@ -943,7 +942,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
bc.triedb.Dereference(root.(common.Hash), common.Hash{})
}
if current%10000 == 0 {
log.Warn("Current trie pruning state", "size", bc.triedb.Size(), "elapsed", bc.procTime)
log.Warn("Current trie pruning state", "size", bc.triedb.Size(), "elapsed", bc.gcproc)
}
}
if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
@@ -1142,7 +1141,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
bc.reportBlock(block, receipts, err)
return i, events, coalescedLogs, err
}
bc.procTime += time.Since(bstart)
proctime := time.Since(bstart)

// Write the block to the chain and get the status.
status, err := bc.WriteBlockWithState(block, receipts, state)
@@ -1159,6 +1158,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
events = append(events, ChainEvent{block, block.Hash(), logs})
lastCanon = block

// Only count canonical blocks for GC processing time
bc.gcproc += proctime

case SideStatTy:
log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))