From af5d375983332c4e5a84b0f88e0913ba42000fef Mon Sep 17 00:00:00 2001 From: draganrakita <dragan0rakita@gmail.com> Date: Mon, 20 Jul 2020 14:47:20 +0200 Subject: [PATCH 1/2] Bump memory-db lib to 0.24.1 and add shrink_to_fit --- Cargo.lock | 7 +++---- ethcore/account-state/Cargo.toml | 2 +- ethcore/light/Cargo.toml | 2 +- ethcore/src/client/client.rs | 13 +++++++------ util/journaldb/Cargo.toml | 2 +- util/journaldb/src/overlayrecentdb.rs | 6 +++++- util/patricia-trie-ethereum/Cargo.toml | 2 +- 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ab5f96a034..b858a21fd88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2852,13 +2852,12 @@ dependencies = [ [[package]] name = "memory-db" -version = "0.21.1" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e20981dbe8b148b10d6151f448f5e38c5eed9f50a44599e4a70edb5c4f5aec" +checksum = "36f36ddb0b2cdc25d38babba472108798e3477f02be5165f038c5e393e50c57a" dependencies = [ - "ahash 0.2.18", "hash-db", - "hashbrown 0.6.3", + "hashbrown 0.8.0", "parity-util-mem", ] diff --git a/ethcore/account-state/Cargo.toml b/ethcore/account-state/Cargo.toml index 9573338af65..4df71e86fc5 100644 --- a/ethcore/account-state/Cargo.toml +++ b/ethcore/account-state/Cargo.toml @@ -19,7 +19,7 @@ keccak-hasher = { path = "../../util/keccak-hasher" } kvdb = "0.7" log = "0.4" lru-cache = "0.1.2" -memory-db = "0.21.1" +memory-db = "0.24.1" parity-bytes = "0.1.0" parity-util-mem = "0.7" parking_lot = "0.10.0" diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index a575d3bcd4d..43bc801e8ce 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -18,7 +18,7 @@ ethcore-blockchain = { path = "../blockchain" } ethereum-types = "0.9.2" executive-state = { path = "../executive-state" } machine = { path = "../machine" } -memory-db = "0.21.1" +memory-db = "0.24.1" trie-db = "0.21.0" patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } ethcore-network = { path = "../../util/network" } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 81340b1790c..5d2a370b663 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -970,7 +970,8 @@ impl Client { // If a snapshot is under way, no pruning happens and memory consumption is allowed to // increase above the memory target until the snapshot has finished. loop { - let needs_pruning = state_db.journal_db().journal_size() >= self.config.history_mem; + let journal_size = state_db.journal_db().journal_size(); + let needs_pruning = journal_size >= self.config.history_mem; if !needs_pruning { break @@ -983,12 +984,12 @@ impl Client { // Note: journal_db().mem_used() can be used for a more accurate memory // consumption measurement but it can be expensive so sticking with the // faster `journal_size()` instead. - trace!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} – Not pruning.", - freeze_at, earliest_era, latest_era, state_db.journal_db().journal_size()); + info!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} – Not pruning.", + freeze_at, earliest_era, latest_era, journal_size); break; } - trace!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}", - earliest_era, latest_era, state_db.journal_db().journal_size()); + info!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}", + earliest_era, latest_era, journal_size); match chain.block_hash(earliest_era) { Some(ancient_hash) => { let mut batch = DBTransaction::new(); @@ -2611,7 +2612,7 @@ impl SnapshotClient for Client { self.snapshotting_at.store(actual_block_nr, Ordering::SeqCst); { scopeguard::defer! {{ - trace!(target: "snapshot", "Re-enabling pruning."); + info!(target: "snapshot", "Re-enabling pruning."); self.snapshotting_at.store(0, Ordering::SeqCst) }}; let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?; diff --git a/util/journaldb/Cargo.toml b/util/journaldb/Cargo.toml index 3a7d345287e..ed7f77f6082 100644 --- a/util/journaldb/Cargo.toml +++ b/util/journaldb/Cargo.toml @@ -14,7 +14,7 @@ parity-util-mem = "0.7" keccak-hasher = { path = "../keccak-hasher" } kvdb = "0.7" log = "0.4" -memory-db = "0.21.1" +memory-db = "0.24.1" parking_lot = "0.10.0" fastmap = { path = "../../util/fastmap" } rlp = "0.4.5" diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 8e4a6d03e26..ad85d3aff60 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -253,7 +253,7 @@ impl JournalDB for OverlayRecentDB { let mut mem = self.transaction_overlay.size_of(&mut ops); let overlay = self.journal_overlay.read(); - mem += overlay.backing_overlay.size_of(&mut ops); + mem += overlay.cumulative_size; //This does not present real HashMap memory footprint but we are doing shrink_to_size after removal so this is good approximation. mem += overlay.pending_overlay.size_of(&mut ops); mem += overlay.journal.size_of(&mut ops); @@ -394,6 +394,10 @@ impl JournalDB for OverlayRecentDB { journal_overlay.earliest_era = Some(end_era + 1); } + //garbage collection on MemoryDB and journal HashMaps; + journal_overlay.backing_overlay.shrink_to_fit(); + journal_overlay.journal.shrink_to_fit(); + Ok(ops as u32) } diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index c1ab8d6fe26..4215a387159 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -15,7 +15,7 @@ ethereum-types = "0.9.2" elastic-array = "0.10" [dev-dependencies] -memory-db = "0.21.1" +memory-db = "0.24.1" keccak-hash = "0.5.0" journaldb = { path = "../journaldb" } criterion = "0.3" From 576c3265c9537c9fef23ddd3d88ab94f4878664d Mon Sep 17 00:00:00 2001 From: draganrakita <dragan0rakita@gmail.com> Date: Tue, 28 Jul 2020 10:29:47 +0200 Subject: [PATCH 2/2] Using improved memory-db size_of --- util/journaldb/src/overlayrecentdb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index ad85d3aff60..4f521ca6d0c 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -253,7 +253,7 @@ impl JournalDB for OverlayRecentDB { let mut mem = self.transaction_overlay.size_of(&mut ops); let overlay = self.journal_overlay.read(); - mem += overlay.cumulative_size; //This does not present real HashMap memory footprint but we are doing shrink_to_size after removal so this is good approximation. + mem += overlay.backing_overlay.size_of(&mut ops); mem += overlay.pending_overlay.size_of(&mut ops); mem += overlay.journal.size_of(&mut ops);