feat(sync): MerkleStage

crates/common/rlp/src/encode.rs

@@ -403,9 +403,9 @@ where
     }
 }
 
-pub fn encode_iter<'a, K>(i: impl Iterator<Item = &'a K> + Clone, out: &mut dyn BufMut)
+pub fn encode_iter<K>(i: impl Iterator<Item = K> + Clone, out: &mut dyn BufMut)
 where
-    K: Encodable + 'a,
+    K: Encodable,
 {
     let mut h = Header { list: true, payload_length: 0 };
     for x in i.clone() {

crates/interfaces/src/consensus.rs

@@ -49,6 +49,8 @@ pub enum Error {
     HeaderGasUsedExceedsGasLimit { gas_used: u64, gas_limit: u64 },
     #[error("Block ommer hash ({got:?}) is different then expected: ({expected:?})")]
     BodyOmmersHashDiff { got: H256, expected: H256 },
+    #[error("Block state root ({got:?}) is different then expected: ({expected:?})")]
+    BodyStateRootDiff { got: H256, expected: H256 },
     #[error("Block transaction root ({got:?}) is different then expected: ({expected:?})")]
     BodyTransactionRootDiff { got: H256, expected: H256 },
     #[error("Block receipts root ({got:?}) is different then expected: ({expected:?}).")]

crates/interfaces/src/test_utils/generators.rs

@@ -173,7 +173,7 @@ pub fn random_eoa_account() -> (Address, Account) {
     (addr, Account { nonce, balance, bytecode_hash: None })
 }
 
-/// Docs
+/// Generate random Externaly Owned Accounts
 pub fn random_eoa_account_range(acc_range: &mut std::ops::Range<u64>) -> Vec<(Address, Account)> {
     let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize);
     for _ in acc_range {
@@ -182,6 +182,19 @@ pub fn random_eoa_account_range(acc_range: &mut std::ops::Range<u64>) -> Vec<(Ad
     accounts
 }
 
+/// Generate random Contract Accounts
+pub fn random_contract_account_range(
+    acc_range: &mut std::ops::Range<u64>,
+) -> Vec<(Address, Account)> {
+    let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize);
+    for _ in acc_range {
+        let (address, eoa_account) = random_eoa_account();
+        let account = Account { bytecode_hash: Some(H256::random()), ..eoa_account };
+        accounts.push((address, account))
+    }
+    accounts
+}
+
 #[cfg(test)]
 mod test {
     use std::str::FromStr;

crates/primitives/src/lib.rs

@@ -54,7 +54,7 @@ pub use log::Log;
 pub use net::NodeRecord;
 pub use peer::{PeerId, WithPeerId};
 pub use receipt::Receipt;
-pub use storage::StorageEntry;
+pub use storage::{StorageEntry, StorageTrieEntry};
 pub use transaction::{
     AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature,
     Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559,

crates/primitives/src/proofs.rs

@@ -20,7 +20,7 @@ pub const EMPTY_ROOT: H256 =
 
 /// A [Hasher] that calculates a keccak256 hash of the given data.
 #[derive(Default, Debug, Clone, PartialEq, Eq)]
-pub(crate) struct KeccakHasher;
+pub struct KeccakHasher;
 
 impl Hasher for KeccakHasher {
     type Out = H256;

crates/primitives/src/storage.rs

@@ -31,3 +31,34 @@ impl Compact for StorageEntry {
         (Self { key, value }, out)
     }
 }
+
+/// Account storage trie node.
+#[derive_arbitrary(compact)]
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)]
+pub struct StorageTrieEntry {
+    /// Hashed storage key.
+    pub hash: H256,
+    /// Encoded node.
+    pub node: Vec<u8>,
+}
+
+// NOTE: Removing main_codec and manually encode subkey
+// and compress second part of the value. If we have compression
+// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey
+impl Compact for StorageTrieEntry {
+    fn to_compact(self, buf: &mut impl bytes::BufMut) -> usize {
+        // for now put full bytes and later compress it.
+        buf.put_slice(&self.hash.to_fixed_bytes()[..]);
+        buf.put_slice(&self.node[..]);
+        self.node.len() + 32
+    }
+
+    fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8])
+    where
+        Self: Sized,
+    {
+        let key = H256::from_slice(&buf[..32]);
+        let node = Vec::from(&buf[32..len]);
+        (Self { hash: key, node }, &buf[len..])
+    }
+}

crates/stages/Cargo.toml

@@ -38,6 +38,11 @@ thiserror = "1.0.37"
 aquamarine = "0.1.12"
 itertools = "0.10.5"
 rayon = "1.6.0"
+
+# trie
+cita_trie = "4.0.0"
+hasher = "0.1.4"
+
 [dev-dependencies]
 # reth
 reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] }
@@ -50,6 +55,10 @@ assert_matches = "1.5.0"
 rand = "0.8.5"
 paste = "1.0"
 
+# trie
+reth-staged-sync = { path = "../staged-sync" }
+triehash = "0.8"
+
 [features]
 default = ["serde"]
-serde = ["dep:serde"]
+serde = ["dep:serde"]

crates/stages/src/db.rs

@@ -13,7 +13,7 @@ use reth_db::{
     transaction::{DbTx, DbTxMut},
     Error,
 };
-use reth_primitives::{BlockHash, BlockNumber, TransitionId, TxNumber};
+use reth_primitives::{BlockHash, BlockNumber, Header, TransitionId, TxNumber};
 
 use crate::{DatabaseIntegrityError, StageError};
 
@@ -164,6 +164,15 @@ where
         Ok((prev_body.start_tx_id + prev_body.tx_count, last_transition))
     }
 
+    /// Query the block header by number
+    pub(crate) fn get_header_by_num(&self, block: BlockNumber) -> Result<Header, StageError> {
+        let key = self.get_block_numhash(block)?;
+        let header = self
+            .get::<tables::Headers>(key)?
+            .ok_or(DatabaseIntegrityError::Header { number: block, hash: key.hash() })?;
+        Ok(header)
+    }
+
     /// Unwind table by some number key
     #[inline]
     pub(crate) fn unwind_table_by_num<T>(&self, num: u64) -> Result<(), Error>

crates/stages/src/lib.rs

@@ -53,6 +53,7 @@ mod error;
 mod id;
 mod pipeline;
 mod stage;
+mod trie;
 mod util;
 
 #[cfg(test)]

crates/stages/src/sets.rs

@@ -167,7 +167,7 @@ impl<DB: Database> StageSet<DB> for HashingStages {
             .add_stage(MerkleStage::Unwind)
             .add_stage(AccountHashingStage::default())
             .add_stage(StorageHashingStage::default())
-            .add_stage(MerkleStage::Execution)
+            .add_stage(MerkleStage::Execution { clean_threshold: 1_000 })
     }
 }