diff --git a/Cargo.lock b/Cargo.lock index 211f8d8e7a80..439f4c88a490 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", "once_cell", @@ -116,9 +116,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0224938f92e7aef515fac2ff2d18bd1115c1394ddf4a092e0c87e8be9499ee5" +checksum = "3e90af4de65aa7b293ef2d09daff88501eb254f58edde2e1ac02c82d873eadad" dependencies = [ "derive_arbitrary", ] @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2 1.0.50", "quote 1.0.23", @@ -308,6 +308,15 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" +[[package]] +name = "basic-toml" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e819b667739967cd44d308b8c7b71305d8bb0729ac44a248aa08f33d01950b4" +dependencies = [ + "serde", +] + [[package]] name = "bech32" version = "0.7.3" @@ -547,9 +556,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cexpr" @@ -625,6 +634,17 @@ dependencies = [ "inout", ] +[[package]] +name = "cita_trie" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afe7baab47f510f52ca8dc9c0eb9082020c627c7f22285bea30edc3511f7ee29" +dependencies = [ + "hasher", + "parking_lot 0.12.1", + "rlp", +] + [[package]] name = "clang-sys" version = "1.4.0" @@ -650,9 +670,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.1" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec7a4128863c188deefe750ac1d1dfe66c236909f845af04beed823638dc1b2" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" dependencies = [ "bitflags", "clap_derive", @@ -850,9 +870,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.0.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" dependencies = [ "crc-catalog", ] @@ -1034,9 +1054,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "322296e2f2e5af4270b54df9e85a02ff037e271af20ba3e7fe1575515dc840b8" dependencies = [ "cc", "cxxbridge-flags", @@ -1046,9 +1066,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "017a1385b05d631e7875b1f151c9f012d37b53491e2a87f65bff5c262b2111d8" dependencies = [ "cc", "codespan-reporting", @@ -1061,15 +1081,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "c26bbb078acf09bc1ecda02d4223f03bdd28bd4874edcb0379138efc499ce971" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e" dependencies = [ "proc-macro2 1.0.50", "quote 1.0.23", @@ -1187,9 +1207,9 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf460bbff5f571bfc762da5102729f59f338be7db17a21fade44c5c4f5005350" +checksum = "8beee4701e2e229e8098bbdecdca12449bc3e322f137d269182fa1291e20bd00" dependencies = [ "proc-macro2 1.0.50", "quote 1.0.23", @@ -1406,9 +1426,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "elliptic-curve" @@ -1628,7 +1648,7 @@ dependencies = [ [[package]] name = "ethers-contract" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" +source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1" dependencies = [ "ethers-core", "ethers-providers", @@ -1644,7 +1664,7 @@ dependencies = [ [[package]] name = "ethers-core" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" +source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1" dependencies = [ "arrayvec", "bytes", @@ -1675,7 +1695,7 @@ dependencies = [ [[package]] name = "ethers-etherscan" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" +source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1" dependencies = [ "ethers-core", "getrandom 0.2.8", @@ -1691,7 +1711,7 @@ dependencies = [ [[package]] name = "ethers-middleware" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" +source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1" dependencies = [ "async-trait", "auto_impl 0.5.0", @@ -1716,7 +1736,7 @@ dependencies = [ [[package]] name = "ethers-providers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" +source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1" dependencies = [ "async-trait", "auto_impl 1.0.1", @@ -1753,7 +1773,7 @@ dependencies = [ [[package]] name = "ethers-signers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" +source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1" dependencies = [ "async-trait", "coins-bip32", @@ -1858,9 +1878,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -1873,9 +1893,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -1883,15 +1903,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -1900,9 +1920,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-lite" @@ -1931,9 +1951,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2 1.0.50", "quote 1.0.23", @@ -1942,15 +1962,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-timer" @@ -1964,9 +1984,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", @@ -2063,9 +2083,9 @@ dependencies = [ [[package]] name = "gloo-net" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9050ff8617e950288d7bf7f300707639fdeda5ca0d0ecf380cff448cfd52f4a6" +checksum = "9902a044653b26b99f7e3693a42f171312d9be8b26b5697bd1e43ad1f8a35e10" dependencies = [ "futures-channel", "futures-core", @@ -2083,9 +2103,9 @@ dependencies = [ [[package]] name = "gloo-timers" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c4a8d6391675c6b2ee1a6c8d06e8e2d03605c44cec1270675985a4c2a5500b" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" dependencies = [ "futures-channel", "futures-core", @@ -2181,10 +2201,19 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.2", + "ahash 0.8.3", "serde", ] +[[package]] +name = "hasher" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbba678b6567f27ce22870d951f4208e1dc2fef64993bd4521b1d497ef8a3aa" +dependencies = [ + "tiny-keccak", +] + [[package]] name = "hashers" version = "1.0.1" @@ -2954,9 +2983,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" @@ -3105,6 +3134,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3142,9 +3180,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" dependencies = [ "num-traits", ] @@ -3310,9 +3348,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366e44391a8af4cfd6002ef6ba072bae071a96aafca98d7d448a34c5dca38b6a" +checksum = "c3840933452adf7b3b9145e27086a5a3376c619dca1a21b1e5a5af0d54979bed" dependencies = [ "arrayvec", "bitvec 1.0.1", @@ -3325,9 +3363,9 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9299338969a3d2f491d65f140b00ddec470858402f888af98e8642fb5e8965cd" +checksum = "86b26a931f824dd4eca30b3e43bb4f31cd5f0d3a403c5f5ff27106b805bfde7b" dependencies = [ "proc-macro-crate", "proc-macro2 1.0.50", @@ -3446,9 +3484,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a" +checksum = "4ab62d2fa33726dbe6321cc97ef96d8cde531e3eeaf858a058de53a8a6d40d8f" dependencies = [ "thiserror", "ucd-trie", @@ -3594,13 +3632,12 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34" dependencies = [ "once_cell", - "thiserror", - "toml", + "toml_edit", ] [[package]] @@ -3866,9 +3903,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -3985,7 +4022,7 @@ name = "reth" version = "0.1.0" dependencies = [ "backon", - "clap 4.1.1", + "clap 4.1.4", "comfy-table", "confy", "crossterm", @@ -4650,12 +4687,16 @@ name = "reth-stages" version = "0.1.0" dependencies = [ "aquamarine", + "arbitrary", "assert_matches", "async-trait", + "cita_trie", "futures-util", + "hasher", "itertools 0.10.5", "metrics", "paste", + "proptest", "rand 0.8.5", "rayon", "reth-db", @@ -4666,12 +4707,14 @@ dependencies = [ "reth-primitives", "reth-provider", "reth-rlp", + "reth-staged-sync", "serde", "tempfile", "thiserror", "tokio", "tokio-stream", "tracing", + "triehash", ] [[package]] @@ -5078,9 +5121,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ "bitflags", "core-foundation", @@ -5091,9 +5134,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -5134,9 +5177,9 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "send_wrapper" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930c0acf610d3fdb5e2ab6213019aaa04e227ebe9547b0649ba599b16d788bd7" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" @@ -5778,9 +5821,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.24.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", "bytes", @@ -5887,13 +5930,30 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" + +[[package]] +name = "toml_edit" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +dependencies = [ + "indexmap", + "nom8", + "toml_datetime", +] + [[package]] name = "toolchain_find" version = "0.2.0" @@ -6158,17 +6218,17 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "trybuild" -version = "1.0.76" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed2c57956f91546d4d33614265a85d55c8e1ab91484853a10335894786d7db6" +checksum = "a44da5a6f2164c8e14d3bbc0657d69c5966af9f5f6930d4f600b1f5c4a673413" dependencies = [ + "basic-toml", "glob", "once_cell", "serde", "serde_derive", "serde_json", "termcolor", - "toml", ] [[package]] @@ -6229,9 +6289,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" @@ -6613,16 +6673,17 @@ dependencies = [ [[package]] name = "ws_stream_wasm" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47ca1ab42f5afed7fc332b22b6e932ca5414b209465412c8cdf0ad23bc0de645" +checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5" dependencies = [ "async_io_stream", "futures", "js-sys", + "log", "pharos", "rustc_version", - "send_wrapper 0.5.0", + "send_wrapper 0.6.0", "thiserror", "wasm-bindgen", "wasm-bindgen-futures", diff --git a/crates/interfaces/src/consensus.rs b/crates/interfaces/src/consensus.rs index b7b9b37433ec..df249aff12a1 100644 --- a/crates/interfaces/src/consensus.rs +++ b/crates/interfaces/src/consensus.rs @@ -43,6 +43,8 @@ pub enum Error { HeaderGasUsedExceedsGasLimit { gas_used: u64, gas_limit: u64 }, #[error("Block ommer hash ({got:?}) is different then expected: ({expected:?})")] BodyOmmersHashDiff { got: H256, expected: H256 }, + #[error("Block state root ({got:?}) is different then expected: ({expected:?})")] + BodyStateRootDiff { got: H256, expected: H256 }, #[error("Block transaction root ({got:?}) is different then expected: ({expected:?})")] BodyTransactionRootDiff { got: H256, expected: H256 }, #[error("Block receipts root ({got:?}) is different then expected: ({expected:?}).")] diff --git a/crates/interfaces/src/test_utils/generators.rs b/crates/interfaces/src/test_utils/generators.rs index dbcc1b492c18..0fc42312a192 100644 --- a/crates/interfaces/src/test_utils/generators.rs +++ b/crates/interfaces/src/test_utils/generators.rs @@ -173,7 +173,7 @@ pub fn random_eoa_account() -> (Address, Account) { (addr, Account { nonce, balance, bytecode_hash: None }) } -/// Docs +/// Generate random Externaly Owned Accounts pub fn random_eoa_account_range(acc_range: &mut std::ops::Range) -> Vec<(Address, Account)> { let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize); for _ in acc_range { @@ -182,6 +182,19 @@ pub fn random_eoa_account_range(acc_range: &mut std::ops::Range) -> Vec<(Ad accounts } +/// Generate random Contract Accounts +pub fn random_contract_account_range( + acc_range: &mut std::ops::Range, +) -> Vec<(Address, Account)> { + let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize); + for _ in acc_range { + let (address, eoa_account) = random_eoa_account(); + let account = Account { bytecode_hash: Some(H256::random()), ..eoa_account }; + accounts.push((address, account)) + } + accounts +} + #[cfg(test)] mod test { use std::str::FromStr; diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 5a5f2d83df64..9b7490313fe6 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -54,7 +54,7 @@ pub use log::Log; pub use net::NodeRecord; pub use peer::{PeerId, WithPeerId}; pub use receipt::Receipt; -pub use storage::StorageEntry; +pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature, Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559, diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index 969c54d0851a..763f56790f13 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -20,7 +20,7 @@ pub const EMPTY_ROOT: H256 = /// A [Hasher] that calculates a keccak256 hash of the given data. #[derive(Default, Debug, Clone, PartialEq, Eq)] -pub(crate) struct KeccakHasher; +pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; diff --git a/crates/primitives/src/storage.rs b/crates/primitives/src/storage.rs index 254499fcece9..f64554c88c47 100644 --- a/crates/primitives/src/storage.rs +++ b/crates/primitives/src/storage.rs @@ -31,3 +31,34 @@ impl Compact for StorageEntry { (Self { key, value }, out) } } + +/// Account storage trie node. +#[derive_arbitrary(compact)] +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +pub struct StorageTrieEntry { + /// Hashed storage key. + pub hash: H256, + /// Encoded node. + pub node: Vec, +} + +// NOTE: Removing main_codec and manually encode subkey +// and compress second part of the value. If we have compression +// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey +impl Compact for StorageTrieEntry { + fn to_compact(self, buf: &mut impl bytes::BufMut) -> usize { + // for now put full bytes and later compress it. + buf.put_slice(&self.hash.to_fixed_bytes()[..]); + buf.put_slice(&self.node[..]); + self.node.len() + 32 + } + + fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) + where + Self: Sized, + { + let key = H256::from_slice(&buf[..32]); + let node = Vec::from(&buf[32..len]); + (Self { hash: key, node }, &buf[len..]) + } +} diff --git a/crates/rlp/src/encode.rs b/crates/rlp/src/encode.rs index 5f09ce8fa895..270f6715749c 100644 --- a/crates/rlp/src/encode.rs +++ b/crates/rlp/src/encode.rs @@ -313,6 +313,7 @@ mod ethereum_types_support { fixed_revm_uint_impl!(RU128, 16); fixed_revm_uint_impl!(RU256, 32); + impl_max_encoded_len!(RU256, { length_of_length(32) + 32 }); } macro_rules! slice_impl { @@ -403,9 +404,9 @@ where } } -pub fn encode_iter<'a, K>(i: impl Iterator + Clone, out: &mut dyn BufMut) +pub fn encode_iter(i: impl Iterator + Clone, out: &mut dyn BufMut) where - K: Encodable + 'a, + K: Encodable, { let mut h = Header { list: true, payload_length: 0 }; for x in i.clone() { diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 608c3d988519..2b44416e3f52 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -38,6 +38,15 @@ thiserror = "1.0.37" aquamarine = "0.2.1" itertools = "0.10.5" rayon = "1.6.0" + +# trie +cita_trie = "4.0.0" +hasher = "0.1.4" + +# arbitrary utils +arbitrary = { version = "1.1.7", features = ["derive"], optional = true } +proptest = { version = "1.0", optional = true } + [dev-dependencies] # reth reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] } @@ -50,6 +59,14 @@ assert_matches = "1.5.0" rand = "0.8.5" paste = "1.0" +# arbitrary utils +arbitrary = { version = "1.1.7", features = ["derive"] } +proptest = { version = "1.0" } + +# trie +reth-staged-sync = { path = "../staged-sync" } +triehash = "0.8" + [features] default = ["serde"] serde = ["dep:serde"] diff --git a/crates/stages/src/db.rs b/crates/stages/src/db.rs index 0c248e0b0b8d..03fb81bc621a 100644 --- a/crates/stages/src/db.rs +++ b/crates/stages/src/db.rs @@ -13,7 +13,7 @@ use reth_db::{ transaction::{DbTx, DbTxMut}, Error, }; -use reth_primitives::{BlockHash, BlockNumber, TransitionId, TxNumber}; +use reth_primitives::{BlockHash, BlockNumber, Header, TransitionId, TxNumber}; use crate::{DatabaseIntegrityError, StageError}; @@ -164,6 +164,15 @@ where Ok((prev_body.start_tx_id + prev_body.tx_count, last_transition)) } + /// Query the block header by number + pub(crate) fn get_header_by_num(&self, block: BlockNumber) -> Result { + let key = self.get_block_numhash(block)?; + let header = self + .get::(key)? + .ok_or(DatabaseIntegrityError::Header { number: block, hash: key.hash() })?; + Ok(header) + } + /// Unwind table by some number key #[inline] pub(crate) fn unwind_table_by_num(&self, num: u64) -> Result<(), Error> diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index cf0af6e7d6f0..29203e1bf201 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -54,6 +54,7 @@ mod error; mod id; mod pipeline; mod stage; +mod trie; mod util; #[cfg(test)] diff --git a/crates/stages/src/sets.rs b/crates/stages/src/sets.rs index 4d1f4b786c7c..ea4d7d55bb53 100644 --- a/crates/stages/src/sets.rs +++ b/crates/stages/src/sets.rs @@ -164,10 +164,10 @@ pub struct HashingStages; impl StageSet for HashingStages { fn builder(self) -> StageSetBuilder { StageSetBuilder::default() - .add_stage(MerkleStage::Unwind) + .add_stage(MerkleStage::default_unwind()) .add_stage(AccountHashingStage::default()) .add_stage(StorageHashingStage::default()) - .add_stage(MerkleStage::Execution) + .add_stage(MerkleStage::default_execution()) } } diff --git a/crates/stages/src/stages/hashing_account.rs b/crates/stages/src/stages/hashing_account.rs index 5fb5610b3027..2c0ee22a0482 100644 --- a/crates/stages/src/stages/hashing_account.rs +++ b/crates/stages/src/stages/hashing_account.rs @@ -56,8 +56,9 @@ impl Stage for AccountHashingStage { // if there are more blocks then threshold it is faster to go over Plain state and hash all // account otherwise take changesets aggregate the sets and apply hashing to - // AccountHashing table - if to_transition - from_transition > self.clean_threshold { + // AccountHashing table. Also, if we start from genesis, we need to hash from scratch, as + // genesis accounts are not in changeset. + if to_transition - from_transition > self.clean_threshold || stage_progress == 0 { // clear table, load all accounts and hash it tx.clear::()?; tx.commit()?; diff --git a/crates/stages/src/stages/hashing_storage.rs b/crates/stages/src/stages/hashing_storage.rs index 023947ce3bf5..12feb87cbba3 100644 --- a/crates/stages/src/stages/hashing_storage.rs +++ b/crates/stages/src/stages/hashing_storage.rs @@ -54,8 +54,9 @@ impl Stage for StorageHashingStage { // if there are more blocks then threshold it is faster to go over Plain state and hash all // account otherwise take changesets aggregate the sets and apply hashing to - // AccountHashing table - if to_transition - from_transition > self.clean_threshold { + // AccountHashing table. Also, if we start from genesis, we need to hash from scratch, as + // genesis accounts are not in changeset, along with their storages. + if to_transition - from_transition > self.clean_threshold || stage_progress == 0 { tx.clear::()?; tx.commit()?; diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index aa5a1c784f5a..0302d5edd3c0 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -1,7 +1,9 @@ use crate::{ - db::Transaction, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput, + db::Transaction, trie::DBTrieLoader, ExecInput, ExecOutput, Stage, StageError, StageId, + UnwindInput, UnwindOutput, }; -use reth_db::database::Database; +use reth_db::{database::Database, tables, transaction::DbTx}; +use reth_interfaces::consensus; use std::fmt::Debug; use tracing::*; @@ -34,10 +36,29 @@ pub const MERKLE_UNWIND: StageId = StageId("MerkleUnwind"); /// - [`MerkleStage::Execution`] #[derive(Debug)] pub enum MerkleStage { - /// The execution portion of the hashing stage. - Execution, - /// The unwind portion of the hasing stage. + /// The execution portion of the merkle stage. + Execution { + /// The threshold for switching from incremental trie building + /// of changes to whole rebuild. Num of transitions. + clean_threshold: u64, + }, + /// The unwind portion of the merkle stage. Unwind, + + #[cfg(test)] + Both { clean_threshold: u64 }, +} + +impl MerkleStage { + /// Stage default for the Execution variant. + pub fn default_execution() -> Self { + Self::Execution { clean_threshold: 5_000 } + } + + /// Stage default for the Unwind variant. + pub fn default_unwind() -> Self { + Self::Unwind + } } #[async_trait::async_trait] @@ -45,23 +66,64 @@ impl Stage for MerkleStage { /// Return the id of the stage fn id(&self) -> StageId { match self { - MerkleStage::Execution => MERKLE_EXECUTION, + MerkleStage::Execution { .. } => MERKLE_EXECUTION, MerkleStage::Unwind => MERKLE_UNWIND, + #[cfg(test)] + MerkleStage::Both { .. } => unreachable!(), } } /// Execute the stage. async fn execute( &mut self, - _tx: &mut Transaction<'_, DB>, + tx: &mut Transaction<'_, DB>, input: ExecInput, ) -> Result { - if matches!(self, MerkleStage::Unwind) { - info!(target: "sync::stages::merkle::unwind", "Stage is always skipped"); - return Ok(ExecOutput { stage_progress: input.previous_stage_progress(), done: true }) - } + let threshold = match self { + MerkleStage::Unwind => { + info!(target: "sync::stages::merkle::unwind", "Stage is always skipped"); + return Ok(ExecOutput { + stage_progress: input.previous_stage_progress(), + done: true, + }) + } + MerkleStage::Execution { clean_threshold } => *clean_threshold, + #[cfg(test)] + MerkleStage::Both { clean_threshold } => *clean_threshold, + }; - // Iterate over changeset (similar to Hashing stages) and take new values + let stage_progress = input.stage_progress.unwrap_or_default(); + let previous_stage_progress = input.previous_stage_progress(); + + let from_transition = tx.get_block_transition(stage_progress)?; + let to_transition = tx.get_block_transition(previous_stage_progress)?; + + let block_root = tx.get_header_by_num(previous_stage_progress)?.state_root; + + let trie_root = if from_transition == to_transition { + block_root + } else if to_transition - from_transition > threshold || stage_progress == 0 { + debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Rebuilding trie"); + // if there are more blocks than threshold it is faster to rebuild the trie + let loader = DBTrieLoader::default(); + loader.calculate_root(tx).map_err(|e| StageError::Fatal(Box::new(e)))? + } else { + debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Updating trie"); + // Iterate over changeset (similar to Hashing stages) and take new values + let current_root = tx.get_header_by_num(stage_progress)?.state_root; + let loader = DBTrieLoader::default(); + loader + .update_root(tx, current_root, from_transition..to_transition) + .map_err(|e| StageError::Fatal(Box::new(e)))? + }; + + if block_root != trie_root { + warn!(target: "sync::stages::merkle::exec", ?previous_stage_progress, got = ?block_root, expected = ?trie_root, "Block's root state failed verification"); + return Err(StageError::Validation { + block: previous_stage_progress, + error: consensus::Error::BodyStateRootDiff { got: trie_root, expected: block_root }, + }) + } info!(target: "sync::stages::merkle::exec", "Stage finished"); Ok(ExecOutput { stage_progress: input.previous_stage_progress(), done: true }) @@ -70,15 +132,419 @@ impl Stage for MerkleStage { /// Unwind the stage. async fn unwind( &mut self, - _tx: &mut Transaction<'_, DB>, + tx: &mut Transaction<'_, DB>, input: UnwindInput, ) -> Result { - if matches!(self, MerkleStage::Execution) { + if matches!(self, MerkleStage::Execution { .. }) { info!(target: "sync::stages::merkle::exec", "Stage is always skipped"); return Ok(UnwindOutput { stage_progress: input.unwind_to }) } + let target_root = tx.get_header_by_num(input.unwind_to)?.state_root; + + // If the merkle stage fails to execute, the trie changes weren't commited + // and the root stayed the same + if tx.get::(target_root)?.is_some() { + info!(target: "sync::stages::merkle::unwind", "Stage skipped"); + return Ok(UnwindOutput { stage_progress: input.unwind_to }) + } + + let loader = DBTrieLoader::default(); + let current_root = tx.get_header_by_num(input.stage_progress)?.state_root; + + let from_transition = tx.get_block_transition(input.unwind_to)?; + let to_transition = tx.get_block_transition(input.stage_progress)?; + + loader + .update_root(tx, current_root, from_transition..to_transition) + .map_err(|e| StageError::Fatal(Box::new(e)))?; + info!(target: "sync::stages::merkle::unwind", "Stage finished"); Ok(UnwindOutput { stage_progress: input.unwind_to }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, + TestTransaction, UnwindStageTestRunner, PREV_STAGE_ID, + }; + use assert_matches::assert_matches; + use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + models::{AccountBeforeTx, BlockNumHash, StoredBlockBody}, + tables, + transaction::{DbTx, DbTxMut}, + }; + use reth_interfaces::test_utils::generators::{ + random_block, random_block_range, random_contract_account_range, + }; + use reth_primitives::{keccak256, Account, Address, SealedBlock, StorageEntry, H256, U256}; + use std::collections::BTreeMap; + + stage_test_suite_ext!(MerkleTestRunner, merkle); + + /// Execute from genesis so as to merkelize whole state + #[tokio::test] + async fn execute_clean_merkle() { + let (previous_stage, stage_progress) = (500, 0); + + // Set up the runner + let mut runner = MerkleTestRunner::default(); + // set low threshold so we hash the whole storage + let input = ExecInput { + previous_stage: Some((PREV_STAGE_ID, previous_stage)), + stage_progress: Some(stage_progress), + }; + + runner.seed_execution(input).expect("failed to seed execution"); + + let rx = runner.execute(input); + + // Assert the successful result + let result = rx.await.unwrap(); + assert_matches!( + result, + Ok(ExecOutput { done, stage_progress }) + if done && stage_progress == previous_stage + ); + + // Validate the stage execution + assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation"); + } + + /// Update small trie + #[tokio::test] + async fn execute_small_merkle() { + let (previous_stage, stage_progress) = (2, 1); + + // Set up the runner + let mut runner = MerkleTestRunner::default(); + let input = ExecInput { + previous_stage: Some((PREV_STAGE_ID, previous_stage)), + stage_progress: Some(stage_progress), + }; + + runner.seed_execution(input).expect("failed to seed execution"); + + let rx = runner.execute(input); + + // Assert the successful result + let result = rx.await.unwrap(); + assert_matches!( + result, + Ok(ExecOutput { done, stage_progress }) + if done && stage_progress == previous_stage + ); + + // Validate the stage execution + assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation"); + } + + struct MerkleTestRunner { + tx: TestTransaction, + clean_threshold: u64, + } + + impl Default for MerkleTestRunner { + fn default() -> Self { + Self { tx: TestTransaction::default(), clean_threshold: 10000 } + } + } + + impl StageTestRunner for MerkleTestRunner { + type S = MerkleStage; + + fn tx(&self) -> &TestTransaction { + &self.tx + } + + fn stage(&self) -> Self::S { + Self::S::Both { clean_threshold: self.clean_threshold } + } + } + + #[async_trait::async_trait] + impl ExecuteStageTestRunner for MerkleTestRunner { + type Seed = Vec; + + fn seed_execution(&mut self, input: ExecInput) -> Result { + let stage_progress = input.stage_progress.unwrap_or_default(); + let end = input.previous_stage_progress() + 1; + + let n_accounts = 31; + let mut accounts = random_contract_account_range(&mut (0..n_accounts)); + + let SealedBlock { header, body, ommers } = + random_block(stage_progress, None, Some(0), None); + let mut header = header.unseal(); + header.state_root = self.generate_initial_trie(&accounts)?; + let sealed_head = SealedBlock { header: header.seal(), body, ommers }; + + let head_hash = sealed_head.hash(); + let mut blocks = vec![sealed_head]; + + blocks.extend(random_block_range((stage_progress + 1)..end, head_hash, 0..3)); + + self.tx.insert_headers(blocks.iter().map(|block| &block.header))?; + + let (mut transition_id, mut tx_id) = (0, 0); + + let mut storages: BTreeMap> = BTreeMap::new(); + + for progress in blocks.iter() { + // Insert last progress data + self.tx.commit(|tx| { + let key: BlockNumHash = (progress.number, progress.hash()).into(); + + let body = StoredBlockBody { + start_tx_id: tx_id, + tx_count: progress.body.len() as u64, + }; + + progress.body.iter().try_for_each(|transaction| { + tx.put::(transaction.hash(), tx_id)?; + tx.put::(tx_id, transaction.clone())?; + tx.put::(tx_id, transition_id)?; + + // seed account changeset + let (addr, prev_acc) = accounts + .get_mut(rand::random::() % n_accounts as usize) + .unwrap(); + let acc_before_tx = + AccountBeforeTx { address: *addr, info: Some(*prev_acc) }; + + tx.put::(transition_id, acc_before_tx)?; + + prev_acc.nonce += 1; + prev_acc.balance = prev_acc.balance.wrapping_add(U256::from(1)); + + let new_entry = StorageEntry { + key: keccak256([rand::random::()]), + value: U256::from(rand::random::() % 30 + 1), + }; + let storage = storages.entry(*addr).or_default(); + let old_value = storage.entry(new_entry.key).or_default(); + + tx.put::( + (transition_id, *addr).into(), + StorageEntry { key: new_entry.key, value: *old_value }, + )?; + + *old_value = new_entry.value; + + tx_id += 1; + transition_id += 1; + + Ok(()) + })?; + + tx.put::(key.number(), transition_id)?; + tx.put::(key, body) + })?; + } + + self.insert_accounts(&accounts)?; + self.insert_storages(&storages)?; + + let last_numhash = self.tx.inner().get_block_numhash(end - 1).unwrap(); + let root = self.state_root()?; + self.tx.commit(|tx| { + let mut last_header = tx.get::(last_numhash)?.unwrap(); + last_header.state_root = root; + tx.put::(last_numhash, last_header) + })?; + + Ok(blocks) + } + + fn validate_execution( + &self, + input: ExecInput, + output: Option, + ) -> Result<(), TestRunnerError> { + if let Some(output) = output { + let start_block = input.stage_progress.unwrap_or_default() + 1; + let end_block = output.stage_progress; + if start_block > end_block { + return Ok(()) + } + } + self.check_root(input.previous_stage_progress()) + } + } + + impl UnwindStageTestRunner for MerkleTestRunner { + fn before_unwind(&self, input: UnwindInput) -> Result<(), TestRunnerError> { + let target_transition = self + .tx + .inner() + .get_block_transition(input.unwind_to) + .map_err(|e| TestRunnerError::Internal(Box::new(e))) + .unwrap(); + + self.tx + .commit(|tx| { + let mut changeset_cursor = + tx.cursor_dup_read::().unwrap(); + let mut hash_cursor = tx.cursor_dup_write::().unwrap(); + + let mut rev_changeset_walker = changeset_cursor.walk_back(None).unwrap(); + + let mut tree: BTreeMap> = BTreeMap::new(); + + while let Some((tid_address, entry)) = + rev_changeset_walker.next().transpose().unwrap() + { + if tid_address.transition_id() < target_transition { + break + } + + tree.entry(keccak256(tid_address.address())) + .or_default() + .insert(keccak256(entry.key), entry.value); + } + for (key, val) in tree.into_iter() { + for (entry_key, entry_val) in val.into_iter() { + hash_cursor.seek_by_key_subkey(key, entry_key).unwrap(); + hash_cursor.delete_current().unwrap(); + + if entry_val != U256::ZERO { + let storage_entry = + StorageEntry { key: entry_key, value: entry_val }; + hash_cursor.append_dup(key, storage_entry).unwrap(); + } + } + } + + let mut changeset_cursor = + tx.cursor_dup_write::().unwrap(); + let mut rev_changeset_walker = changeset_cursor.walk_back(None).unwrap(); + + while let Some((transition_id, account_before_tx)) = + rev_changeset_walker.next().transpose().unwrap() + { + if transition_id < target_transition { + break + } + + match account_before_tx.info { + Some(acc) => { + tx.put::(account_before_tx.address, acc) + .unwrap(); + tx.put::( + keccak256(account_before_tx.address), + acc, + ) + .unwrap(); + } + None => { + tx.delete::( + account_before_tx.address, + None, + ) + .unwrap(); + tx.delete::( + keccak256(account_before_tx.address), + None, + ) + .unwrap(); + } + } + } + Ok(()) + }) + .unwrap(); + Ok(()) + } + + fn validate_unwind(&self, input: UnwindInput) -> Result<(), TestRunnerError> { + self.check_root(input.unwind_to) + } + } + + impl MerkleTestRunner { + fn state_root(&self) -> Result { + Ok(DBTrieLoader::default().calculate_root(&self.tx.inner()).unwrap()) + } + + pub(crate) fn generate_initial_trie( + &self, + accounts: &[(Address, Account)], + ) -> Result { + self.insert_accounts(accounts)?; + + let loader = DBTrieLoader::default(); + + let mut tx = self.tx.inner(); + let root = loader.calculate_root(&tx).expect("couldn't create initial trie"); + + tx.commit()?; + + Ok(root) + } + + pub(crate) fn insert_accounts( + &self, + accounts: &[(Address, Account)], + ) -> Result<(), TestRunnerError> { + for (addr, acc) in accounts.iter() { + self.tx.commit(|tx| { + tx.put::(*addr, *acc)?; + tx.put::(keccak256(addr), *acc)?; + Ok(()) + })?; + } + + Ok(()) + } + + fn insert_storages( + &self, + storages: &BTreeMap>, + ) -> Result<(), TestRunnerError> { + self.tx + .commit(|tx| { + storages.iter().try_for_each(|(&addr, storage)| { + storage.iter().try_for_each(|(&key, &value)| { + let entry = StorageEntry { key, value }; + tx.put::(addr, entry) + }) + })?; + storages + .iter() + .map(|(addr, storage)| { + ( + keccak256(addr), + storage + .iter() + .filter(|(_, &value)| value != U256::ZERO) + .map(|(key, value)| (keccak256(key), value)), + ) + }) + .collect::>() + .into_iter() + .try_for_each(|(addr, storage)| { + storage.into_iter().try_for_each(|(key, &value)| { + let entry = StorageEntry { key, value }; + tx.put::(addr, entry) + }) + })?; + Ok(()) + }) + .map_err(|e| e.into()) + } + + fn check_root(&self, previous_stage_progress: u64) -> Result<(), TestRunnerError> { + if previous_stage_progress != 0 { + let block_root = + self.tx.inner().get_header_by_num(previous_stage_progress).unwrap().state_root; + let root = DBTrieLoader::default().calculate_root(&self.tx.inner()).unwrap(); + assert_eq!(block_root, root); + } + Ok(()) + } + } +} diff --git a/crates/stages/src/test_utils/macros.rs b/crates/stages/src/test_utils/macros.rs index e3849f1b4e5c..56f6a561a242 100644 --- a/crates/stages/src/test_utils/macros.rs +++ b/crates/stages/src/test_utils/macros.rs @@ -60,6 +60,8 @@ macro_rules! stage_test_suite { // Seed the database runner.seed_execution(crate::stage::ExecInput::default()).expect("failed to seed"); + runner.before_unwind(input).expect("failed to execute before_unwind hook"); + // Run stage unwind let rx = runner.unwind(input).await; assert_matches::assert_matches!( @@ -97,12 +99,15 @@ macro_rules! stage_test_suite { ); assert!(runner.validate_execution(execute_input, result.ok()).is_ok(), "execution validation"); + // Run stage unwind let unwind_input = crate::stage::UnwindInput { unwind_to: stage_progress, stage_progress: previous_stage, bad_block: None, }; - let rx = runner.unwind(unwind_input).await; + runner.before_unwind(unwind_input).expect("Failed to unwind state"); + + let rx = runner.unwind(unwind_input).await; // Assert the successful unwind result assert_matches::assert_matches!( rx, diff --git a/crates/stages/src/test_utils/runner.rs b/crates/stages/src/test_utils/runner.rs index ff574e669080..b63d0dcc67d3 100644 --- a/crates/stages/src/test_utils/runner.rs +++ b/crates/stages/src/test_utils/runner.rs @@ -75,4 +75,9 @@ pub(crate) trait UnwindStageTestRunner: StageTestRunner { }); Box::pin(rx).await.unwrap() } + + /// Run a hook before [Stage::unwind]. Required for MerkleStage. + fn before_unwind(&self, _input: UnwindInput) -> Result<(), TestRunnerError> { + Ok(()) + } } diff --git a/crates/stages/src/trie/mod.rs b/crates/stages/src/trie/mod.rs new file mode 100644 index 000000000000..8559ade862fb --- /dev/null +++ b/crates/stages/src/trie/mod.rs @@ -0,0 +1,630 @@ +use crate::Transaction; +use cita_trie::{PatriciaTrie, Trie}; +use hasher::HasherKeccak; +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, + database::Database, + models::{AccountBeforeTx, TransitionIdAddress}, + tables, + transaction::{DbTx, DbTxMut}, +}; +use reth_primitives::{ + keccak256, proofs::EMPTY_ROOT, Account, Address, StorageEntry, StorageTrieEntry, TransitionId, + H256, KECCAK_EMPTY, U256, +}; +use reth_rlp::{ + encode_fixed_size, Decodable, DecodeError, Encodable, RlpDecodable, RlpEncodable, + EMPTY_STRING_CODE, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::Range, + sync::Arc, +}; +use tracing::*; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum TrieError { + #[error("Some error occurred: {0}")] + InternalError(#[from] cita_trie::TrieError), + #[error("The root node wasn't found in the DB")] + MissingRoot(H256), + #[error("{0:?}")] + DatabaseError(#[from] reth_db::Error), + #[error("{0:?}")] + DecodeError(#[from] DecodeError), +} + +/// Database wrapper implementing HashDB trait. +struct HashDatabase<'tx, 'itx, DB: Database> { + tx: &'tx Transaction<'itx, DB>, +} + +impl<'tx, 'itx, DB> cita_trie::DB for HashDatabase<'tx, 'itx, DB> +where + DB: Database, +{ + type Error = TrieError; + + fn get(&self, key: &[u8]) -> Result>, Self::Error> { + Ok(self.tx.get::(H256::from_slice(key))?) + } + + fn contains(&self, key: &[u8]) -> Result { + Ok(::get(self, key)?.is_some()) + } + + fn insert(&self, key: Vec, value: Vec) -> Result<(), Self::Error> { + // Caching and bulk inserting shouldn't be needed, as the data is ordered + self.tx.put::(H256::from_slice(key.as_slice()), value)?; + Ok(()) + } + + fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { + self.tx.delete::(H256::from_slice(key), None)?; + Ok(()) + } + + fn flush(&self) -> Result<(), Self::Error> { + Ok(()) + } +} + +impl<'tx, 'itx, DB: Database> HashDatabase<'tx, 'itx, DB> { + /// Instantiates a new Database for the accounts trie, with an empty root + fn new(tx: &'tx Transaction<'itx, DB>) -> Result { + let root = EMPTY_ROOT; + if tx.get::(root)?.is_none() { + tx.put::(root, [EMPTY_STRING_CODE].to_vec())?; + } + Ok(Self { tx }) + } + + /// Instantiates a new Database for the accounts trie, with an existing root + fn from_root(tx: &'tx Transaction<'itx, DB>, root: H256) -> Result { + if root == EMPTY_ROOT { + return Self::new(tx) + } + tx.get::(root)?.ok_or(TrieError::MissingRoot(root))?; + Ok(Self { tx }) + } +} + +/// Database wrapper implementing HashDB trait. +struct DupHashDatabase<'tx, 'itx, DB: Database> { + tx: &'tx Transaction<'itx, DB>, + key: H256, +} + +impl<'tx, 'itx, DB> cita_trie::DB for DupHashDatabase<'tx, 'itx, DB> +where + DB: Database, +{ + type Error = TrieError; + + fn get(&self, key: &[u8]) -> Result>, Self::Error> { + let mut cursor = self.tx.cursor_dup_read::()?; + Ok(cursor.seek_by_key_subkey(self.key, H256::from_slice(key))?.map(|entry| entry.node)) + } + + fn contains(&self, key: &[u8]) -> Result { + Ok(::get(self, key)?.is_some()) + } + + fn insert(&self, key: Vec, value: Vec) -> Result<(), Self::Error> { + // Caching and bulk inserting shouldn't be needed, as the data is ordered + self.tx.put::( + self.key, + StorageTrieEntry { hash: H256::from_slice(key.as_slice()), node: value }, + )?; + Ok(()) + } + + fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { + let mut cursor = self.tx.cursor_dup_write::()?; + cursor + .seek_by_key_subkey(self.key, H256::from_slice(key))? + .map(|_| cursor.delete_current()) + .transpose()?; + Ok(()) + } + + fn flush(&self) -> Result<(), Self::Error> { + Ok(()) + } +} + +impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> { + /// Instantiates a new Database for the storage trie, with an empty root + fn new(tx: &'tx Transaction<'itx, DB>, key: H256) -> Result { + let root = EMPTY_ROOT; + let mut cursor = tx.cursor_dup_write::()?; + if cursor.seek_by_key_subkey(key, root)?.is_none() { + tx.put::( + key, + StorageTrieEntry { hash: root, node: [EMPTY_STRING_CODE].to_vec() }, + )?; + } + Ok(Self { tx, key }) + } + + /// Instantiates a new Database for the storage trie, with an existing root + fn from_root(tx: &'tx Transaction<'itx, DB>, key: H256, root: H256) -> Result { + if root == EMPTY_ROOT { + return Self::new(tx, key) + } + tx.cursor_dup_read::()? + .seek_by_key_subkey(key, root)? + .ok_or(TrieError::MissingRoot(root))?; + Ok(Self { tx, key }) + } +} + +/// An Ethereum account, for RLP encoding traits deriving. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)] +pub(crate) struct EthAccount { + /// Account nonce. + nonce: u64, + /// Account balance. + balance: U256, + /// Account's storage root. + storage_root: H256, + /// Hash of the account's bytecode. + code_hash: H256, +} + +impl From for EthAccount { + fn from(acc: Account) -> Self { + EthAccount { + nonce: acc.nonce, + balance: acc.balance, + storage_root: EMPTY_ROOT, + code_hash: acc.bytecode_hash.unwrap_or(KECCAK_EMPTY), + } + } +} + +impl EthAccount { + pub(crate) fn from_with_root(acc: Account, storage_root: H256) -> EthAccount { + Self { storage_root, ..Self::from(acc) } + } +} + +#[derive(Debug, Default)] +pub(crate) struct DBTrieLoader; + +impl DBTrieLoader { + /// Calculates the root of the state trie, saving intermediate hashes in the database. + pub(crate) fn calculate_root( + &self, + tx: &Transaction<'_, DB>, + ) -> Result { + tx.clear::()?; + tx.clear::()?; + + let mut accounts_cursor = tx.cursor_read::()?; + let mut walker = accounts_cursor.walk(H256::zero())?; + + let db = Arc::new(HashDatabase::new(tx)?); + + let hasher = Arc::new(HasherKeccak::new()); + + let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher)); + + while let Some((hashed_address, account)) = walker.next().transpose()? { + let value = EthAccount::from_with_root( + account, + self.calculate_storage_root(tx, hashed_address)?, + ); + + let mut out = Vec::new(); + Encodable::encode(&value, &mut out); + trie.insert(hashed_address.as_bytes().to_vec(), out)?; + } + let root = H256::from_slice(trie.root()?.as_slice()); + + Ok(root) + } + + fn calculate_storage_root( + &self, + tx: &Transaction<'_, DB>, + address: H256, + ) -> Result { + let db = Arc::new(DupHashDatabase::new(tx, address)?); + + let hasher = Arc::new(HasherKeccak::new()); + + let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher)); + + let mut storage_cursor = tx.cursor_dup_read::()?; + + // Should be able to use walk_dup, but any call to next() causes an assert fail in mdbx.c + // let mut walker = storage_cursor.walk_dup(address, H256::zero())?; + let mut current = storage_cursor.seek_by_key_subkey(address, H256::zero())?; + + while let Some(StorageEntry { key: storage_key, value }) = current { + let out = encode_fixed_size(&value).to_vec(); + trie.insert(storage_key.to_vec(), out)?; + current = storage_cursor.next_dup()?.map(|(_, v)| v); + } + + let root = H256::from_slice(trie.root()?.as_slice()); + + Ok(root) + } + + /// Calculates the root of the state trie by updating an existing trie. + pub(crate) fn update_root( + &self, + tx: &Transaction<'_, DB>, + root: H256, + tid_range: Range, + ) -> Result { + let mut accounts_cursor = tx.cursor_read::()?; + + let changed_accounts = self.gather_changes(tx, tid_range)?; + + let db = Arc::new(HashDatabase::from_root(tx, root)?); + + let hasher = Arc::new(HasherKeccak::new()); + + let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; + + for (address, changed_storages) in changed_accounts { + if let Some(account) = trie.get(address.as_slice())? { + let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root; + trie.remove(address.as_bytes())?; + + if let Some((_, account)) = accounts_cursor.seek_exact(address)? { + let value = EthAccount::from_with_root( + account, + self.update_storage_root(tx, storage_root, address, changed_storages)?, + ); + + let mut out = Vec::new(); + Encodable::encode(&value, &mut out); + trie.insert(address.as_bytes().to_vec(), out)?; + } + } + } + + let root = H256::from_slice(trie.root()?.as_slice()); + + Ok(root) + } + + fn update_storage_root( + &self, + tx: &Transaction<'_, DB>, + root: H256, + address: H256, + changed_storages: BTreeSet, + ) -> Result { + let db = Arc::new(DupHashDatabase::from_root(tx, address, root)?); + + let hasher = Arc::new(HasherKeccak::new()); + + let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; + let mut storage_cursor = tx.cursor_dup_read::()?; + + for key in changed_storages { + if let Some(StorageEntry { value, .. }) = + storage_cursor.seek_by_key_subkey(address, key)? + { + let out = encode_fixed_size(&value).to_vec(); + trie.insert(key.as_bytes().to_vec(), out)?; + } else { + trie.remove(key.as_bytes())?; + } + } + + let root = H256::from_slice(trie.root()?.as_slice()); + + Ok(root) + } + + fn gather_changes( + &self, + tx: &Transaction<'_, DB>, + tid_range: Range, + ) -> Result>, TrieError> { + let mut account_cursor = tx.cursor_read::()?; + + let mut account_changes: BTreeMap> = BTreeMap::new(); + + let mut walker = account_cursor.walk_range(tid_range.clone())?; + + while let Some((_, AccountBeforeTx { address, .. })) = walker.next().transpose()? { + account_changes.insert(address, Default::default()); + } + + let mut storage_cursor = tx.cursor_dup_read::()?; + + let start = (tid_range.start, Address::zero()).into(); + let end = (tid_range.end, Address::zero()).into(); + let mut walker = storage_cursor.walk_range(start..end)?; + + while let Some((TransitionIdAddress((_, address)), StorageEntry { key, .. })) = + walker.next().transpose()? + { + account_changes.entry(address).or_default().insert(key); + } + + let hashed_changes = account_changes + .into_iter() + .map(|(address, storage)| { + (keccak256(address), storage.into_iter().map(keccak256).collect()) + }) + .collect(); + + Ok(hashed_changes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use reth_db::{mdbx::test_utils::create_test_rw_db, tables, transaction::DbTxMut}; + use reth_primitives::{ + hex_literal::hex, + keccak256, + proofs::{genesis_state_root, KeccakHasher, EMPTY_ROOT}, + Address, ChainSpec, + }; + use reth_staged_sync::utils::chainspec::chain_spec_value_parser; + use std::{collections::HashMap, str::FromStr}; + use triehash::sec_trie_root; + + #[test] + fn empty_trie() { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + assert_matches!(trie.calculate_root(&tx), Ok(got) if got == EMPTY_ROOT); + } + + #[test] + fn single_account_trie() { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); + let account = Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }; + tx.put::(keccak256(address), account).unwrap(); + let mut encoded_account = Vec::new(); + EthAccount::from(account).encode(&mut encoded_account); + let expected = H256(sec_trie_root::([(address, encoded_account)]).0); + assert_matches!( + trie.calculate_root(&tx), + Ok(got) if got == expected + ); + } + + #[test] + fn two_accounts_trie() { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + + let accounts = [ + ( + Address::from(hex!("9fe4abd71ad081f091bd06dd1c16f7e92927561e")), + Account { nonce: 155, balance: U256::from(414241124), bytecode_hash: None }, + ), + ( + Address::from(hex!("f8a6edaad4a332e6e550d0915a7fd5300b0b12d1")), + Account { nonce: 3, balance: U256::from(78978), bytecode_hash: None }, + ), + ]; + for (address, account) in accounts { + tx.put::(keccak256(address), account).unwrap(); + } + let encoded_accounts = accounts.iter().map(|(k, v)| { + let mut out = Vec::new(); + EthAccount::from(*v).encode(&mut out); + (k, out) + }); + let expected = H256(sec_trie_root::(encoded_accounts).0); + assert_matches!( + trie.calculate_root(&tx), + Ok(got) if got == expected + ); + } + + #[test] + fn single_storage_trie() { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + + let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); + let hashed_address = keccak256(address); + + let storage = Vec::from([(H256::from_low_u64_be(2), U256::from(1))]); + for (k, v) in storage.clone() { + tx.put::( + hashed_address, + StorageEntry { key: keccak256(k), value: v }, + ) + .unwrap(); + } + let encoded_storage = storage.iter().map(|(k, v)| { + let out = encode_fixed_size(v).to_vec(); + (k, out) + }); + let expected = H256(sec_trie_root::(encoded_storage).0); + assert_matches!( + trie.calculate_storage_root(&tx, hashed_address), + Ok(got) if got == expected + ); + } + + #[test] + fn single_account_with_storage_trie() { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + + let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); + let hashed_address = keccak256(address); + + let storage = HashMap::from([ + (H256::zero(), U256::from(3)), + (H256::from_low_u64_be(2), U256::from(1)), + ]); + let code = "el buen fla"; + let account = Account { + nonce: 155, + balance: U256::from(414241124u32), + bytecode_hash: Some(keccak256(code)), + }; + tx.put::(hashed_address, account).unwrap(); + + for (k, v) in storage.clone() { + tx.put::( + hashed_address, + StorageEntry { key: keccak256(k), value: v }, + ) + .unwrap(); + } + let mut out = Vec::new(); + + let encoded_storage = storage.iter().map(|(k, v)| { + let out = encode_fixed_size(v).to_vec(); + (k, out) + }); + + let eth_account = EthAccount::from_with_root( + account, + H256(sec_trie_root::(encoded_storage).0), + ); + eth_account.encode(&mut out); + + let expected = H256(sec_trie_root::([(address, out)]).0); + assert_matches!( + trie.calculate_root(&tx), + Ok(got) if got == expected + ); + } + + #[test] + fn verify_genesis() { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let mut tx = Transaction::new(db.as_ref()).unwrap(); + let ChainSpec { genesis, .. } = chain_spec_value_parser("mainnet").unwrap(); + + // Insert account state + for (address, account) in &genesis.alloc { + tx.put::( + keccak256(address), + Account { + nonce: account.nonce.unwrap_or_default(), + balance: account.balance, + bytecode_hash: None, + }, + ) + .unwrap(); + } + tx.commit().unwrap(); + + let state_root = genesis_state_root(genesis.alloc); + + assert_matches!( + trie.calculate_root(&tx), + Ok(got) if got == state_root + ); + } + + #[test] + fn gather_changes() { + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + + let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); + let hashed_address = keccak256(address); + + let storage = HashMap::from([ + (H256::zero(), U256::from(3)), + (H256::from_low_u64_be(2), U256::from(1)), + ]); + let code = "el buen fla"; + let account = Account { + nonce: 155, + balance: U256::from(414241124u32), + bytecode_hash: Some(keccak256(code)), + }; + tx.put::(hashed_address, account).unwrap(); + tx.put::(31, AccountBeforeTx { address, info: None }).unwrap(); + + for (k, v) in storage { + tx.put::( + hashed_address, + StorageEntry { key: keccak256(k), value: v }, + ) + .unwrap(); + tx.put::( + (32, address).into(), + StorageEntry { key: k, value: U256::ZERO }, + ) + .unwrap(); + } + + let expected = BTreeMap::from([( + hashed_address, + BTreeSet::from([keccak256(H256::zero()), keccak256(H256::from_low_u64_be(2))]), + )]); + assert_matches!( + DBTrieLoader::default().gather_changes(&tx, 32..33), + Ok(got) if got == expected + ); + } + + fn test_with_accounts(accounts: BTreeMap)>) { + let trie = DBTrieLoader::default(); + let db = create_test_rw_db(); + let tx = Transaction::new(db.as_ref()).unwrap(); + + let encoded_accounts = accounts + .into_iter() + .map(|(address, (account, storage))| { + let hashed_address = keccak256(address); + tx.put::(hashed_address, account).unwrap(); + // This is to mimic real data. Only contract accounts have storage. + let storage_root = if account.has_bytecode() { + let encoded_storage = storage.into_iter().map(|StorageEntry { key, value }| { + let hashed_key = keccak256(key); + let out = encode_fixed_size(&value).to_vec(); + tx.put::( + hashed_address, + StorageEntry { key: hashed_key, value }, + ) + .unwrap(); + (key, out) + }); + H256(sec_trie_root::(encoded_storage).0) + } else { + EMPTY_ROOT + }; + let mut out = Vec::new(); + EthAccount::from_with_root(account, storage_root).encode(&mut out); + (address, out) + }) + .collect::)>>(); + + let expected = H256(sec_trie_root::(encoded_accounts).0); + assert_matches!( + trie.calculate_root(&tx), + Ok(got) if got == expected + , "where expected is {expected:?}"); + } + + #[test] + fn arbitrary() { + proptest::proptest!(|(accounts: BTreeMap)>)| { + test_with_accounts(accounts); + }); + } +} diff --git a/crates/storage/db/src/abstraction/cursor.rs b/crates/storage/db/src/abstraction/cursor.rs index e581afbaa57a..f56458db95ba 100644 --- a/crates/storage/db/src/abstraction/cursor.rs +++ b/crates/storage/db/src/abstraction/cursor.rs @@ -105,7 +105,7 @@ pub trait DbCursorRW<'tx, T: Table> { /// Read Write Cursor over DupSorted table. pub trait DbDupCursorRW<'tx, T: DupSort> { - /// Append value to next cursor item + /// Delete all duplicate entries for current key. fn delete_current_duplicates(&mut self) -> Result<(), Error>; /// Append duplicate value. diff --git a/crates/storage/db/src/tables/codecs/compact.rs b/crates/storage/db/src/tables/codecs/compact.rs index b58bda1d05b5..1c9471903c3c 100644 --- a/crates/storage/db/src/tables/codecs/compact.rs +++ b/crates/storage/db/src/tables/codecs/compact.rs @@ -40,6 +40,7 @@ impl_compression_for_compact!( Receipt, TxType, StorageEntry, + StorageTrieEntry, StoredBlockBody, StoredBlockOmmers ); diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 1437cbf92733..47c3213903e1 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -18,7 +18,7 @@ use crate::{ }; use reth_primitives::{ Account, Address, BlockHash, BlockNumber, Header, IntegerList, Receipt, StorageEntry, - TransactionSigned, TransitionId, TxHash, TxNumber, H256, + StorageTrieEntry, TransactionSigned, TransitionId, TxHash, TxNumber, H256, }; use self::models::{storage_sharded_key::StorageShardedKey, StoredBlockBody}; @@ -33,7 +33,7 @@ pub enum TableType { } /// Default tables that should be present inside database. -pub const TABLES: [(TableType, &str); 25] = [ +pub const TABLES: [(TableType, &str); 27] = [ (TableType::Table, CanonicalHeaders::const_name()), (TableType::Table, HeaderTD::const_name()), (TableType::Table, HeaderNumbers::const_name()), @@ -56,6 +56,8 @@ pub const TABLES: [(TableType, &str); 25] = [ (TableType::DupSort, StorageChangeSet::const_name()), (TableType::Table, HashedAccount::const_name()), (TableType::DupSort, HashedStorage::const_name()), + (TableType::Table, AccountsTrie::const_name()), + (TableType::DupSort, StoragesTrie::const_name()), (TableType::Table, TxSenders::const_name()), (TableType::Table, Config::const_name()), (TableType::Table, SyncStage::const_name()), @@ -271,6 +273,16 @@ dupsort!( ( HashedStorage ) H256 | [H256] StorageEntry ); +table!( + /// Stores the current state's Merkle Patricia Tree. + ( AccountsTrie ) H256 | Vec +); + +dupsort!( + /// Stores the Merkle Patricia Trees of each [`Account`]'s storage. + ( StoragesTrie ) H256 | [H256] StorageTrieEntry +); + table!( /// Stores the transaction sender for each transaction. /// It is needed to speed up execution stage and allows fetching signer without doing