From 9cd21548a461d6cd866da8fa049f2242f7955d47 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 30 Sep 2022 11:48:09 +0300 Subject: [PATCH 1/8] beefy-mmr: reuse sp_runtime::traits::Keccak256 --- Cargo.lock | 2 +- frame/beefy-mmr/primitives/Cargo.toml | 8 ++++---- frame/beefy-mmr/primitives/src/lib.rs | 28 ++++++--------------------- frame/beefy-mmr/src/lib.rs | 27 ++++++++++---------------- frame/beefy-mmr/src/mock.rs | 2 +- 5 files changed, 22 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de50d4ec27105..2464ffc8a45c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -525,7 +525,7 @@ dependencies = [ "env_logger", "log", "sp-api", - "tiny-keccak", + "sp-runtime", ] [[package]] diff --git a/frame/beefy-mmr/primitives/Cargo.toml b/frame/beefy-mmr/primitives/Cargo.toml index 1aa2573c7f680..a097da0fc30fd 100644 --- a/frame/beefy-mmr/primitives/Cargo.toml +++ b/frame/beefy-mmr/primitives/Cargo.toml @@ -11,10 +11,10 @@ homepage = "https://substrate.io" [dependencies] array-bytes = { version = "4.1", optional = true } log = { version = "0.4", default-features = false, optional = true } -tiny-keccak = { version = "2.0.2", features = ["keccak"], optional = true } beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/beefy" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } [dev-dependencies] array-bytes = "4.1" @@ -22,9 +22,9 @@ env_logger = "0.9" [features] debug = ["array-bytes", "log"] -default = ["debug", "keccak", "std"] -keccak = ["tiny-keccak"] +default = ["debug", "std"] std = [ "beefy-primitives/std", - "sp-api/std" + "sp-api/std", + "sp-runtime/std" ] diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 38831d7914715..7c657e0b7a745 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -37,6 +37,8 @@ extern crate alloc; use alloc::vec::Vec; use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet}; +use sp_runtime::traits::Hash as HashT; +pub use sp_runtime::traits::Keccak256; /// Supported hashing output size. /// @@ -53,30 +55,12 @@ pub trait Hasher { fn hash(data: &[u8]) -> Hash; } -#[cfg(feature = "keccak")] -mod keccak256 { - use tiny_keccak::{Hasher as _, Keccak}; - - /// Keccak256 hasher implementation. - pub struct Keccak256; - impl Keccak256 { - /// Hash given data. - pub fn hash(data: &[u8]) -> super::Hash { - ::hash(data) - } - } - impl super::Hasher for Keccak256 { - fn hash(data: &[u8]) -> super::Hash { - let mut keccak = Keccak::v256(); - keccak.update(data); - let mut output = [0_u8; 32]; - keccak.finalize(&mut output); - output - } +/// Keccak256 hasher implementation. +impl Hasher for Keccak256 { + fn hash(data: &[u8]) -> Hash { + ::hash(data).into() } } -#[cfg(feature = "keccak")] -pub use keccak256::Keccak256; /// Construct a root hash of a Binary Merkle Tree created from given leaves. /// diff --git a/frame/beefy-mmr/src/lib.rs b/frame/beefy-mmr/src/lib.rs index 456d6e77aa8eb..fd8b35d019943 100644 --- a/frame/beefy-mmr/src/lib.rs +++ b/frame/beefy-mmr/src/lib.rs @@ -33,7 +33,7 @@ //! //! and thanks to versioning can be easily updated in the future. -use sp_runtime::traits::{Convert, Hash, Member}; +use sp_runtime::traits::{Convert, Member}; use sp_std::prelude::*; use beefy_primitives::{ @@ -142,10 +142,7 @@ pub mod pallet { StorageValue<_, BeefyNextAuthoritySet>, ValueQuery>; } -impl LeafDataProvider for Pallet -where - MerkleRootOf: From + Into, -{ +impl LeafDataProvider for Pallet { type LeafData = MmrLeaf< ::BlockNumber, ::Hash, @@ -163,19 +160,11 @@ where } } -impl beefy_merkle_tree::Hasher for Pallet -where - MerkleRootOf: Into, -{ - fn hash(data: &[u8]) -> beefy_merkle_tree::Hash { - ::Hashing::hash(data).into() - } -} - impl beefy_primitives::OnNewValidatorSet<::BeefyId> for Pallet where T: pallet::Config, - MerkleRootOf: From + Into, + ::Hashing: beefy_merkle_tree::Hasher, + beefy_merkle_tree::Hash: Into>, { /// Compute and cache BEEFY authority sets based on updated BEEFY validator sets. fn on_new_validator_set( @@ -192,7 +181,8 @@ where impl Pallet where - MerkleRootOf: From + Into, + ::Hashing: beefy_merkle_tree::Hasher, + beefy_merkle_tree::Hash: Into>, { /// Return the currently active BEEFY authority set proof. pub fn authority_set_proof() -> BeefyAuthoritySet> { @@ -220,7 +210,10 @@ where .map(T::BeefyAuthorityToMerkleLeaf::convert) .collect::>(); let len = beefy_addresses.len() as u32; - let root = beefy_merkle_tree::merkle_root::(beefy_addresses).into(); + let root = beefy_merkle_tree::merkle_root::<::Hashing, _, _>( + beefy_addresses, + ) + .into(); BeefyAuthoritySet { id, len, root } } } diff --git a/frame/beefy-mmr/src/mock.rs b/frame/beefy-mmr/src/mock.rs index 602d0aa5fe1a6..e648cca2e3858 100644 --- a/frame/beefy-mmr/src/mock.rs +++ b/frame/beefy-mmr/src/mock.rs @@ -147,7 +147,7 @@ impl BeefyDataProvider> for DummyDataProvider { fn extra_data() -> Vec { let mut col = vec![(15, vec![1, 2, 3]), (5, vec![4, 5, 6])]; col.sort(); - beefy_merkle_tree::merkle_root::, _, _>( + beefy_merkle_tree::merkle_root::<::Hashing, _, _>( col.into_iter().map(|pair| pair.encode()), ) .to_vec() From 0af2b277830782806d9d3706b2e03cbb069809cc Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 30 Sep 2022 12:14:06 +0300 Subject: [PATCH 2/8] beefy-mmr: use sp_runtime::traits:Hash for generating merkle proofs --- frame/beefy-mmr/primitives/src/lib.rs | 107 ++++++++++++++------------ 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 7c657e0b7a745..8358ee3318d0c 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -37,8 +37,8 @@ extern crate alloc; use alloc::vec::Vec; use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet}; -use sp_runtime::traits::Hash as HashT; pub use sp_runtime::traits::Keccak256; +use sp_runtime::{app_crypto::sp_core, traits::Hash as HashT}; /// Supported hashing output size. /// @@ -69,24 +69,26 @@ impl Hasher for Keccak256 { /// In case an empty list of leaves is passed the function returns a 0-filled hash. pub fn merkle_root(leaves: I) -> Hash where - H: Hasher, + H: HashT, + ::Output: Into, I: IntoIterator, T: AsRef<[u8]>, { - let iter = leaves.into_iter().map(|l| H::hash(l.as_ref())); - merkelize::(iter, &mut ()) + let iter = leaves.into_iter().map(|l| ::hash(l.as_ref())); + merkelize::(iter, &mut ()).into() } -fn merkelize(leaves: I, visitor: &mut V) -> Hash +fn merkelize(leaves: I, visitor: &mut V) -> Out where - H: Hasher, - V: Visitor, - I: Iterator, + H: HashT, + Out: AsRef<[u8]> + Default, + V: Visitor, + I: Iterator, { let upper = Vec::with_capacity(leaves.size_hint().0); - let mut next = match merkelize_row::(leaves, upper, visitor) { + let mut next = match merkelize_row::(leaves, upper, visitor) { Ok(root) => return root, - Err(next) if next.is_empty() => return Hash::default(), + Err(next) if next.is_empty() => return Out::default(), Err(next) => next, }; @@ -94,7 +96,7 @@ where loop { visitor.move_up(); - match merkelize_row::(next.drain(..), upper, visitor) { + match merkelize_row::(next.drain(..), upper, visitor) { Ok(root) => return root, Err(t) => { // swap collections to avoid allocations @@ -132,7 +134,7 @@ pub struct MerkleProof { /// /// It can be passed to [`merkelize_row`] or [`merkelize`] functions and will be notified /// about tree traversal. -trait Visitor { +trait Visitor { /// We are moving one level up in the tree. fn move_up(&mut self); @@ -142,13 +144,13 @@ trait Visitor { /// The method will also visit the `root` hash (level 0). /// /// The `index` is an index of `left` item. - fn visit(&mut self, index: usize, left: &Option, right: &Option); + fn visit(&mut self, index: usize, left: &Option, right: &Option); } /// No-op implementation of the visitor. -impl Visitor for () { +impl Visitor for () { fn move_up(&mut self) {} - fn visit(&mut self, _index: usize, _left: &Option, _right: &Option) {} + fn visit(&mut self, _index: usize, _left: &Option, _right: &Option) {} } /// Construct a Merkle Proof for leaves given by indices. @@ -161,16 +163,17 @@ impl Visitor for () { /// # Panic /// /// The function will panic if given `leaf_index` is greater than the number of leaves. -pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof +pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof where - H: Hasher, + H: HashT, + Out: Default + Copy + Into + AsRef<[u8]>, I: IntoIterator, I::IntoIter: ExactSizeIterator, T: AsRef<[u8]>, { let mut leaf = None; let iter = leaves.into_iter().enumerate().map(|(idx, l)| { - let hash = H::hash(l.as_ref()); + let hash = ::hash(l.as_ref()); if idx == leaf_index { leaf = Some(l); } @@ -178,23 +181,23 @@ where }); /// The struct collects a proof for single leaf. - struct ProofCollection { - proof: Vec, + struct ProofCollection { + proof: Vec, position: usize, } - impl ProofCollection { + impl ProofCollection { fn new(position: usize) -> Self { ProofCollection { proof: Default::default(), position } } } - impl Visitor for ProofCollection { + impl Visitor for ProofCollection { fn move_up(&mut self) { self.position /= 2; } - fn visit(&mut self, index: usize, left: &Option, right: &Option) { + fn visit(&mut self, index: usize, left: &Option, right: &Option) { // we are at left branch - right goes to the proof. if self.position == index { if let Some(right) = right { @@ -213,7 +216,7 @@ where let number_of_leaves = iter.len(); let mut collect_proof = ProofCollection::new(leaf_index); - let root = merkelize::(iter, &mut collect_proof); + let root = merkelize::(iter, &mut collect_proof); let leaf = leaf.expect("Requested `leaf_index` is greater than number of leaves."); #[cfg(feature = "debug")] @@ -222,11 +225,17 @@ where collect_proof .proof .iter() - .map(|s| array_bytes::bytes2hex("", s)) + .map(|s| array_bytes::bytes2hex("", s.as_ref())) .collect::>() ); - MerkleProof { root, proof: collect_proof.proof, number_of_leaves, leaf_index, leaf } + MerkleProof { + root: root.into(), + proof: collect_proof.proof.iter().cloned().map(|s| s.into()).collect::>(), + number_of_leaves, + leaf_index, + leaf, + } } /// Leaf node for proof verification. @@ -314,22 +323,24 @@ where /// /// In case only one element is provided it is returned via `Ok` result, in any other case (also an /// empty iterator) an `Err` with the inner nodes of upper layer is returned. -fn merkelize_row( +fn merkelize_row( mut iter: I, - mut next: Vec, + mut next: Vec, visitor: &mut V, -) -> Result> +) -> Result> where - H: Hasher, - V: Visitor, - I: Iterator, + H: HashT, + Out: AsRef<[u8]>, + V: Visitor, + I: Iterator, { #[cfg(feature = "debug")] log::debug!("[merkelize_row]"); next.clear(); + let hash_len = ::LENGTH; let mut index = 0; - let mut combined = [0_u8; 64]; + let mut combined = vec![0_u8; hash_len * 2]; loop { let a = iter.next(); let b = iter.next(); @@ -338,17 +349,17 @@ where #[cfg(feature = "debug")] log::debug!( " {:?}\n {:?}", - a.as_ref().map(|s| array_bytes::bytes2hex("", s)), - b.as_ref().map(|s| array_bytes::bytes2hex("", s)) + a.as_ref().map(|s| array_bytes::bytes2hex("", s.as_ref())), + b.as_ref().map(|s| array_bytes::bytes2hex("", s.as_ref())) ); index += 2; match (a, b) { (Some(a), Some(b)) => { - combined[0..32].copy_from_slice(&a); - combined[32..64].copy_from_slice(&b); + combined[..hash_len].copy_from_slice(a.as_ref()); + combined[hash_len..].copy_from_slice(b.as_ref()); - next.push(H::hash(&combined)); + next.push(::hash(&combined)); }, // Odd number of items. Promote the item to the upper layer. (Some(a), None) if !next.is_empty() => { @@ -361,7 +372,7 @@ where #[cfg(feature = "debug")] log::debug!( "[merkelize_row] Next: {:?}", - next.iter().map(|s| array_bytes::bytes2hex("", s)).collect::>() + next.iter().map(|s| array_bytes::bytes2hex("", s.as_ref())).collect::>() ); return Err(next) }, @@ -476,7 +487,7 @@ mod tests { let data = vec!["a", "b", "c"]; // when - let proof0 = merkle_proof::(data.clone(), 0); + let proof0 = merkle_proof::(data.clone(), 0); assert!(verify_proof::( &proof0.root, proof0.proof.clone(), @@ -485,7 +496,7 @@ mod tests { &proof0.leaf, )); - let proof1 = merkle_proof::(data.clone(), 1); + let proof1 = merkle_proof::(data.clone(), 1); assert!(verify_proof::( &proof1.root, proof1.proof, @@ -494,7 +505,7 @@ mod tests { &proof1.leaf, )); - let proof2 = merkle_proof::(data.clone(), 2); + let proof2 = merkle_proof::(data.clone(), 2); assert!(verify_proof::( &proof2.root, proof2.proof, @@ -540,7 +551,7 @@ mod tests { for l in 0..data.len() { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); // then assert!(verify_proof::( &proof.root, @@ -566,7 +577,7 @@ mod tests { for l in 0..data.len() { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); // then assert!(verify_proof::( &proof.root, @@ -590,7 +601,7 @@ mod tests { for l in (0..data.len()).step_by(13) { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); // then assert!(verify_proof::( &proof.root, @@ -606,7 +617,7 @@ mod tests { #[should_panic] fn should_panic_on_invalid_leaf_index() { let _ = env_logger::try_init(); - merkle_proof::(vec!["a"], 5); + merkle_proof::(vec!["a"], 5); } #[test] @@ -791,7 +802,7 @@ mod tests { for l in 0..data.len() { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); assert_eq!(array_bytes::bytes2hex("", &proof.root), array_bytes::bytes2hex("", &root)); assert_eq!(proof.leaf_index, l); assert_eq!(&proof.leaf, &data[l]); @@ -806,7 +817,7 @@ mod tests { )); } - let proof = merkle_proof::(data.clone(), data.len() - 1); + let proof = merkle_proof::(data.clone(), data.len() - 1); assert_eq!( proof, From 471881f9a577b6fa1076835478dce13cd29c3f45 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 30 Sep 2022 14:23:38 +0300 Subject: [PATCH 3/8] beefy-mmr: use sp_runtime::traits:Hash for validating merkle proofs --- frame/beefy-mmr/primitives/src/lib.rs | 118 +++++++++++++------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 8358ee3318d0c..f7ba835e4fab3 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -111,14 +111,14 @@ where /// /// The structure contains all necessary data to later on verify the proof and the leaf itself. #[derive(Debug, PartialEq, Eq)] -pub struct MerkleProof { +pub struct MerkleProof { /// Root hash of generated merkle tree. - pub root: Hash, + pub root: H, /// Proof items (does not contain the leaf hash, nor the root obviously). /// /// This vec contains all inner node hashes necessary to reconstruct the root hash given the /// leaf hash. - pub proof: Vec, + pub proof: Vec, /// Number of leaves in the original tree. /// /// This is needed to detect a case where we have an odd number of leaves that "get promoted" @@ -127,7 +127,7 @@ pub struct MerkleProof { /// Index of the leaf the proof is for (0-based). pub leaf_index: usize, /// Leaf content. - pub leaf: T, + pub leaf: L, } /// A trait of object inspecting merkle root creation. @@ -163,7 +163,7 @@ impl Visitor for () { /// # Panic /// /// The function will panic if given `leaf_index` is greater than the number of leaves. -pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof +pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof where H: HashT, Out: Default + Copy + Into + AsRef<[u8]>, @@ -229,13 +229,7 @@ where .collect::>() ); - MerkleProof { - root: root.into(), - proof: collect_proof.proof.iter().cloned().map(|s| s.into()).collect::>(), - number_of_leaves, - leaf_index, - leaf, - } + MerkleProof { root, proof: collect_proof.proof, number_of_leaves, leaf_index, leaf } } /// Leaf node for proof verification. @@ -243,25 +237,19 @@ where /// Can be either a value that needs to be hashed first, /// or the hash itself. #[derive(Debug, PartialEq, Eq)] -pub enum Leaf<'a> { +pub enum Leaf<'a, H> { /// Leaf content. Value(&'a [u8]), /// Hash of the leaf content. - Hash(Hash), + Hash(H), } -impl<'a, T: AsRef<[u8]>> From<&'a T> for Leaf<'a> { +impl<'a, H, T: AsRef<[u8]>> From<&'a T> for Leaf<'a, H> { fn from(v: &'a T) -> Self { Leaf::Value(v.as_ref()) } } -impl<'a> From for Leaf<'a> { - fn from(v: Hash) -> Self { - Leaf::Hash(v) - } -} - /// Verify Merkle Proof correctness versus given root hash. /// /// The proof is NOT expected to contain leaf hash as the first @@ -269,46 +257,48 @@ impl<'a> From for Leaf<'a> { /// concatenating and hashing end up with given root hash. /// /// The proof must not contain the root hash. -pub fn verify_proof<'a, H, P, L>( - root: &'a Hash, +pub fn verify_proof<'a, H, Out, P, L>( + root: &'a Out, proof: P, number_of_leaves: usize, leaf_index: usize, leaf: L, ) -> bool where - H: Hasher, - P: IntoIterator, - L: Into>, + H: HashT, + Out: AsRef<[u8]> + PartialEq, + P: IntoIterator, + L: Into>, { if leaf_index >= number_of_leaves { return false } let leaf_hash = match leaf.into() { - Leaf::Value(content) => H::hash(content), + Leaf::Value(content) => ::hash(content), Leaf::Hash(hash) => hash, }; - let mut combined = [0_u8; 64]; + let hash_len = ::LENGTH; + let mut combined = vec![0_u8; hash_len * 2]; let mut position = leaf_index; let mut width = number_of_leaves; let computed = proof.into_iter().fold(leaf_hash, |a, b| { if position % 2 == 1 || position + 1 == width { - combined[0..32].copy_from_slice(&b); - combined[32..64].copy_from_slice(&a); + combined[..hash_len].copy_from_slice(&b.as_ref()); + combined[hash_len..].copy_from_slice(&a.as_ref()); } else { - combined[0..32].copy_from_slice(&a); - combined[32..64].copy_from_slice(&b); + combined[..hash_len].copy_from_slice(&a.as_ref()); + combined[hash_len..].copy_from_slice(&b.as_ref()); } - let hash = H::hash(&combined); + let hash = ::hash(&combined); #[cfg(feature = "debug")] log::debug!( "[verify_proof]: (a, b) {:?}, {:?} => {:?} ({:?}) hash", - array_bytes::bytes2hex("", &a), - array_bytes::bytes2hex("", &b), - array_bytes::bytes2hex("", &hash), - array_bytes::bytes2hex("", &combined) + array_bytes::bytes2hex("", &a.as_ref()), + array_bytes::bytes2hex("", &b.as_ref()), + array_bytes::bytes2hex("", &hash.as_ref()), + array_bytes::bytes2hex("", &combined.as_ref()) ); position /= 2; width = ((width - 1) / 2) + 1; @@ -398,6 +388,7 @@ sp_api::decl_runtime_apis! { #[cfg(test)] mod tests { use super::*; + use crate::sp_core::H256; #[test] fn should_generate_empty_root() { @@ -488,7 +479,7 @@ mod tests { // when let proof0 = merkle_proof::(data.clone(), 0); - assert!(verify_proof::( + assert!(verify_proof::( &proof0.root, proof0.proof.clone(), data.len(), @@ -497,7 +488,7 @@ mod tests { )); let proof1 = merkle_proof::(data.clone(), 1); - assert!(verify_proof::( + assert!(verify_proof::( &proof1.root, proof1.proof, data.len(), @@ -506,7 +497,7 @@ mod tests { )); let proof2 = merkle_proof::(data.clone(), 2); - assert!(verify_proof::( + assert!(verify_proof::( &proof2.root, proof2.proof, data.len(), @@ -516,26 +507,27 @@ mod tests { // then assert_eq!( - array_bytes::bytes2hex("", &proof0.root), - array_bytes::bytes2hex("", &proof1.root) + array_bytes::bytes2hex("", &proof0.root.as_ref()), + array_bytes::bytes2hex("", &proof1.root.as_ref()) ); assert_eq!( - array_bytes::bytes2hex("", &proof2.root), - array_bytes::bytes2hex("", &proof1.root) + array_bytes::bytes2hex("", &proof2.root.as_ref()), + array_bytes::bytes2hex("", &proof1.root.as_ref()) ); - assert!(!verify_proof::( + assert!(!verify_proof::( &array_bytes::hex2array_unchecked( "fb3b3be94be9e983ba5e094c9c51a7d96a4fa2e5d8e891df00ca89ba05bb1239" - ), + ) + .into(), proof0.proof, data.len(), proof0.leaf_index, &proof0.leaf )); - assert!(!verify_proof::( - &proof0.root, + assert!(!verify_proof::( + &proof0.root.into(), vec![], data.len(), proof0.leaf_index, @@ -553,7 +545,7 @@ mod tests { // when let proof = merkle_proof::(data.clone(), l); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -579,7 +571,7 @@ mod tests { // when let proof = merkle_proof::(data.clone(), l); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -603,7 +595,7 @@ mod tests { // when let proof = merkle_proof::(data.clone(), l); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -791,9 +783,10 @@ mod tests { "0xA4cDc98593CE52d01Fe5Ca47CB3dA5320e0D7592", "0xc26B34D375533fFc4c5276282Fa5D660F3d8cbcB", ]; - let root = array_bytes::hex2array_unchecked( + let root: H256 = array_bytes::hex2array_unchecked( "72b0acd7c302a84f1f6b6cefe0ba7194b7398afb440e1b44a9dbbe270394ca53", - ); + ) + .into(); let data = addresses .into_iter() @@ -803,12 +796,15 @@ mod tests { for l in 0..data.len() { // when let proof = merkle_proof::(data.clone(), l); - assert_eq!(array_bytes::bytes2hex("", &proof.root), array_bytes::bytes2hex("", &root)); + assert_eq!( + array_bytes::bytes2hex("", &proof.root.as_ref()), + array_bytes::bytes2hex("", &root.as_ref()) + ); assert_eq!(proof.leaf_index, l); assert_eq!(&proof.leaf, &data[l]); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -826,16 +822,20 @@ mod tests { proof: vec![ array_bytes::hex2array_unchecked( "340bcb1d49b2d82802ddbcf5b85043edb3427b65d09d7f758fbc76932ad2da2f" - ), + ) + .into(), array_bytes::hex2array_unchecked( "ba0580e5bd530bc93d61276df7969fb5b4ae8f1864b4a28c280249575198ff1f" - ), + ) + .into(), array_bytes::hex2array_unchecked( "d02609d2bbdb28aa25f58b85afec937d5a4c85d37925bce6d0cf802f9d76ba79" - ), + ) + .into(), array_bytes::hex2array_unchecked( "ae3f8991955ed884613b0a5f40295902eea0e0abe5858fc520b72959bc016d4e" - ), + ) + .into(), ], number_of_leaves: data.len(), leaf_index: data.len() - 1, From d42a2ffd8425be78740431c3ccc4c06211e40f8e Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 30 Sep 2022 14:35:02 +0300 Subject: [PATCH 4/8] beefy-mmr: remove primitives::Hasher and primitives::Hash --- frame/beefy-mmr/primitives/src/lib.rs | 48 ++++++++------------------- frame/beefy-mmr/src/lib.rs | 10 ++---- frame/beefy-mmr/src/mock.rs | 3 +- 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index f7ba835e4fab3..f7f6e8c3da54f 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -40,37 +40,15 @@ use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet}; pub use sp_runtime::traits::Keccak256; use sp_runtime::{app_crypto::sp_core, traits::Hash as HashT}; -/// Supported hashing output size. -/// -/// The size is restricted to 32 bytes to allow for a more optimised implementation. -pub type Hash = [u8; 32]; - -/// Generic hasher trait. -/// -/// Implement the function to support custom way of hashing data. -/// The implementation must return a [Hash](type@Hash) type, so only 32-byte output hashes are -/// supported. -pub trait Hasher { - /// Hash given arbitrary-length piece of data. - fn hash(data: &[u8]) -> Hash; -} - -/// Keccak256 hasher implementation. -impl Hasher for Keccak256 { - fn hash(data: &[u8]) -> Hash { - ::hash(data).into() - } -} - /// Construct a root hash of a Binary Merkle Tree created from given leaves. /// /// See crate-level docs for details about Merkle Tree construction. /// /// In case an empty list of leaves is passed the function returns a 0-filled hash. -pub fn merkle_root(leaves: I) -> Hash +pub fn merkle_root(leaves: I) -> Out where - H: HashT, - ::Output: Into, + H: HashT, + Out: Default + AsRef<[u8]>, I: IntoIterator, T: AsRef<[u8]>, { @@ -166,7 +144,7 @@ impl Visitor for () { pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof where H: HashT, - Out: Default + Copy + Into + AsRef<[u8]>, + Out: Default + Copy + AsRef<[u8]>, I: IntoIterator, I::IntoIter: ExactSizeIterator, T: AsRef<[u8]>, @@ -374,7 +352,6 @@ sp_api::decl_runtime_apis! { /// API useful for BEEFY light clients. pub trait BeefyMmrApi where - H: From + Into, BeefyAuthoritySet: sp_api::Decode, { /// Return the currently active BEEFY authority set proof. @@ -397,11 +374,11 @@ mod tests { let data: Vec<[u8; 1]> = Default::default(); // when - let out = merkle_root::(data); + let out = merkle_root::(data); // then assert_eq!( - array_bytes::bytes2hex("", &out), + array_bytes::bytes2hex("", out.as_ref()), "0000000000000000000000000000000000000000000000000000000000000000" ); } @@ -415,11 +392,11 @@ mod tests { )]; // when - let out = merkle_root::(data); + let out = merkle_root::(data); // then assert_eq!( - array_bytes::bytes2hex("", &out), + array_bytes::bytes2hex("", out.as_ref()), "aeb47a269393297f4b0a3c9c9cfd00c7a4195255274cf39d83dabc2fcc9ff3d7" ); } @@ -434,11 +411,11 @@ mod tests { ]; // when - let out = merkle_root::(data); + let out = merkle_root::(data); // then assert_eq!( - array_bytes::bytes2hex("", &out), + array_bytes::bytes2hex("", out.as_ref()), "697ea2a8fe5b03468548a7a413424a6292ab44a82a6f5cc594c3fa7dda7ce402" ); } @@ -447,7 +424,10 @@ mod tests { fn should_generate_root_complex() { let _ = env_logger::try_init(); let test = |root, data| { - assert_eq!(array_bytes::bytes2hex("", &merkle_root::(data)), root); + assert_eq!( + array_bytes::bytes2hex("", &merkle_root::(data).as_ref()), + root + ); }; test( diff --git a/frame/beefy-mmr/src/lib.rs b/frame/beefy-mmr/src/lib.rs index fd8b35d019943..52cba4ac3571e 100644 --- a/frame/beefy-mmr/src/lib.rs +++ b/frame/beefy-mmr/src/lib.rs @@ -163,8 +163,6 @@ impl LeafDataProvider for Pallet { impl beefy_primitives::OnNewValidatorSet<::BeefyId> for Pallet where T: pallet::Config, - ::Hashing: beefy_merkle_tree::Hasher, - beefy_merkle_tree::Hash: Into>, { /// Compute and cache BEEFY authority sets based on updated BEEFY validator sets. fn on_new_validator_set( @@ -179,11 +177,7 @@ where } } -impl Pallet -where - ::Hashing: beefy_merkle_tree::Hasher, - beefy_merkle_tree::Hash: Into>, -{ +impl Pallet { /// Return the currently active BEEFY authority set proof. pub fn authority_set_proof() -> BeefyAuthoritySet> { Pallet::::beefy_authorities() @@ -210,7 +204,7 @@ where .map(T::BeefyAuthorityToMerkleLeaf::convert) .collect::>(); let len = beefy_addresses.len() as u32; - let root = beefy_merkle_tree::merkle_root::<::Hashing, _, _>( + let root = beefy_merkle_tree::merkle_root::<::Hashing, _, _, _>( beefy_addresses, ) .into(); diff --git a/frame/beefy-mmr/src/mock.rs b/frame/beefy-mmr/src/mock.rs index e648cca2e3858..58d46d56f9559 100644 --- a/frame/beefy-mmr/src/mock.rs +++ b/frame/beefy-mmr/src/mock.rs @@ -147,9 +147,10 @@ impl BeefyDataProvider> for DummyDataProvider { fn extra_data() -> Vec { let mut col = vec![(15, vec![1, 2, 3]), (5, vec![4, 5, 6])]; col.sort(); - beefy_merkle_tree::merkle_root::<::Hashing, _, _>( + beefy_merkle_tree::merkle_root::<::Hashing, _, _, _>( col.into_iter().map(|pair| pair.encode()), ) + .as_ref() .to_vec() } } From 52348da4cffa0168f078849775e81ab3ea3386f3 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 30 Sep 2022 14:57:20 +0300 Subject: [PATCH 5/8] fixes --- frame/beefy-mmr/primitives/src/lib.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index f7f6e8c3da54f..5be6b5fd08a02 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -25,20 +25,17 @@ //! compilation targets. //! //! Merkle Tree is constructed from arbitrary-length leaves, that are initially hashed using the -//! same [Hasher] as the inner nodes. +//! same hasher as the inner nodes. //! Inner nodes are created by concatenating child hashes and hashing again. The implementation //! does not perform any sorting of the input data (leaves) nor when inner nodes are created. //! //! If the number of leaves is not even, last leave (hash of) is promoted to the upper layer. -#[cfg(not(feature = "std"))] -extern crate alloc; -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; +pub use sp_runtime::traits::Keccak256; +use sp_runtime::{app_crypto::sp_core, sp_std, traits::Hash as HashT}; +use sp_std::{vec, vec::Vec}; use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet}; -pub use sp_runtime::traits::Keccak256; -use sp_runtime::{app_crypto::sp_core, traits::Hash as HashT}; /// Construct a root hash of a Binary Merkle Tree created from given leaves. /// From d5cd3e50fead130fb50764990902144ff3f4d0af Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Mon, 3 Oct 2022 11:40:12 +0300 Subject: [PATCH 6/8] beefy-mmr: reduce the number of generic parameters for merkle_root() --- frame/beefy-mmr/primitives/src/lib.rs | 108 +++++++++++++------------- frame/beefy-mmr/src/lib.rs | 2 +- frame/beefy-mmr/src/mock.rs | 2 +- 3 files changed, 56 insertions(+), 56 deletions(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 5be6b5fd08a02..4872ad524ff5d 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -42,28 +42,28 @@ use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet}; /// See crate-level docs for details about Merkle Tree construction. /// /// In case an empty list of leaves is passed the function returns a 0-filled hash. -pub fn merkle_root(leaves: I) -> Out +pub fn merkle_root(leaves: I) -> H::Output where - H: HashT, - Out: Default + AsRef<[u8]>, - I: IntoIterator, - T: AsRef<[u8]>, + H: HashT, + H::Output: Default + AsRef<[u8]>, + I: IntoIterator, + I::Item: AsRef<[u8]>, { let iter = leaves.into_iter().map(|l| ::hash(l.as_ref())); - merkelize::(iter, &mut ()).into() + merkelize::(iter, &mut ()).into() } -fn merkelize(leaves: I, visitor: &mut V) -> Out +fn merkelize(leaves: I, visitor: &mut V) -> H::Output where - H: HashT, - Out: AsRef<[u8]> + Default, - V: Visitor, - I: Iterator, + H: HashT, + H::Output: Default + AsRef<[u8]>, + V: Visitor, + I: Iterator, { - let upper = Vec::with_capacity(leaves.size_hint().0); - let mut next = match merkelize_row::(leaves, upper, visitor) { + let upper = Vec::with_capacity(leaves.size_hint().1.unwrap()); + let mut next = match merkelize_row::(leaves, upper, visitor) { Ok(root) => return root, - Err(next) if next.is_empty() => return Out::default(), + Err(next) if next.is_empty() => return H::Output::default(), Err(next) => next, }; @@ -71,7 +71,7 @@ where loop { visitor.move_up(); - match merkelize_row::(next.drain(..), upper, visitor) { + match merkelize_row::(next.drain(..), upper, visitor) { Ok(root) => return root, Err(t) => { // swap collections to avoid allocations @@ -138,10 +138,10 @@ impl Visitor for () { /// # Panic /// /// The function will panic if given `leaf_index` is greater than the number of leaves. -pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof +pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof where - H: HashT, - Out: Default + Copy + AsRef<[u8]>, + H: HashT, + H::Output: Default + Copy + AsRef<[u8]>, I: IntoIterator, I::IntoIter: ExactSizeIterator, T: AsRef<[u8]>, @@ -191,7 +191,7 @@ where let number_of_leaves = iter.len(); let mut collect_proof = ProofCollection::new(leaf_index); - let root = merkelize::(iter, &mut collect_proof); + let root = merkelize::(iter, &mut collect_proof); let leaf = leaf.expect("Requested `leaf_index` is greater than number of leaves."); #[cfg(feature = "debug")] @@ -232,18 +232,18 @@ impl<'a, H, T: AsRef<[u8]>> From<&'a T> for Leaf<'a, H> { /// concatenating and hashing end up with given root hash. /// /// The proof must not contain the root hash. -pub fn verify_proof<'a, H, Out, P, L>( - root: &'a Out, +pub fn verify_proof<'a, H, P, L>( + root: &'a H::Output, proof: P, number_of_leaves: usize, leaf_index: usize, leaf: L, ) -> bool where - H: HashT, - Out: AsRef<[u8]> + PartialEq, - P: IntoIterator, - L: Into>, + H: HashT, + H::Output: PartialEq + AsRef<[u8]>, + P: IntoIterator, + L: Into>, { if leaf_index >= number_of_leaves { return false @@ -288,16 +288,16 @@ where /// /// In case only one element is provided it is returned via `Ok` result, in any other case (also an /// empty iterator) an `Err` with the inner nodes of upper layer is returned. -fn merkelize_row( +fn merkelize_row( mut iter: I, - mut next: Vec, + mut next: Vec, visitor: &mut V, -) -> Result> +) -> Result> where - H: HashT, - Out: AsRef<[u8]>, - V: Visitor, - I: Iterator, + H: HashT, + H::Output: AsRef<[u8]>, + V: Visitor, + I: Iterator, { #[cfg(feature = "debug")] log::debug!("[merkelize_row]"); @@ -371,7 +371,7 @@ mod tests { let data: Vec<[u8; 1]> = Default::default(); // when - let out = merkle_root::(data); + let out = merkle_root::(data); // then assert_eq!( @@ -389,7 +389,7 @@ mod tests { )]; // when - let out = merkle_root::(data); + let out = merkle_root::(data); // then assert_eq!( @@ -408,7 +408,7 @@ mod tests { ]; // when - let out = merkle_root::(data); + let out = merkle_root::(data); // then assert_eq!( @@ -422,7 +422,7 @@ mod tests { let _ = env_logger::try_init(); let test = |root, data| { assert_eq!( - array_bytes::bytes2hex("", &merkle_root::(data).as_ref()), + array_bytes::bytes2hex("", &merkle_root::(data).as_ref()), root ); }; @@ -455,8 +455,8 @@ mod tests { let data = vec!["a", "b", "c"]; // when - let proof0 = merkle_proof::(data.clone(), 0); - assert!(verify_proof::( + let proof0 = merkle_proof::(data.clone(), 0); + assert!(verify_proof::( &proof0.root, proof0.proof.clone(), data.len(), @@ -464,8 +464,8 @@ mod tests { &proof0.leaf, )); - let proof1 = merkle_proof::(data.clone(), 1); - assert!(verify_proof::( + let proof1 = merkle_proof::(data.clone(), 1); + assert!(verify_proof::( &proof1.root, proof1.proof, data.len(), @@ -473,8 +473,8 @@ mod tests { &proof1.leaf, )); - let proof2 = merkle_proof::(data.clone(), 2); - assert!(verify_proof::( + let proof2 = merkle_proof::(data.clone(), 2); + assert!(verify_proof::( &proof2.root, proof2.proof, data.len(), @@ -492,7 +492,7 @@ mod tests { array_bytes::bytes2hex("", &proof1.root.as_ref()) ); - assert!(!verify_proof::( + assert!(!verify_proof::( &array_bytes::hex2array_unchecked( "fb3b3be94be9e983ba5e094c9c51a7d96a4fa2e5d8e891df00ca89ba05bb1239" ) @@ -503,7 +503,7 @@ mod tests { &proof0.leaf )); - assert!(!verify_proof::( + assert!(!verify_proof::( &proof0.root.into(), vec![], data.len(), @@ -520,9 +520,9 @@ mod tests { for l in 0..data.len() { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -546,9 +546,9 @@ mod tests { for l in 0..data.len() { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -570,9 +570,9 @@ mod tests { for l in (0..data.len()).step_by(13) { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -586,7 +586,7 @@ mod tests { #[should_panic] fn should_panic_on_invalid_leaf_index() { let _ = env_logger::try_init(); - merkle_proof::(vec!["a"], 5); + merkle_proof::(vec!["a"], 5); } #[test] @@ -772,7 +772,7 @@ mod tests { for l in 0..data.len() { // when - let proof = merkle_proof::(data.clone(), l); + let proof = merkle_proof::(data.clone(), l); assert_eq!( array_bytes::bytes2hex("", &proof.root.as_ref()), array_bytes::bytes2hex("", &root.as_ref()) @@ -781,7 +781,7 @@ mod tests { assert_eq!(&proof.leaf, &data[l]); // then - assert!(verify_proof::( + assert!(verify_proof::( &proof.root, proof.proof, data.len(), @@ -790,7 +790,7 @@ mod tests { )); } - let proof = merkle_proof::(data.clone(), data.len() - 1); + let proof = merkle_proof::(data.clone(), data.len() - 1); assert_eq!( proof, diff --git a/frame/beefy-mmr/src/lib.rs b/frame/beefy-mmr/src/lib.rs index 52cba4ac3571e..5b82c89ce84b6 100644 --- a/frame/beefy-mmr/src/lib.rs +++ b/frame/beefy-mmr/src/lib.rs @@ -204,7 +204,7 @@ impl Pallet { .map(T::BeefyAuthorityToMerkleLeaf::convert) .collect::>(); let len = beefy_addresses.len() as u32; - let root = beefy_merkle_tree::merkle_root::<::Hashing, _, _, _>( + let root = beefy_merkle_tree::merkle_root::<::Hashing, _>( beefy_addresses, ) .into(); diff --git a/frame/beefy-mmr/src/mock.rs b/frame/beefy-mmr/src/mock.rs index 58d46d56f9559..0a64ad3fc9976 100644 --- a/frame/beefy-mmr/src/mock.rs +++ b/frame/beefy-mmr/src/mock.rs @@ -147,7 +147,7 @@ impl BeefyDataProvider> for DummyDataProvider { fn extra_data() -> Vec { let mut col = vec![(15, vec![1, 2, 3]), (5, vec![4, 5, 6])]; col.sort(); - beefy_merkle_tree::merkle_root::<::Hashing, _, _, _>( + beefy_merkle_tree::merkle_root::<::Hashing, _>( col.into_iter().map(|pair| pair.encode()), ) .as_ref() From e1afc0ea9ac830026bd2d57b72612e4c636d4f6a Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Tue, 4 Oct 2022 10:06:41 +0300 Subject: [PATCH 7/8] fix --- frame/beefy-mmr/primitives/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 4872ad524ff5d..9fc3d355ddb7b 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -60,7 +60,7 @@ where V: Visitor, I: Iterator, { - let upper = Vec::with_capacity(leaves.size_hint().1.unwrap()); + let upper = Vec::with_capacity(leaves.size_hint().0); let mut next = match merkelize_row::(leaves, upper, visitor) { Ok(root) => return root, Err(next) if next.is_empty() => return H::Output::default(), From 811ae444ad68fa96699ba2dc29b8fdceaf6edb9a Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Tue, 4 Oct 2022 10:26:26 +0300 Subject: [PATCH 8/8] compute upper Vec capacity more accurately --- frame/beefy-mmr/primitives/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 9fc3d355ddb7b..f56be8bcafe5b 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -60,14 +60,14 @@ where V: Visitor, I: Iterator, { - let upper = Vec::with_capacity(leaves.size_hint().0); + let upper = Vec::with_capacity((leaves.size_hint().1.unwrap_or(0).saturating_add(1)) / 2); let mut next = match merkelize_row::(leaves, upper, visitor) { Ok(root) => return root, Err(next) if next.is_empty() => return H::Output::default(), Err(next) => next, }; - let mut upper = Vec::with_capacity((next.len() + 1) / 2); + let mut upper = Vec::with_capacity((next.len().saturating_add(1)) / 2); loop { visitor.move_up();