Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

adds chained_merkle_root to shredder arguments #34952

Merged
merged 1 commit into from
Jan 27, 2024
Merged

adds chained_merkle_root to shredder arguments #34952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion core/benches/shredder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
extern crate test;

use {
rand::seq::SliceRandom,
rand::{seq::SliceRandom, Rng},
raptorq::{Decoder, Encoder},
solana_entry::entry::{create_ticks, Entry},
solana_ledger::shred::{
Expand Down Expand Up @@ -50,6 +50,7 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
&Keypair::new(),
&entries,
true, // is_last_in_slot
None, // chained_merkle_root
0, // next_shred_index
0, // next_code_index
false, // merkle_variant
Expand Down Expand Up @@ -80,12 +81,14 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
let num_ticks = max_ticks_per_n_shreds(1, Some(LEGACY_SHRED_DATA_CAPACITY)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
let reed_solomon_cache = ReedSolomonCache::default();
let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
bencher.iter(|| {
let shredder = Shredder::new(1, 0, 0, 0).unwrap();
shredder.entries_to_shreds(
&kp,
&entries,
true,
chained_merkle_root,
0,
0,
true, // merkle_variant
Expand All @@ -107,6 +110,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
Some(shred_size),
);
let entries = make_large_unchained_entries(txs_per_entry, num_entries);
let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
let reed_solomon_cache = ReedSolomonCache::default();
// 1Mb
bencher.iter(|| {
Expand All @@ -115,6 +119,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
&kp,
&entries,
true,
chained_merkle_root,
0,
0,
true, // merkle_variant
Expand All @@ -133,10 +138,12 @@ fn bench_deshredder(bencher: &mut Bencher) {
let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
let shredder = Shredder::new(1, 0, 0, 0).unwrap();
let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
let (data_shreds, _) = shredder.entries_to_shreds(
&kp,
&entries,
true,
chained_merkle_root,
0,
0,
true, // merkle_variant
Expand Down
1 change: 1 addition & 0 deletions core/src/window_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,7 @@ mod test {
keypair,
entries,
true, // is_last_in_slot
None, // chained_merkle_root
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
Expand Down
2 changes: 2 additions & 0 deletions gossip/src/duplicate_shred.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,8 @@ pub(crate) mod tests {
keypair,
&entries,
is_last_in_slot,
// chained_merkle_root
Some(Hash::new_from_array(rng.gen())),
next_shred_index,
next_code_index, // next_code_index
merkle_variant,
Expand Down
41 changes: 27 additions & 14 deletions ledger/src/blockstore.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ use {
crossbeam_channel::{bounded, Receiver, Sender, TrySendError},
dashmap::DashSet,
log::*,
rand::Rng,
rayon::{
iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator},
ThreadPool,
Expand Down Expand Up @@ -1956,6 +1957,7 @@ impl Blockstore {
let mut all_shreds = vec![];
let mut slot_entries = vec![];
let reed_solomon_cache = ReedSolomonCache::default();
let mut chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen()));
// Find all the entries for start_slot
for entry in entries.into_iter() {
if remaining_ticks_in_slot == 0 {
Expand All @@ -1973,7 +1975,8 @@ impl Blockstore {
let (mut data_shreds, mut coding_shreds) = shredder.entries_to_shreds(
keypair,
&current_entries,
true, // is_last_in_slot
true, // is_last_in_slot
chained_merkle_root,
start_index, // next_shred_index
start_index, // next_code_index
true, // merkle_variant
Expand All @@ -1982,6 +1985,7 @@ impl Blockstore {
);
all_shreds.append(&mut data_shreds);
all_shreds.append(&mut coding_shreds);
chained_merkle_root = Some(coding_shreds.last().unwrap().merkle_root().unwrap());
shredder = Shredder::new(
current_slot,
parent_slot,
Expand All @@ -2002,6 +2006,7 @@ impl Blockstore {
keypair,
&slot_entries,
is_full_slot,
chained_merkle_root,
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
Expand Down Expand Up @@ -4285,6 +4290,8 @@ pub fn create_new_ledger(
&Keypair::new(),
&entries,
true, // is_last_in_slot
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
Expand Down Expand Up @@ -4546,6 +4553,8 @@ pub fn entries_to_test_shreds(
&Keypair::new(),
entries,
is_full_slot,
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
0, // next_shred_index,
0, // next_code_index
merkle_variant,
Expand Down Expand Up @@ -4806,6 +4815,7 @@ pub mod tests {
InnerInstruction, InnerInstructions, Reward, Rewards, TransactionTokenBalance,
},
std::{cmp::Ordering, thread::Builder, time::Duration},
test_case::test_case,
};

// used for tests only
Expand Down Expand Up @@ -7434,7 +7444,7 @@ pub mod tests {
#[test]
fn test_insert_multiple_is_last() {
solana_logger::setup();
let (shreds, _) = make_slot_entries(0, 0, 20, /*merkle_variant:*/ true);
let (shreds, _) = make_slot_entries(0, 0, 19, /*merkle_variant:*/ true);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any reason to change this from 20 -> 19?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test tries to insert a shred where index > already recorded last_index.
Because of the embedded chained Merkle root, shreds have smaller capacity, so with 20 this line generates the same number of shreds as L7460.
So we won't see any shred with index > last_index when inserting the second set of shreds.

So I reduced this to 19 so that the added assertion in L7461 still holds as before.

let num_shreds = shreds.len() as u64;
let ledger_path = get_tmp_ledger_path_auto_delete!();
let blockstore = Blockstore::open(ledger_path.path()).unwrap();
Expand All @@ -7448,6 +7458,7 @@ pub mod tests {
assert!(slot_meta.is_full());

let (shreds, _) = make_slot_entries(0, 0, 22, /*merkle_variant:*/ true);
assert!(shreds.len() > num_shreds as usize);
blockstore.insert_shreds(shreds, None, false).unwrap();
let slot_meta = blockstore.meta(0).unwrap().unwrap();

Expand Down Expand Up @@ -9863,7 +9874,9 @@ pub mod tests {
let (data_shreds, coding_shreds) = shredder.entries_to_shreds(
&leader_keypair,
&entries,
true, // is_last_in_slot
true, // is_last_in_slot
// chained_merkle_root
Some(Hash::new_from_array(rand::thread_rng().gen())),
fec_set_index, // next_shred_index
fec_set_index, // next_code_index
true, // merkle_variant
Expand Down Expand Up @@ -9916,18 +9929,21 @@ pub mod tests {
assert_eq!(num_coding_in_index, num_coding);
}

#[test]
fn test_duplicate_slot() {
#[test_case(false)]
#[test_case(true)]
fn test_duplicate_slot(chained: bool) {
let slot = 0;
let entries1 = make_slot_entries_with_transactions(1);
let entries2 = make_slot_entries_with_transactions(1);
let leader_keypair = Arc::new(Keypair::new());
let reed_solomon_cache = ReedSolomonCache::default();
let shredder = Shredder::new(slot, 0, 0, 0).unwrap();
let chained_merkle_root = chained.then(|| Hash::new_from_array(rand::thread_rng().gen()));
let (shreds, _) = shredder.entries_to_shreds(
&leader_keypair,
&entries1,
true, // is_last_in_slot
chained_merkle_root,
0, // next_shred_index
0, // next_code_index,
true, // merkle_variant
Expand All @@ -9938,6 +9954,7 @@ pub mod tests {
&leader_keypair,
&entries2,
true, // is_last_in_slot
chained_merkle_root,
0, // next_shred_index
0, // next_code_index
true, // merkle_variant
Expand Down Expand Up @@ -10323,7 +10340,11 @@ pub mod tests {
let num_unique_entries = max_ticks_per_n_shreds(1, None) + 1;
let (mut original_shreds, original_entries) =
make_slot_entries(0, 0, num_unique_entries, /*merkle_variant:*/ true);

let mut duplicate_shreds = original_shreds.clone();
// Mutate signature so that payloads are not the same as the originals.
for shred in &mut duplicate_shreds {
shred.sign(&Keypair::new());
}
// Discard first shred, so that the slot is not full
assert!(original_shreds.len() > 1);
let last_index = original_shreds.last().unwrap().index() as u64;
Expand All @@ -10345,14 +10366,6 @@ pub mod tests {
assert!(!blockstore.is_full(0));
}

let duplicate_shreds = entries_to_test_shreds(
&original_entries,
0, // slot
0, // parent_slot
true, // is_full_slot
0, // version
true, // merkle_variant
);
let num_shreds = duplicate_shreds.len() as u64;
blockstore
.insert_shreds(duplicate_shreds, None, false)
Expand Down
Loading
Loading