From a9cb121a22c09841ef1c7100f6554f2e9e04bbd5 Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Fri, 6 Dec 2024 23:44:24 -0500 Subject: [PATCH 1/2] blockstore: mark slot as dead on data shred merkle root conflict (#3970) (cherry picked from commit 5564a941ae65d6a43d8a4d97936fdc09792f92a8) # Conflicts: # ledger/src/blockstore.rs --- ledger/src/blockstore.rs | 63 +++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 07d9a6d0cb5404..9d942e7886a531 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -1692,6 +1692,15 @@ impl Blockstore { &shred, duplicate_shreds, ) { + // This indicates there is an alternate version of this block. + // Similar to the last index case above, we might never get all the + // shreds for our current version, never replay this slot, and make no + // progress. We cannot determine if we have the version that will eventually + // be complete, so we take the conservative approach and mark the slot as dead + // so that replay can dump and repair the correct version. + self.dead_slots_cf + .put_in_batch(write_batch, slot, &true) + .unwrap(); return Err(InsertDataShredError::InvalidShred); } } @@ -5950,21 +5959,6 @@ pub mod tests { .insert_shreds(shreds, None, false) .expect("Expected successful write of shreds"); - let mut shreds1 = entries_to_test_shreds( - &entries[4..], - 1, - 0, - false, - 0, - false, // merkle_variant - ); - for (i, b) in shreds1.iter_mut().enumerate() { - b.set_index(8 + i as u32); - } - blockstore - .insert_shreds(shreds1, None, false) - .expect("Expected successful write of shreds"); - assert_eq!( blockstore.get_slot_entries(1, 0).unwrap()[2..4], entries[2..4], @@ -7830,9 +7824,21 @@ pub mod tests { ShredSource::Turbine, ) .is_err()); +<<<<<<< HEAD // No insert, notify duplicate assert_eq!(duplicates.len(), 1); +======= + let ShredInsertionTracker { + merkle_root_metas, + duplicate_shreds, + write_batch, + .. + } = shred_insertion_tracker; + + // No insert, notify duplicate, and block is dead + assert_eq!(duplicate_shreds.len(), 1); +>>>>>>> 5564a941ae (blockstore: mark slot as dead on data shred merkle root conflict (#3970)) assert_matches!( duplicates[0], PossibleDuplicateShred::MerkleRootConflict(_, _) @@ -7857,6 +7863,11 @@ pub mod tests { index ); + // Block is now dead + blockstore.db.write(write_batch).unwrap(); + assert!(blockstore.is_dead(slot)); + blockstore.remove_dead_slot(slot).unwrap(); + // Blockstore should also have the merkle root meta of the original shred assert_eq!( blockstore @@ -7888,6 +7899,8 @@ pub mod tests { fec_set_index + 30, ); + let mut shred_insertion_tracker = + ShredInsertionTracker::new(data_shreds.len(), blockstore.db.batch().unwrap()); blockstore .check_insert_data_shred( new_data_shred.clone(), @@ -7904,23 +7917,33 @@ pub mod tests { ShredSource::Turbine, ) .unwrap(); +<<<<<<< HEAD +======= + let ShredInsertionTracker { + merkle_root_metas, + write_batch, + .. + } = shred_insertion_tracker; + blockstore.db.write(write_batch).unwrap(); +>>>>>>> 5564a941ae (blockstore: mark slot as dead on data shred merkle root conflict (#3970)) // Verify that we still have the merkle root meta for the original shred // and the new shred - assert_eq!(merkle_root_metas.len(), 2); assert_eq!( - merkle_root_metas - .get(&data_shred.erasure_set()) + blockstore + .merkle_root_meta(data_shred.erasure_set()) .unwrap() .as_ref() + .unwrap() .merkle_root(), data_shred.merkle_root().ok() ); assert_eq!( - merkle_root_metas - .get(&data_shred.erasure_set()) + blockstore + .merkle_root_meta(data_shred.erasure_set()) .unwrap() .as_ref() + .unwrap() .first_received_shred_index(), index ); From 1cb9da8a75a659e40b816686e4cf523987ea819f Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Thu, 12 Dec 2024 17:36:23 +0000 Subject: [PATCH 2/2] fix conflicts --- ledger/src/blockstore.rs | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 9d942e7886a531..d63e60789290c5 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -1698,9 +1698,7 @@ impl Blockstore { // progress. We cannot determine if we have the version that will eventually // be complete, so we take the conservative approach and mark the slot as dead // so that replay can dump and repair the correct version. - self.dead_slots_cf - .put_in_batch(write_batch, slot, &true) - .unwrap(); + write_batch.put::(slot, &true).unwrap(); return Err(InsertDataShredError::InvalidShred); } } @@ -7824,21 +7822,9 @@ pub mod tests { ShredSource::Turbine, ) .is_err()); -<<<<<<< HEAD // No insert, notify duplicate assert_eq!(duplicates.len(), 1); -======= - let ShredInsertionTracker { - merkle_root_metas, - duplicate_shreds, - write_batch, - .. - } = shred_insertion_tracker; - - // No insert, notify duplicate, and block is dead - assert_eq!(duplicate_shreds.len(), 1); ->>>>>>> 5564a941ae (blockstore: mark slot as dead on data shred merkle root conflict (#3970)) assert_matches!( duplicates[0], PossibleDuplicateShred::MerkleRootConflict(_, _) @@ -7899,8 +7885,7 @@ pub mod tests { fec_set_index + 30, ); - let mut shred_insertion_tracker = - ShredInsertionTracker::new(data_shreds.len(), blockstore.db.batch().unwrap()); + let mut write_batch = blockstore.db.batch().unwrap(); blockstore .check_insert_data_shred( new_data_shred.clone(), @@ -7917,15 +7902,7 @@ pub mod tests { ShredSource::Turbine, ) .unwrap(); -<<<<<<< HEAD -======= - let ShredInsertionTracker { - merkle_root_metas, - write_batch, - .. - } = shred_insertion_tracker; blockstore.db.write(write_batch).unwrap(); ->>>>>>> 5564a941ae (blockstore: mark slot as dead on data shred merkle root conflict (#3970)) // Verify that we still have the merkle root meta for the original shred // and the new shred