Skip to content

Commit

Permalink
re-factor: cleanup snapshot config usage
Browse files Browse the repository at this point in the history 
resolve call sites

more cleanup

fix up tests

cleanup

more cleanup

fix compile issue

fix waring

minor cleanup
  • Loading branch information
@JoeySlomowitz
JoeySlomowitz committed Jan 25, 2025
1 parent 8e1c067 commit 7ba47c0
Show file tree
Hide file tree
Showing 25 changed files with 1,160 additions and 671 deletions.
34 changes: 17 additions & 17 deletions core/src/accounts_hash_verifier.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use {
solana_measure::measure_us,
solana_runtime::{
serde_snapshot::BankIncrementalSnapshotPersistence,
snapshot_config::SnapshotConfig,
snapshot_mode::SnapshotMode,
snapshot_package::{
self, AccountsHashAlgorithm, AccountsPackage, AccountsPackageKind, SnapshotKind,
SnapshotPackage,
Expand Down Expand Up @@ -43,7 +43,7 @@ impl AccountsHashVerifier {
accounts_package_receiver: Receiver<AccountsPackage>,
pending_snapshot_packages: Arc<Mutex<PendingSnapshotPackages>>,
exit: Arc<AtomicBool>,
snapshot_config: SnapshotConfig,
snapshot_mode: SnapshotMode,
) -> Self {
// If there are no accounts packages to process, limit how often we re-check
const LOOP_LIMITER: Duration = Duration::from_millis(DEFAULT_MS_PER_SLOT);
Expand Down Expand Up @@ -74,7 +74,7 @@ impl AccountsHashVerifier {
let (result, handling_time_us) = measure_us!(Self::process_accounts_package(
accounts_package,
&pending_snapshot_packages,
&snapshot_config,
&snapshot_mode,
));
if let Err(err) = result {
error!(
Expand Down Expand Up @@ -214,19 +214,19 @@ impl AccountsHashVerifier {
fn process_accounts_package(
accounts_package: AccountsPackage,
pending_snapshot_packages: &Mutex<PendingSnapshotPackages>,
snapshot_config: &SnapshotConfig,
snapshot_mode: &SnapshotMode,
) -> IoResult<()> {
let (merkle_or_lattice_accounts_hash, bank_incremental_snapshot_persistence) =
Self::calculate_and_verify_accounts_hash(&accounts_package, snapshot_config)?;
Self::calculate_and_verify_accounts_hash(&accounts_package, snapshot_mode)?;

Self::save_epoch_accounts_hash(&accounts_package, &merkle_or_lattice_accounts_hash);

Self::purge_old_accounts_hashes(&accounts_package, snapshot_config);
Self::purge_old_accounts_hashes(&accounts_package, snapshot_mode);

Self::submit_for_packaging(
accounts_package,
pending_snapshot_packages,
snapshot_config,
snapshot_mode,
merkle_or_lattice_accounts_hash,
bank_incremental_snapshot_persistence,
);
Expand All @@ -237,7 +237,7 @@ impl AccountsHashVerifier {
/// returns calculated accounts hash
fn calculate_and_verify_accounts_hash(
accounts_package: &AccountsPackage,
snapshot_config: &SnapshotConfig,
snapshot_mode: &SnapshotMode,
) -> IoResult<(
MerkleOrLatticeAccountsHash,
Option<BankIncrementalSnapshotPersistence>,
Expand Down Expand Up @@ -284,6 +284,7 @@ impl AccountsHashVerifier {
let Some((base_accounts_hash, base_capitalization)) =
accounts_db.get_accounts_hash(base_slot)
else {
let snapshot_load_config = &snapshot_mode.get_snapshot_load_config();
panic!(
"incremental snapshot requires accounts hash and capitalization from \
the full snapshot it is based on\n\
Expand All @@ -295,9 +296,11 @@ impl AccountsHashVerifier {
accounts_db.get_accounts_hashes(),
accounts_db.get_incremental_accounts_hashes(),
snapshot_utils::get_full_snapshot_archives(
&snapshot_config.full_snapshot_archives_dir,
&snapshot_load_config.full_snapshot_config.archives_dir,
),
snapshot_utils::get_bank_snapshots(
&snapshot_load_config.bank_snapshots_dir
),
snapshot_utils::get_bank_snapshots(&snapshot_config.bank_snapshots_dir),
);
};
let (incremental_accounts_hash, incremental_capitalization) =
Expand Down Expand Up @@ -455,12 +458,9 @@ impl AccountsHashVerifier {
}
}

fn purge_old_accounts_hashes(
accounts_package: &AccountsPackage,
snapshot_config: &SnapshotConfig,
) {
fn purge_old_accounts_hashes(accounts_package: &AccountsPackage, snapshot_mode: &SnapshotMode) {
let should_purge = match (
snapshot_config.should_generate_snapshots(),
snapshot_mode.should_generate_snapshots(),
accounts_package.package_kind,
) {
(false, _) => {
Expand Down Expand Up @@ -489,11 +489,11 @@ impl AccountsHashVerifier {
fn submit_for_packaging(
accounts_package: AccountsPackage,
pending_snapshot_packages: &Mutex<PendingSnapshotPackages>,
snapshot_config: &SnapshotConfig,
snapshot_mode: &SnapshotMode,
merkle_or_lattice_accounts_hash: MerkleOrLatticeAccountsHash,
bank_incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
) {
if !snapshot_config.should_generate_snapshots()
if !snapshot_mode.should_generate_snapshots()
|| !matches!(
accounts_package.package_kind,
AccountsPackageKind::Snapshot(_)
Expand Down
28 changes: 19 additions & 9 deletions core/src/snapshot_packager_service.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use {
solana_measure::{measure::Measure, measure_us},
solana_perf::thread::renice_this_thread,
solana_runtime::{
snapshot_config::SnapshotConfig, snapshot_hash::StartingSnapshotHashes,
snapshot_hash::StartingSnapshotHashes, snapshot_mode::SnapshotMode,
snapshot_package::SnapshotPackage, snapshot_utils,
},
std::{
Expand All @@ -33,14 +33,20 @@ impl SnapshotPackagerService {
starting_snapshot_hashes: Option<StartingSnapshotHashes>,
exit: Arc<AtomicBool>,
cluster_info: Arc<ClusterInfo>,
snapshot_config: SnapshotConfig,
snapshot_mode: SnapshotMode,
enable_gossip_push: bool,
) -> Self {
let t_snapshot_packager = Builder::new()
.name("solSnapshotPkgr".to_string())
.spawn(move || {
info!("SnapshotPackagerService has started");
renice_this_thread(snapshot_config.packager_thread_niceness_adj).unwrap();
renice_this_thread(
snapshot_mode
.get_snapshot_generate_config()
.unwrap()
.packager_thread_niceness_adj,
)
.unwrap();
let mut snapshot_gossip_manager = enable_gossip_push
.then(|| SnapshotGossipManager::new(cluster_info, starting_snapshot_hashes));

Expand Down Expand Up @@ -69,7 +75,7 @@ impl SnapshotPackagerService {
let (archive_result, archive_time_us) =
measure_us!(snapshot_utils::serialize_and_archive_snapshot_package(
snapshot_package,
&snapshot_config,
&snapshot_mode,
));
if let Err(err) = archive_result {
error!(
Expand All @@ -85,12 +91,16 @@ impl SnapshotPackagerService {
.push_snapshot_hash(snapshot_kind, (snapshot_slot, snapshot_hash));
}

let snapshot_load_config = snapshot_mode.get_snapshot_load_config().clone();
let incremental_snapshot_config =
snapshot_load_config.incremental_snapshot_config.unwrap();

let (_, purge_archives_time_us) =
measure_us!(snapshot_utils::purge_old_snapshot_archives(
&snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
snapshot_config.maximum_full_snapshot_archives_to_retain,
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
&snapshot_load_config.full_snapshot_config.archives_dir,
&incremental_snapshot_config.archives_dir, // FIXME: Should this param be optional too?
snapshot_load_config.full_snapshot_config.archives_to_retain,
incremental_snapshot_config.archives_to_retain,
));

// Now that this snapshot package has been archived, it is safe to remove
Expand All @@ -99,7 +109,7 @@ impl SnapshotPackagerService {
// booting from local state.
let (_, purge_bank_snapshots_time_us) =
measure_us!(snapshot_utils::purge_bank_snapshots_older_than_slot(
&snapshot_config.bank_snapshots_dir,
&snapshot_load_config.bank_snapshots_dir,
snapshot_slot,
));

Expand Down
Loading

0 comments on commit 7ba47c0

Please sign in to comment.