Skip to content

Commit

Permalink
ledger-tool: Condense ProcessOptions parsing logic (solana-labs#34694)
Browse files Browse the repository at this point in the history 
The code to parse process options was repeated in several (8) commands
that replay block processing. So, move the argument parsing into a
common helper that can be used by all of those commands.
  • Loading branch information
steviez authored Jan 11, 2024
1 parent 8e29436 commit a203f14
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 148 deletions.
66 changes: 63 additions & 3 deletions ledger-tool/src/args.rs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,76 @@
use {
crate::LEDGER_TOOL_DIRECTORY,
clap::{value_t, values_t, values_t_or_exit, ArgMatches},
clap::{value_t, value_t_or_exit, values_t, values_t_or_exit, ArgMatches},
solana_accounts_db::{
accounts_db::{AccountsDb, AccountsDbConfig},
accounts_index::{AccountsIndexConfig, IndexLimitMb},
partitioned_rewards::TestPartitionedEpochRewards,
},
solana_runtime::snapshot_utils,
solana_clap_utils::input_parsers::pubkeys_of,
solana_ledger::{
blockstore_processor::ProcessOptions,
use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
},
solana_runtime::{runtime_config::RuntimeConfig, snapshot_utils},
solana_sdk::clock::Slot,
std::path::{Path, PathBuf},
std::{
collections::HashSet,
path::{Path, PathBuf},
sync::Arc,
},
};

/// Parse a `ProcessOptions` from subcommand arguments. This function attempts
/// to parse all flags related to `ProcessOptions`; however, subcommands that
/// use this function may not support all flags.
pub fn parse_process_options(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> ProcessOptions {
let new_hard_forks = hardforks_of(arg_matches, "hard_forks");
let accounts_db_config = Some(get_accounts_db_config(ledger_path, arg_matches));
let runtime_config = RuntimeConfig::default();

if arg_matches.is_present("skip_poh_verify") {
eprintln!("--skip-poh-verify is deprecated. Replace with --skip-verification.");
}
let run_verification =
!(arg_matches.is_present("skip_poh_verify") || arg_matches.is_present("skip_verification"));
let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
let use_snapshot_archives_at_startup = value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
);
let accounts_db_skip_shrink = arg_matches.is_present("accounts_db_skip_shrink");
let accounts_db_test_hash_calculation =
arg_matches.is_present("accounts_db_test_hash_calculation");
let verify_index = arg_matches.is_present("verify_accounts_index");
let limit_load_slot_count_from_snapshot =
value_t!(arg_matches, "limit_load_slot_count_from_snapshot", usize).ok();
let on_halt_store_hash_raw_data_for_debug =
arg_matches.is_present("halt_at_slot_store_hash_raw_data");
let run_final_accounts_hash_calc = arg_matches.is_present("run_final_hash_calc");
let debug_keys = pubkeys_of(arg_matches, "debug_key")
.map(|pubkeys| Arc::new(pubkeys.into_iter().collect::<HashSet<_>>()));
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");

ProcessOptions {
new_hard_forks,
runtime_config,
accounts_db_config,
accounts_db_skip_shrink,
accounts_db_test_hash_calculation,
verify_index,
limit_load_slot_count_from_snapshot,
on_halt_store_hash_raw_data_for_debug,
run_final_accounts_hash_calc,
debug_keys,
run_verification,
allow_dead_slots,
halt_at_slot,
use_snapshot_archives_at_startup,
..ProcessOptions::default()
}
}

// Build an `AccountsDbConfig` from subcommand arguments. All of the arguments
// matched by this functional are either optional or have a default value.
// Thus, a subcommand need not support all of the arguments that are matched
Expand Down
10 changes: 10 additions & 0 deletions ledger-tool/src/ledger_utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use {
blockstore_processor::{
self, BlockstoreProcessorError, ProcessOptions, TransactionStatusSender,
},
use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup,
},
solana_measure::measure,
solana_rpc::transaction_status_service::TransactionStatusService,
Expand Down Expand Up @@ -564,3 +565,12 @@ pub fn get_program_ids(tx: &VersionedTransaction) -> impl Iterator<Item = &Pubke
.iter()
.map(|ix| ix.program_id(account_keys))
}

/// Get the AccessType required, based on `process_options`
pub(crate) fn get_access_type(process_options: &ProcessOptions) -> AccessType {
match process_options.use_snapshot_archives_at_startup {
UseSnapshotArchivesAtStartup::Always => AccessType::Secondary,
UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance,
UseSnapshotArchivesAtStartup::WhenNewest => AccessType::PrimaryForMaintenance,
}
}
136 changes: 18 additions & 118 deletions ledger-tool/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,12 @@ use {
solana_ledger::{
blockstore::{create_new_ledger, Blockstore},
blockstore_options::{AccessType, LedgerColumnOptions},
blockstore_processor::ProcessOptions,
use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
use_snapshot_archives_at_startup,
},
solana_measure::{measure, measure::Measure},
solana_runtime::{
bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats},
bank_forks::BankForks,
runtime_config::RuntimeConfig,
snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_bank_utils,
snapshot_minimizer::SnapshotMinimizer,
Expand Down Expand Up @@ -561,15 +559,6 @@ fn assert_capitalization(bank: &Bank) {
assert!(bank.calculate_and_verify_capitalization(debug_verify));
}

/// Get the AccessType required, based on `process_options`
fn get_access_type(process_options: &ProcessOptions) -> AccessType {
match process_options.use_snapshot_archives_at_startup {
UseSnapshotArchivesAtStartup::Always => AccessType::Secondary,
UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance,
UseSnapshotArchivesAtStartup::WhenNewest => AccessType::PrimaryForMaintenance,
}
}

#[cfg(not(target_env = "msvc"))]
use jemallocator::Jemalloc;

Expand Down Expand Up @@ -969,7 +958,8 @@ fn main() {
.arg(&disable_disk_index)
.arg(&accountsdb_verify_refcounts)
.arg(&accounts_db_skip_initial_hash_calc_arg)
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash),
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash)
.arg(&use_snapshot_archives_at_startup),
)
.subcommand(
SubCommand::with_name("bank-hash")
Expand All @@ -981,7 +971,8 @@ fn main() {
.arg(&disable_disk_index)
.arg(&accountsdb_verify_refcounts)
.arg(&accounts_db_skip_initial_hash_calc_arg)
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash),
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash)
.arg(&use_snapshot_archives_at_startup),
)
.subcommand(
SubCommand::with_name("verify")
Expand Down Expand Up @@ -1555,13 +1546,12 @@ fn main() {
println!("{}", open_genesis_config_by(&output_directory, arg_matches));
}
("shred-version", Some(arg_matches)) => {
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot: Some(0),
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
..ProcessOptions::default()
};
let mut process_options = parse_process_options(&ledger_path, arg_matches);
// Respect a user-set --halt-at-slot; otherwise, set Some(0) to avoid
// processing any additional banks and just use the snapshot bank
if process_options.halt_at_slot.is_none() {
process_options.halt_at_slot = Some(0);
}
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
Expand All @@ -1586,13 +1576,7 @@ fn main() {
);
}
("bank-hash", Some(arg_matches)) => {
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
Expand Down Expand Up @@ -1623,44 +1607,7 @@ fn main() {
},
);

let debug_keys = pubkeys_of(arg_matches, "debug_key")
.map(|pubkeys| Arc::new(pubkeys.into_iter().collect::<HashSet<_>>()));

if arg_matches.is_present("skip_poh_verify") {
eprintln!(
"--skip-poh-verify is deprecated. Replace with --skip-verification."
);
}

let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
run_verification: !(arg_matches.is_present("skip_poh_verify")
|| arg_matches.is_present("skip_verification")),
on_halt_store_hash_raw_data_for_debug: arg_matches
.is_present("halt_at_slot_store_hash_raw_data"),
run_final_accounts_hash_calc: arg_matches.is_present("run_final_hash_calc"),
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
debug_keys,
limit_load_slot_count_from_snapshot: value_t!(
arg_matches,
"limit_load_slot_count_from_snapshot",
usize
)
.ok(),
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
verify_index: arg_matches.is_present("verify_accounts_index"),
allow_dead_slots: arg_matches.is_present("allow_dead_slots"),
accounts_db_test_hash_calculation: arg_matches
.is_present("accounts_db_test_hash_calculation"),
accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"),
runtime_config: RuntimeConfig::default(),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
let write_bank_file = arg_matches.is_present("write_bank_file");
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
Expand Down Expand Up @@ -1706,19 +1653,7 @@ fn main() {
),
};

let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};

let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
Expand Down Expand Up @@ -1771,7 +1706,6 @@ fn main() {
});
let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts");
let new_hard_forks = hardforks_of(arg_matches, "hard_forks");

let faucet_pubkey = pubkey_of(arg_matches, "faucet_pubkey");
let faucet_lamports =
Expand Down Expand Up @@ -1834,18 +1768,8 @@ fn main() {
NonZeroUsize
);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let mut process_options = ProcessOptions {
new_hard_forks,
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let mut process_options = parse_process_options(&ledger_path, arg_matches);

let blockstore = Arc::new(open_blockstore(
&ledger_path,
arg_matches,
Expand Down Expand Up @@ -2266,19 +2190,7 @@ fn main() {
);
}
("accounts", Some(arg_matches)) => {
let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot,
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
let blockstore = open_blockstore(
Expand Down Expand Up @@ -2360,19 +2272,7 @@ fn main() {
}
}
("capitalization", Some(arg_matches)) => {
let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot,
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
Expand Down
Loading

0 comments on commit a203f14

Please sign in to comment.