From b619098b760ff8d6c4c0070bc29103310ae1aba8 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Mon, 10 Oct 2022 23:14:17 -0500 Subject: [PATCH 01/14] improve readability --- tip-distributor/src/lib.rs | 81 +++++++++++-------- .../src/merkle_root_generator_workflow.rs | 16 ++-- .../src/stake_meta_generator_workflow.rs | 40 ++++----- 3 files changed, 75 insertions(+), 62 deletions(-) diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index 6e29ca132c..bba089889c 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -16,7 +16,6 @@ use { solana_runtime::bank::Bank, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, - bs58, clock::Slot, hash::{Hash, Hasher}, pubkey::Pubkey, @@ -52,8 +51,6 @@ impl GeneratedMerkleTreeCollection { stake_meta_coll: StakeMetaCollection, merkle_root_upload_authority: Pubkey, ) -> Result { - let b58_merkle_root_upload_authority = - bs58::encode(merkle_root_upload_authority.as_ref()).into_string(); let generated_merkle_trees = stake_meta_coll .stake_metas .into_iter() @@ -61,7 +58,7 @@ impl GeneratedMerkleTreeCollection { if let Some(tip_distribution_meta) = stake_meta.maybe_tip_distribution_meta.as_ref() { tip_distribution_meta.merkle_root_upload_authority - == b58_merkle_root_upload_authority + == merkle_root_upload_authority } else { false } @@ -79,12 +76,6 @@ impl GeneratedMerkleTreeCollection { let tip_distribution_meta = stake_meta.maybe_tip_distribution_meta.unwrap(); - let tip_distribution_account = - match tip_distribution_meta.tip_distribution_account.parse() { - Err(_) => return Some(Err(Error::Base58DecodeError)), - Ok(tip_distribution_account) => tip_distribution_account, - }; - let merkle_tree = MerkleTree::new(&hashed_nodes[..], true); let max_num_nodes = tree_nodes.len() as u64; @@ -94,7 +85,7 @@ impl GeneratedMerkleTreeCollection { Some(Ok(GeneratedMerkleTree { max_num_nodes, - tip_distribution_account, + tip_distribution_account: tip_distribution_meta.tip_distribution_account, merkle_tree, tree_nodes, max_total_claim: tip_distribution_meta.total_tips, @@ -140,17 +131,13 @@ pub struct TreeNode { impl TreeNode { fn vec_from_stake_meta(stake_meta: &StakeMeta) -> Result>, Error> { - let validator_vote_account = stake_meta - .validator_vote_account - .parse() - .map_err(|_| Error::Base58DecodeError)?; if let Some(tip_distribution_meta) = stake_meta.maybe_tip_distribution_meta.as_ref() { let validator_fee = calc_validator_fee( tip_distribution_meta.total_tips, tip_distribution_meta.validator_fee_bps, ); let mut tree_nodes = vec![TreeNode { - claimant: validator_vote_account, + claimant: stake_meta.validator_vote_account, amount: validator_fee, proof: None, }]; @@ -201,7 +188,7 @@ impl TreeNode { } Ok(TreeNode { - claimant: delegation.stake_account.parse().map_err(|_| Error::Base58DecodeError)?, + claimant: delegation.stake_account, amount: amount.to_u64().unwrap(), proof: None }) @@ -233,7 +220,8 @@ pub struct StakeMetaCollection { pub stake_metas: Vec, /// base58 encoded tip-distribution program id. - pub tip_distribution_program_id: String, + #[serde(with = "pubkey_string_conversion")] + pub tip_distribution_program_id: Pubkey, /// Base58 encoded bank hash this object was generated at. pub bank_hash: String, @@ -247,8 +235,8 @@ pub struct StakeMetaCollection { #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct StakeMeta { - /// The validator's base58 encoded vote account. - pub validator_vote_account: String, + #[serde(with = "pubkey_string_conversion")] + pub validator_vote_account: Pubkey, /// The validator's tip-distribution meta if it exists. pub maybe_tip_distribution_meta: Option, @@ -265,11 +253,11 @@ pub struct StakeMeta { #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct TipDistributionMeta { - /// The account authorized to generate and upload a merkle_root for the validator. - pub merkle_root_upload_authority: String, + #[serde(with = "pubkey_string_conversion")] + pub merkle_root_upload_authority: Pubkey, - /// The validator's base58 encoded [TipDistributionAccount]. - pub tip_distribution_account: String, + #[serde(with = "pubkey_string_conversion")] + pub tip_distribution_account: Pubkey, /// The validator's total tips in the [TipDistributionAccount]. pub total_tips: u64, @@ -286,8 +274,7 @@ impl TipDistributionMeta { rent_exempt_amount: u64, ) -> Result { Ok(TipDistributionMeta { - tip_distribution_account: bs58::encode(tda_wrapper.tip_distribution_account_pubkey) - .into_string(), + tip_distribution_account: tda_wrapper.tip_distribution_account_pubkey, total_tips: tda_wrapper .account_data .lamports() @@ -296,22 +283,22 @@ impl TipDistributionMeta { validator_fee_bps: tda_wrapper .tip_distribution_account .validator_commission_bps, - merkle_root_upload_authority: bs58::encode( - tda_wrapper - .tip_distribution_account - .merkle_root_upload_authority, - ) - .into_string(), + merkle_root_upload_authority: tda_wrapper + .tip_distribution_account + .merkle_root_upload_authority, }) } } #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct Delegation { - /// The stake account of interest base58 encoded. - pub stake_account: String, + #[serde(with = "pubkey_string_conversion")] + pub stake_account: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub owner: Pubkey, - /// Amount delegated by this account. + /// Lamports delegated by the stake account pub amount_delegated: u64, } @@ -476,6 +463,30 @@ mod math { } } +mod pubkey_string_conversion { + use { + serde::{self, Deserialize, Deserializer, Serializer}, + solana_sdk::pubkey::Pubkey, + std::str::FromStr, + }; + + pub fn serialize(pubkey: &Pubkey, serializer: S) -> Result + where + S: Serializer, + { + let s = format!("{}", pubkey.to_string()); + serializer.serialize_str(&s) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Pubkey::from_str(&s).map_err(serde::de::Error::custom) + } +} + #[cfg(test)] mod tests { use {super::*, solana_sdk::bs58, tip_distribution::merkle_proof}; diff --git a/tip-distributor/src/merkle_root_generator_workflow.rs b/tip-distributor/src/merkle_root_generator_workflow.rs index d5d2fb811c..77d6d38a16 100644 --- a/tip-distributor/src/merkle_root_generator_workflow.rs +++ b/tip-distributor/src/merkle_root_generator_workflow.rs @@ -71,19 +71,16 @@ pub fn run_workflow( write_to_json_file(&merkle_tree_coll, out_path)?; if should_upload_roots { - let tip_distribution_program_id = stake_meta_coll - .tip_distribution_program_id - .parse() - .map_err(|_| Error::Base58DecodeError)?; - let rpc_client = Arc::new(RpcClient::new_with_commitment( rpc_url, CommitmentConfig { commitment: CommitmentLevel::Finalized, }, )); - let (config, _) = - Pubkey::find_program_address(&[Config::SEED], &tip_distribution_program_id); + let (config, _) = Pubkey::find_program_address( + &[Config::SEED], + &stake_meta_coll.tip_distribution_program_id, + ); let recent_blockhash = { let rpc_client = rpc_client.clone(); @@ -96,7 +93,7 @@ pub fn run_workflow( upload_roots( rpc_client, merkle_tree_coll, - tip_distribution_program_id, + stake_meta_coll.tip_distribution_program_id, config, my_keypair, recent_blockhash, @@ -119,7 +116,8 @@ fn write_to_json_file( ) -> Result<(), Error> { let file = File::create(file_path)?; let mut writer = BufWriter::new(file); - serde_json::to_writer(&mut writer, merkle_tree_coll)?; + let json = serde_json::to_string_pretty(&merkle_tree_coll).unwrap(); + writer.write(json.as_bytes())?; writer.flush()?; Ok(()) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 0127e24c55..c9dccc2686 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -20,10 +20,7 @@ use { vote_account::VoteAccount, }, solana_sdk::{ - account::ReadableAccount, - bs58, - clock::{Epoch, Slot}, - pubkey::Pubkey, + account::ReadableAccount, clock::Slot, genesis_config::GenesisConfig, pubkey::Pubkey, }, std::{ collections::HashMap, @@ -127,7 +124,8 @@ fn write_to_json_file( ) -> Result<(), Error> { let file = File::create(out_path)?; let mut writer = BufWriter::new(file); - serde_json::to_writer(&mut writer, stake_meta_coll)?; + let json = serde_json::to_string_pretty(&stake_meta_coll).unwrap(); + writer.write(json.as_bytes())?; writer.flush()?; Ok(()) @@ -152,12 +150,15 @@ pub fn generate_stake_meta_collection( let l_stakes = bank.stakes_cache.stakes(); let delegations = l_stakes.stake_delegations(); - let account_fetcher = if let Some(rpc_client) = maybe_rpc_client { + // the last leader in an epoch may not crank the tip_receiver before the epoch is over. + // to get the most accurate number for a given epoch, fetch the tip distribution account + // balance from an rpc server. + let tda_account_fetcher = if let Some(rpc_client) = maybe_rpc_client { Box::new(RpcAccountFetcher { rpc_client }) as Box } else { Box::new(BankAccountFetcher { bank: bank.clone() }) as Box }; - let account_fetcher = Arc::new(account_fetcher); + let tda_account_fetcher = Arc::new(tda_account_fetcher); let vote_pk_and_maybe_tdas: Vec<( (Pubkey, &VoteAccount), @@ -166,7 +167,7 @@ pub fn generate_stake_meta_collection( .iter() .map(|(&vote_pubkey, (_total_stake, vote_account))| { let tda = fetch_and_deserialize_tip_distribution_account( - account_fetcher.clone(), + tda_account_fetcher.clone(), &vote_pubkey, &tip_distribution_program_id, bank.epoch(), @@ -177,7 +178,7 @@ pub fn generate_stake_meta_collection( }) .collect::>()?; - let voter_pubkey_to_delegations = group_delegations_by_voter_pubkey(delegations, bank.epoch()); + let voter_pubkey_to_delegations = group_delegations_by_voter_pubkey(delegations, bank); let mut stake_metas = vec![]; for ((vote_pubkey, vote_account), maybe_tda) in vote_pk_and_maybe_tdas { @@ -200,7 +201,7 @@ pub fn generate_stake_meta_collection( stake_metas.push(StakeMeta { maybe_tip_distribution_meta, - validator_vote_account: bs58::encode(vote_pubkey).into_string(), + validator_vote_account: vote_pubkey, delegations: delegations.clone(), total_delegated, commission: vote_account.vote_state().as_ref().unwrap().commission, @@ -215,8 +216,7 @@ pub fn generate_stake_meta_collection( Ok(StakeMetaCollection { stake_metas, - tip_distribution_program_id: bs58::encode(tip_distribution_program_id.as_ref()) - .into_string(), + tip_distribution_program_id, bank_hash: bank.hash().to_string(), epoch: bank.epoch(), slot: bank.slot(), @@ -226,21 +226,25 @@ pub fn generate_stake_meta_collection( /// Given an [EpochStakes] object, return delegations grouped by voter_pubkey (validator delegated to). fn group_delegations_by_voter_pubkey( delegations: &im::HashMap, - epoch: Epoch, + bank: &Bank, ) -> HashMap> { delegations .into_iter() - .filter(|(_stake_pubkey, stake_account)| stake_account.delegation().stake(epoch, None) > 0) - .into_group_map_by(|(_stake_pubkey, stake_account)| stake_account.delegation().voter_pubkey) + .filter(|(_stake_pubkey, delegation)| delegation.stake(bank.epoch(), None) > 0) + .into_group_map_by(|(_stake_pubkey, delegation)| delegation.voter_pubkey) .into_iter() .map(|(voter_pubkey, group)| { ( voter_pubkey, group .into_iter() - .map(|(stake_pubkey, stake_account)| crate::Delegation { - stake_account: bs58::encode(stake_pubkey).into_string(), - amount_delegated: stake_account.delegation().stake, + .map(|(stake_pubkey, delegation)| crate::Delegation { + stake_account: *stake_pubkey, + owner: bank + .get_account(stake_pubkey) + .map(|acc| acc.owner().clone()) + .unwrap_or_default(), + amount_delegated: delegation.stake, }) .collect::>(), ) From c95a1117b61f8b14d897ffd237c3154e5ae5a5f6 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Wed, 12 Oct 2022 23:07:51 -0500 Subject: [PATCH 02/14] wip --- tip-distributor/src/stake_meta_generator_workflow.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index c9dccc2686..125265d5dd 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -230,21 +230,23 @@ fn group_delegations_by_voter_pubkey( ) -> HashMap> { delegations .into_iter() - .filter(|(_stake_pubkey, delegation)| delegation.stake(bank.epoch(), None) > 0) - .into_group_map_by(|(_stake_pubkey, delegation)| delegation.voter_pubkey) + .filter(|(_stake_pubkey, stake_account)| { + stake_account.delegation().stake(bank.epoch(), None) > 0 + }) + .into_group_map_by(|(_stake_pubkey, stake_account)| stake_account.delegation().voter_pubkey) .into_iter() .map(|(voter_pubkey, group)| { ( voter_pubkey, group .into_iter() - .map(|(stake_pubkey, delegation)| crate::Delegation { + .map(|(stake_pubkey, stake_account)| crate::Delegation { stake_account: *stake_pubkey, owner: bank .get_account(stake_pubkey) .map(|acc| acc.owner().clone()) .unwrap_or_default(), - amount_delegated: delegation.stake, + amount_delegated: stake_account.delegation().stake, }) .collect::>(), ) From 23ed14add01f2d994ff633cbc77dc24258077d83 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 00:14:39 -0500 Subject: [PATCH 03/14] simplify --- .../src/bin/stake-meta-generator.rs | 8 +- tip-distributor/src/lib.rs | 95 ++----------------- .../src/stake_meta_generator_workflow.rs | 79 ++++++++------- 3 files changed, 54 insertions(+), 128 deletions(-) diff --git a/tip-distributor/src/bin/stake-meta-generator.rs b/tip-distributor/src/bin/stake-meta-generator.rs index 780afd7ede..f0bf447257 100644 --- a/tip-distributor/src/bin/stake-meta-generator.rs +++ b/tip-distributor/src/bin/stake-meta-generator.rs @@ -58,10 +58,10 @@ fn main() { run_workflow( &args.ledger_path, - args.snapshot_slot, - args.tip_distribution_program_id, - args.out_path, - rpc_client, + &args.snapshot_slot, + &args.tip_distribution_program_id, + &args.out_path, + &rpc_client, ) .expect("Workflow failed."); } diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index bba089889c..a536c89e30 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -6,14 +6,10 @@ use { merkle_root_generator_workflow::Error, stake_meta_generator_workflow::Error::CheckedMathError, }, - anchor_lang::AccountDeserialize, bigdecimal::{num_bigint::BigUint, BigDecimal}, - log::*, num_traits::{CheckedDiv, CheckedMul, ToPrimitive}, serde::{Deserialize, Serialize}, - solana_client::rpc_client::RpcClient, solana_merkle_tree::MerkleTree, - solana_runtime::bank::Bank, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::Slot, @@ -21,10 +17,7 @@ use { pubkey::Pubkey, stake_history::Epoch, }, - std::{ - ops::{Div, Mul}, - sync::Arc, - }, + std::ops::{Div, Mul}, tip_distribution::state::TipDistributionAccount, }; @@ -85,7 +78,7 @@ impl GeneratedMerkleTreeCollection { Some(Ok(GeneratedMerkleTree { max_num_nodes, - tip_distribution_account: tip_distribution_meta.tip_distribution_account, + tip_distribution_account: tip_distribution_meta.tip_distribution_pubkey, merkle_tree, tree_nodes, max_total_claim: tip_distribution_meta.total_tips, @@ -257,7 +250,7 @@ pub struct TipDistributionMeta { pub merkle_root_upload_authority: Pubkey, #[serde(with = "pubkey_string_conversion")] - pub tip_distribution_account: Pubkey, + pub tip_distribution_pubkey: Pubkey, /// The validator's total tips in the [TipDistributionAccount]. pub total_tips: u64, @@ -274,7 +267,7 @@ impl TipDistributionMeta { rent_exempt_amount: u64, ) -> Result { Ok(TipDistributionMeta { - tip_distribution_account: tda_wrapper.tip_distribution_account_pubkey, + tip_distribution_pubkey: tda_wrapper.tip_distribution_pubkey, total_tips: tda_wrapper .account_data .lamports() @@ -306,7 +299,7 @@ pub struct Delegation { pub struct TipDistributionAccountWrapper { pub tip_distribution_account: TipDistributionAccount, pub account_data: AccountSharedData, - pub tip_distribution_account_pubkey: Pubkey, + pub tip_distribution_pubkey: Pubkey, } // TODO: move to program's sdk @@ -325,80 +318,6 @@ pub fn derive_tip_distribution_account_address( ) } -pub trait AccountFetcher { - fn fetch_account( - &self, - pubkey: &Pubkey, - ) -> Result, stake_meta_generator_workflow::Error>; -} - -/// Fetches and deserializes the vote_pubkey's corresponding [TipDistributionAccount]. -pub fn fetch_and_deserialize_tip_distribution_account( - account_fetcher: Arc>, - vote_pubkey: &Pubkey, - tip_distribution_program_id: &Pubkey, - epoch: Epoch, -) -> Result, stake_meta_generator_workflow::Error> { - let tip_distribution_account_pubkey = - derive_tip_distribution_account_address(tip_distribution_program_id, vote_pubkey, epoch).0; - - match account_fetcher.fetch_account(&tip_distribution_account_pubkey)? { - None => { - warn!( - "TipDistributionAccount not found for vote_pubkey {}, epoch {}, tip_distribution_account_pubkey {}", - vote_pubkey, - epoch, - tip_distribution_account_pubkey, - ); - Ok(None) - } - Some(account_data) => Ok(Some(TipDistributionAccountWrapper { - tip_distribution_account: TipDistributionAccount::try_deserialize( - &mut account_data.data(), - )?, - account_data, - tip_distribution_account_pubkey, - })), - } -} - -struct BankAccountFetcher { - bank: Arc, -} - -impl AccountFetcher for BankAccountFetcher { - /// Fetches the vote_pubkey's corresponding [TipDistributionAccount] from the accounts DB. - fn fetch_account( - &self, - pubkey: &Pubkey, - ) -> Result, stake_meta_generator_workflow::Error> { - Ok(self.bank.get_account(pubkey)) - } -} - -struct RpcAccountFetcher { - rpc_client: RpcClient, -} - -impl AccountFetcher for RpcAccountFetcher { - /// Fetches the vote_pubkey's corresponding [TipDistributionAccount] from an RPC node. - fn fetch_account( - &self, - pubkey: &Pubkey, - ) -> Result, stake_meta_generator_workflow::Error> { - match self - .rpc_client - .get_account_with_commitment(pubkey, self.rpc_client.commitment()) - { - Ok(resp) => Ok(resp.value.map(|a| a.into())), - Err(e) => { - error!("error fetching account {}", e); - Err(e.into()) - } - } - } -} - /// Calculate validator fee denominated in lamports pub fn calc_validator_fee(total_tips: u64, validator_commission_bps: u16) -> u64 { let validator_commission_rate = @@ -555,7 +474,7 @@ mod tests { validator_vote_account: validator_vote_account_0.clone(), maybe_tip_distribution_meta: Some(TipDistributionMeta { merkle_root_upload_authority: b58_merkle_root_upload_authority.clone(), - tip_distribution_account: tda_0.clone(), + tip_distribution_pubkey: tda_0.clone(), total_tips: 1_900_122_111_000, validator_fee_bps: 100, }), @@ -576,7 +495,7 @@ mod tests { validator_vote_account: validator_vote_account_1.clone(), maybe_tip_distribution_meta: Some(TipDistributionMeta { merkle_root_upload_authority: b58_merkle_root_upload_authority.clone(), - tip_distribution_account: tda_1.clone(), + tip_distribution_pubkey: tda_1.clone(), total_tips: 1_900_122_111_333, validator_fee_bps: 200, }), diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 125265d5dd..853c193f4a 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -1,9 +1,9 @@ use { crate::{ - fetch_and_deserialize_tip_distribution_account, AccountFetcher, BankAccountFetcher, - RpcAccountFetcher, StakeMeta, StakeMetaCollection, TipDistributionAccountWrapper, - TipDistributionMeta, + derive_tip_distribution_account_address, StakeMeta, StakeMetaCollection, + TipDistributionAccount, TipDistributionAccountWrapper, TipDistributionMeta, }, + anchor_lang::AccountDeserialize, itertools::Itertools, log::*, solana_client::{client_error::ClientError, rpc_client::RpcClient}, @@ -20,7 +20,9 @@ use { vote_account::VoteAccount, }, solana_sdk::{ - account::ReadableAccount, clock::Slot, genesis_config::GenesisConfig, pubkey::Pubkey, + account::{AccountSharedData, ReadableAccount}, + clock::Slot, + pubkey::Pubkey, }, std::{ collections::HashMap, @@ -68,17 +70,17 @@ impl Display for Error { /// to a JSON file. pub fn run_workflow( ledger_path: &Path, - snapshot_slot: Slot, - tip_distribution_program_id: Pubkey, - out_path: String, - rpc_client: RpcClient, + snapshot_slot: &Slot, + tip_distribution_program_id: &Pubkey, + out_path: &str, + rpc_client: &RpcClient, ) -> Result<(), Error> { info!("Creating bank from ledger path..."); let bank = create_bank_from_snapshot(ledger_path, snapshot_slot)?; info!("Generating stake_meta_collection object..."); let stake_meta_coll = - generate_stake_meta_collection(&bank, tip_distribution_program_id, Some(rpc_client))?; + generate_stake_meta_collection(&bank, tip_distribution_program_id, rpc_client)?; info!("Writing stake_meta_collection to JSON {}...", out_path); write_to_json_file(&stake_meta_coll, out_path)?; @@ -86,7 +88,7 @@ pub fn run_workflow( Ok(()) } -fn create_bank_from_snapshot(ledger_path: &Path, snapshot_slot: Slot) -> Result, Error> { +fn create_bank_from_snapshot(ledger_path: &Path, snapshot_slot: &Slot) -> Result, Error> { let genesis_config = open_genesis_config(ledger_path, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE); let snapshot_config = SnapshotConfig { full_snapshot_archive_interval_slots: Slot::MAX, @@ -109,7 +111,7 @@ fn create_bank_from_snapshot(ledger_path: &Path, snapshot_slot: Slot) -> Result< let working_bank = bank_forks.read().unwrap().working_bank(); assert_eq!( working_bank.slot(), - snapshot_slot, + *snapshot_slot, "expected working bank slot {}, found {}", snapshot_slot, working_bank.slot() @@ -118,10 +120,7 @@ fn create_bank_from_snapshot(ledger_path: &Path, snapshot_slot: Slot) -> Result< Ok(working_bank) } -fn write_to_json_file( - stake_meta_coll: &StakeMetaCollection, - out_path: String, -) -> Result<(), Error> { +fn write_to_json_file(stake_meta_coll: &StakeMetaCollection, out_path: &str) -> Result<(), Error> { let file = File::create(out_path)?; let mut writer = BufWriter::new(file); let json = serde_json::to_string_pretty(&stake_meta_coll).unwrap(); @@ -135,9 +134,8 @@ fn write_to_json_file( pub fn generate_stake_meta_collection( bank: &Arc, // Used to derive the PDA and fetch the account data from the Bank. - tip_distribution_program_id: Pubkey, - // Optionally used to fetch the tip distribution accounts from an RPC node. - maybe_rpc_client: Option, + tip_distribution_program_id: &Pubkey, + rpc_client: &RpcClient, ) -> Result { assert!(bank.is_frozen()); @@ -153,28 +151,37 @@ pub fn generate_stake_meta_collection( // the last leader in an epoch may not crank the tip_receiver before the epoch is over. // to get the most accurate number for a given epoch, fetch the tip distribution account // balance from an rpc server. - let tda_account_fetcher = if let Some(rpc_client) = maybe_rpc_client { - Box::new(RpcAccountFetcher { rpc_client }) as Box - } else { - Box::new(BankAccountFetcher { bank: bank.clone() }) as Box - }; - let tda_account_fetcher = Arc::new(tda_account_fetcher); - let vote_pk_and_maybe_tdas: Vec<( (Pubkey, &VoteAccount), Option, )> = epoch_vote_accounts .iter() - .map(|(&vote_pubkey, (_total_stake, vote_account))| { - let tda = fetch_and_deserialize_tip_distribution_account( - tda_account_fetcher.clone(), + .map(|(vote_pubkey, (_total_stake, vote_account))| { + // the tip distribution account for a given epoch will exist in the bank, but might + // not have an accurate number of lamports in it, so only fetch the lamport balance + // from RPC if it exists in the bank (checking bank way faster than rpc). + let tip_distribution_pubkey = derive_tip_distribution_account_address( + tip_distribution_program_id, &vote_pubkey, - &tip_distribution_program_id, bank.epoch(), ) - .map_err(Error::from)?; - - Ok(((vote_pubkey, vote_account), tda)) + .0; + let tda = if bank.get_account(&tip_distribution_pubkey).is_some() { + let account_data: AccountSharedData = rpc_client + .get_account(&tip_distribution_pubkey) + .expect("error reading account from RPC") + .into(); + Some(TipDistributionAccountWrapper { + tip_distribution_account: TipDistributionAccount::try_deserialize( + &mut account_data.data(), + )?, + account_data, + tip_distribution_pubkey, + }) + } else { + None + }; + Ok(((*vote_pubkey, vote_account), tda)) }) .collect::>()?; @@ -216,7 +223,7 @@ pub fn generate_stake_meta_collection( Ok(StakeMetaCollection { stake_metas, - tip_distribution_program_id, + tip_distribution_program_id: *tip_distribution_program_id, bank_hash: bank.hash().to_string(), epoch: bank.epoch(), slot: bank.slot(), @@ -590,7 +597,7 @@ mod tests { merkle_root_upload_authority.as_ref(), ) .into_string(), - tip_distribution_account: bs58::encode(tda_0_fields.0.as_ref()).into_string(), + tip_distribution_pubkey: bs58::encode(tda_0_fields.0.as_ref()).into_string(), total_tips: tip_distro_0_tips .checked_sub( bank.get_minimum_balance_for_rent_exemption( @@ -621,7 +628,7 @@ mod tests { merkle_root_upload_authority.as_ref(), ) .into_string(), - tip_distribution_account: bs58::encode(tda_1_fields.0.as_ref()).into_string(), + tip_distribution_pubkey: bs58::encode(tda_1_fields.0.as_ref()).into_string(), total_tips: tip_distro_1_tips .checked_sub( bank.get_minimum_balance_for_rent_exemption( @@ -652,7 +659,7 @@ mod tests { merkle_root_upload_authority.as_ref(), ) .into_string(), - tip_distribution_account: bs58::encode(tda_2_fields.0.as_ref()).into_string(), + tip_distribution_pubkey: bs58::encode(tda_2_fields.0.as_ref()).into_string(), total_tips: tip_distro_2_tips .checked_sub( bank.get_minimum_balance_for_rent_exemption( From f0faaea654ca11828e11f76dc79f3c9e59dc687b Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 00:40:44 -0500 Subject: [PATCH 04/14] logs --- tip-distributor/src/stake_meta_generator_workflow.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 853c193f4a..8c9d121d52 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -167,6 +167,10 @@ pub fn generate_stake_meta_collection( ) .0; let tda = if bank.get_account(&tip_distribution_pubkey).is_some() { + info!( + "fetching tip distribution account for pubkey: {}", + tip_distribution_pubkey + ); let account_data: AccountSharedData = rpc_client .get_account(&tip_distribution_pubkey) .expect("error reading account from RPC") @@ -179,6 +183,10 @@ pub fn generate_stake_meta_collection( tip_distribution_pubkey, }) } else { + info!( + "no tip distribution account for pubkey: {}", + tip_distribution_pubkey + ); None }; Ok(((*vote_pubkey, vote_account), tda)) From ea91eda3adada5926e2b995bb7bd8f20782e59e7 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 15:12:53 -0500 Subject: [PATCH 05/14] mas --- runtime/src/stake_account.rs | 2 +- .../src/bin/stake-meta-generator.rs | 9 +- tip-distributor/src/lib.rs | 35 ++++---- .../src/stake_meta_generator_workflow.rs | 87 +++++++++++-------- 4 files changed, 76 insertions(+), 57 deletions(-) diff --git a/runtime/src/stake_account.rs b/runtime/src/stake_account.rs index 71bf4d4ae9..39e500a1fd 100644 --- a/runtime/src/stake_account.rs +++ b/runtime/src/stake_account.rs @@ -41,7 +41,7 @@ impl StakeAccount { } #[inline] - pub(crate) fn stake_state(&self) -> &StakeState { + pub fn stake_state(&self) -> &StakeState { &self.stake_state } diff --git a/tip-distributor/src/bin/stake-meta-generator.rs b/tip-distributor/src/bin/stake-meta-generator.rs index f0bf447257..872b216f0a 100644 --- a/tip-distributor/src/bin/stake-meta-generator.rs +++ b/tip-distributor/src/bin/stake-meta-generator.rs @@ -56,12 +56,15 @@ fn main() { let rpc_client = RpcClient::new(args.rpc_url); - run_workflow( + if let Err(e) = run_workflow( &args.ledger_path, &args.snapshot_slot, &args.tip_distribution_program_id, &args.out_path, &rpc_client, - ) - .expect("Workflow failed."); + ) { + error!("error producing stake-meta: {:?}", e); + } else { + info!("produced stake meta"); + } } diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index a536c89e30..37deafe0e4 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -154,11 +154,11 @@ impl TreeNode { .iter() .map(|delegation| { // TODO(seg): Check this math! - let amount_delegated = BigDecimal::try_from(delegation.amount_delegated as f64) + let amount_delegated = BigDecimal::try_from(delegation.lamports_delegated as f64) .expect(&*format!( "failed to convert amount_delegated to BigDecimal [stake_account={}, amount_delegated={}]", - delegation.stake_account, - delegation.amount_delegated, + delegation.stake_account_pubkey, + delegation.lamports_delegated, )); let mut weight = amount_delegated.div(&total_delegated); @@ -169,7 +169,7 @@ impl TreeNode { } let truncated_weight = weight.to_u128() - .expect(&*format!("failed to convert weight to u128 [stake_account={}, weight={}]", delegation.stake_account, weight)); + .expect(&*format!("failed to convert weight to u128 [stake_account={}, weight={}]", delegation.stake_account_pubkey, weight)); let truncated_weight = BigUint::from(truncated_weight); let mut amount = truncated_weight @@ -181,7 +181,7 @@ impl TreeNode { } Ok(TreeNode { - claimant: delegation.stake_account, + claimant: delegation.stake_account_pubkey, amount: amount.to_u64().unwrap(), proof: None }) @@ -286,13 +286,16 @@ impl TipDistributionMeta { #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct Delegation { #[serde(with = "pubkey_string_conversion")] - pub stake_account: Pubkey, + pub stake_account_pubkey: Pubkey, #[serde(with = "pubkey_string_conversion")] - pub owner: Pubkey, + pub staker_pubkey: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub withdrawer_pubkey: Pubkey, /// Lamports delegated by the stake account - pub amount_delegated: u64, + pub lamports_delegated: u64, } /// Convenience wrapper around [TipDistributionAccount] @@ -480,12 +483,12 @@ mod tests { }), delegations: vec![ Delegation { - stake_account: stake_account_0.clone(), - amount_delegated: 123_999_123_555, + stake_account_pubkey: stake_account_0.clone(), + lamports_delegated: 123_999_123_555, }, Delegation { - stake_account: stake_account_1.clone(), - amount_delegated: 144_555_444_556, + stake_account_pubkey: stake_account_1.clone(), + lamports_delegated: 144_555_444_556, }, ], total_delegated: 1_555_123_000_333_454_000, @@ -501,12 +504,12 @@ mod tests { }), delegations: vec![ Delegation { - stake_account: stake_account_2.clone(), - amount_delegated: 224_555_444, + stake_account_pubkey: stake_account_2.clone(), + lamports_delegated: 224_555_444, }, Delegation { - stake_account: stake_account_3.clone(), - amount_delegated: 700_888_944_555, + stake_account_pubkey: stake_account_3.clone(), + lamports_delegated: 700_888_944_555, }, ], total_delegated: 2_565_318_909_444_123, diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 8c9d121d52..e3744a5741 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -178,7 +178,8 @@ pub fn generate_stake_meta_collection( Some(TipDistributionAccountWrapper { tip_distribution_account: TipDistributionAccount::try_deserialize( &mut account_data.data(), - )?, + ) + .expect("deserialize tda"), account_data, tip_distribution_pubkey, }) @@ -199,7 +200,7 @@ pub fn generate_stake_meta_collection( for ((vote_pubkey, vote_account), maybe_tda) in vote_pk_and_maybe_tdas { if let Some(delegations) = voter_pubkey_to_delegations.get(&vote_pubkey).cloned() { let total_delegated = delegations.iter().fold(0u64, |sum, delegation| { - sum.checked_add(delegation.amount_delegated).unwrap() + sum.checked_add(delegation.lamports_delegated).unwrap() }); let maybe_tip_distribution_meta = if let Some(tda) = maybe_tda { @@ -256,12 +257,18 @@ fn group_delegations_by_voter_pubkey( group .into_iter() .map(|(stake_pubkey, stake_account)| crate::Delegation { - stake_account: *stake_pubkey, - owner: bank - .get_account(stake_pubkey) - .map(|acc| acc.owner().clone()) + stake_account_pubkey: *stake_pubkey, + staker_pubkey: stake_account + .stake_state() + .authorized() + .map(|a| a.staker) + .unwrap_or_default(), + withdrawer_pubkey: stake_account + .stake_state() + .authorized() + .map(|a| a.withdrawer) .unwrap_or_default(), - amount_delegated: stake_account.delegation().stake, + lamports_delegated: stake_account.delegation().stake, }) .collect::>(), ) @@ -370,9 +377,11 @@ mod tests { /* 3. Delegate some stake to the initial set of validators */ let mut validator_0_delegations = vec![crate::Delegation { - stake_account: bs58::encode(validator_keypairs_0.stake_keypair.pubkey().as_ref()) - .into_string(), - amount_delegated: INITIAL_VALIDATOR_STAKES, + stake_account_pubkey: bs58::encode( + validator_keypairs_0.stake_keypair.pubkey().as_ref(), + ) + .into_string(), + lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( &bank, @@ -381,8 +390,8 @@ mod tests { 30_000_000_000, ); validator_0_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 30_000_000_000, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 30_000_000_000, }); let stake_account = delegate_stake_helper( &bank, @@ -391,8 +400,8 @@ mod tests { 3_000_000_000, ); validator_0_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 3_000_000_000, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 3_000_000_000, }); let stake_account = delegate_stake_helper( &bank, @@ -401,14 +410,16 @@ mod tests { 33_000_000_000, ); validator_0_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 33_000_000_000, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 33_000_000_000, }); let mut validator_1_delegations = vec![crate::Delegation { - stake_account: bs58::encode(validator_keypairs_1.stake_keypair.pubkey().as_ref()) - .into_string(), - amount_delegated: INITIAL_VALIDATOR_STAKES, + stake_account_pubkey: bs58::encode( + validator_keypairs_1.stake_keypair.pubkey().as_ref(), + ) + .into_string(), + lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( &bank, @@ -417,8 +428,8 @@ mod tests { 4_222_364_000, ); validator_1_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 4_222_364_000, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 4_222_364_000, }); let stake_account = delegate_stake_helper( &bank, @@ -427,14 +438,16 @@ mod tests { 6_000_000_527, ); validator_1_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 6_000_000_527, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 6_000_000_527, }); let mut validator_2_delegations = vec![crate::Delegation { - stake_account: bs58::encode(validator_keypairs_2.stake_keypair.pubkey().as_ref()) - .into_string(), - amount_delegated: INITIAL_VALIDATOR_STAKES, + stake_account_pubkey: bs58::encode( + validator_keypairs_2.stake_keypair.pubkey().as_ref(), + ) + .into_string(), + lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( &bank, @@ -443,8 +456,8 @@ mod tests { 1_300_123_156, ); validator_2_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 1_300_123_156, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 1_300_123_156, }); let stake_account = delegate_stake_helper( &bank, @@ -453,8 +466,8 @@ mod tests { 1_610_565_420, ); validator_2_delegations.push(crate::Delegation { - stake_account: bs58::encode(stake_account.as_ref()).into_string(), - amount_delegated: 1_610_565_420, + stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + lamports_delegated: 1_610_565_420, }); /* 4. Run assertions */ @@ -493,17 +506,17 @@ mod tests { let mut bank = Arc::new(bank); let mut stake_pubkeys = validator_0_delegations .iter() - .map(|v| Pubkey::from_str(&*v.stake_account).unwrap()) + .map(|v| Pubkey::from_str(&*v.stake_account_pubkey).unwrap()) .collect::>(); stake_pubkeys.extend( validator_1_delegations .iter() - .map(|v| Pubkey::from_str(&*v.stake_account).unwrap()), + .map(|v| Pubkey::from_str(&*v.stake_account_pubkey).unwrap()), ); stake_pubkeys.extend( validator_2_delegations .iter() - .map(|v| Pubkey::from_str(&*v.stake_account).unwrap()), + .map(|v| Pubkey::from_str(&*v.stake_account_pubkey).unwrap()), ); loop { if warmed_up(&bank, &stake_pubkeys[..]) { @@ -598,7 +611,7 @@ mod tests { total_delegated: validator_0_delegations .iter() .fold(0u64, |sum, delegation| { - sum.checked_add(delegation.amount_delegated).unwrap() + sum.checked_add(delegation.lamports_delegated).unwrap() }), maybe_tip_distribution_meta: Some(TipDistributionMeta { merkle_root_upload_authority: bs58::encode( @@ -629,7 +642,7 @@ mod tests { total_delegated: validator_1_delegations .iter() .fold(0u64, |sum, delegation| { - sum.checked_add(delegation.amount_delegated).unwrap() + sum.checked_add(delegation.lamports_delegated).unwrap() }), maybe_tip_distribution_meta: Some(TipDistributionMeta { merkle_root_upload_authority: bs58::encode( @@ -660,7 +673,7 @@ mod tests { total_delegated: validator_2_delegations .iter() .fold(0u64, |sum, delegation| { - sum.checked_add(delegation.amount_delegated).unwrap() + sum.checked_add(delegation.lamports_delegated).unwrap() }), maybe_tip_distribution_meta: Some(TipDistributionMeta { merkle_root_upload_authority: bs58::encode( @@ -729,7 +742,7 @@ mod tests { let actual_delegation = actual_stake_meta .delegations .iter() - .find(|d| d.stake_account == expected_delegation.stake_account) + .find(|d| d.stake_account_pubkey == expected_delegation.stake_account_pubkey) .unwrap(); assert_eq!(expected_delegation, actual_delegation); From 0a863ad40526dbfe053ed90563f4c3211609d164 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 17:37:56 -0500 Subject: [PATCH 06/14] wip --- tip-distributor/src/lib.rs | 2 ++ tip-distributor/src/stake_meta_generator_workflow.rs | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index 37deafe0e4..385b9df092 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -31,6 +31,7 @@ pub struct GeneratedMerkleTreeCollection { #[derive(Eq, Debug, Hash, PartialEq, Deserialize, Serialize)] pub struct GeneratedMerkleTree { + #[serde(with = "pubkey_string_conversion")] pub tip_distribution_account: Pubkey, #[serde(skip_serializing, skip_deserializing)] pub merkle_tree: MerkleTree, @@ -113,6 +114,7 @@ pub fn get_proof(merkle_tree: &MerkleTree, i: usize) -> Vec<[u8; 32]> { #[derive(Clone, Eq, Debug, Hash, PartialEq, Deserialize, Serialize)] pub struct TreeNode { /// The account entitled to redeem. + #[serde(with = "pubkey_string_conversion")] pub claimant: Pubkey, /// The amount this account is entitled to. diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index e3744a5741..9bd8f0f6a3 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -148,9 +148,12 @@ pub fn generate_stake_meta_collection( let l_stakes = bank.stakes_cache.stakes(); let delegations = l_stakes.stake_delegations(); - // the last leader in an epoch may not crank the tip_receiver before the epoch is over. - // to get the most accurate number for a given epoch, fetch the tip distribution account - // balance from an rpc server. + // the last leader in an epoch may not crank the tip program before the epoch is over, which + // would result in MEV rewards for epoch N not being cranked until epoch N + 1. This means that + // the account balance in the snapshot could be incorrect. + // We assume that by the time the call to read the balance from the RPC server is made that + // the tip payment program has been cranked and the balance for the tip distribution account + // is reflected accurately by reading the tip distribution amount on-chain. let vote_pk_and_maybe_tdas: Vec<( (Pubkey, &VoteAccount), Option, From 8666607815f3f16f42b8325156e54722df07a67e Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 19:33:16 -0500 Subject: [PATCH 07/14] mas --- Cargo.lock | 1 + tip-distributor/Cargo.toml | 1 + .../src/bin/stake-meta-generator.rs | 13 +-- tip-distributor/src/lib.rs | 46 ++++++--- .../src/merkle_root_generator_workflow.rs | 7 +- .../src/stake_meta_generator_workflow.rs | 95 +++++++++++-------- 6 files changed, 99 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18f8d056c1..cb52de2391 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6813,6 +6813,7 @@ dependencies = [ "solana-stake-program", "thiserror", "tip-distribution", + "tip-payment", "tokio", ] diff --git a/tip-distributor/Cargo.toml b/tip-distributor/Cargo.toml index 8bd15e9524..028238df2c 100644 --- a/tip-distributor/Cargo.toml +++ b/tip-distributor/Cargo.toml @@ -27,6 +27,7 @@ solana-sdk = { path = "../sdk", version = "=1.15.0" } solana-stake-program = { path = "../programs/stake", version = "=1.15.0" } thiserror = "1.0.31" tip-distribution = { path = "../jito-programs/tip-payment/programs/tip-distribution", features = ["no-entrypoint"] } +tip-payment = { path = "../jito-programs/tip-payment/programs/tip-payment", features = ["no-entrypoint"] } tokio = { version = "1.12.0", features = ["rt-multi-thread", "macros", "sync", "time", "full"] } [[bin]] diff --git a/tip-distributor/src/bin/stake-meta-generator.rs b/tip-distributor/src/bin/stake-meta-generator.rs index 872b216f0a..c92a49da4e 100644 --- a/tip-distributor/src/bin/stake-meta-generator.rs +++ b/tip-distributor/src/bin/stake-meta-generator.rs @@ -5,7 +5,6 @@ use { clap::Parser, log::*, - solana_client::rpc_client::RpcClient, solana_sdk::{clock::Slot, pubkey::Pubkey}, solana_tip_distributor::{self, stake_meta_generator_workflow::run_workflow}, std::{ @@ -26,6 +25,10 @@ struct Args { #[clap(long, env)] tip_distribution_program_id: Pubkey, + /// The tip-payment program id. + #[clap(long, env)] + tip_payment_program_id: Pubkey, + /// Path to JSON file populated with the [StakeMetaCollection] object. #[clap(long, env)] out_path: String, @@ -33,10 +36,6 @@ struct Args { /// The expected snapshot slot. #[clap(long, env)] snapshot_slot: Slot, - - /// The RPC to fetch lamports from for the tip distribution accounts. - #[clap(long, env)] - rpc_url: String, } impl Args { @@ -54,14 +53,12 @@ fn main() { let args: Args = Args::parse(); - let rpc_client = RpcClient::new(args.rpc_url); - if let Err(e) = run_workflow( &args.ledger_path, &args.snapshot_slot, &args.tip_distribution_program_id, &args.out_path, - &rpc_client, + &args.tip_payment_program_id, ) { error!("error producing stake-meta: {:?}", e); } else { diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index 385b9df092..ad16828111 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -19,6 +19,11 @@ use { }, std::ops::{Div, Mul}, tip_distribution::state::TipDistributionAccount, + tip_payment::{ + Config, CONFIG_ACCOUNT_SEED, TIP_ACCOUNT_SEED_0, TIP_ACCOUNT_SEED_1, TIP_ACCOUNT_SEED_2, + TIP_ACCOUNT_SEED_3, TIP_ACCOUNT_SEED_4, TIP_ACCOUNT_SEED_5, TIP_ACCOUNT_SEED_6, + TIP_ACCOUNT_SEED_7, + }, }; #[derive(Deserialize, Serialize, Debug)] @@ -33,6 +38,8 @@ pub struct GeneratedMerkleTreeCollection { pub struct GeneratedMerkleTree { #[serde(with = "pubkey_string_conversion")] pub tip_distribution_account: Pubkey, + #[serde(with = "pubkey_string_conversion")] + pub merkle_root_upload_authority: Pubkey, #[serde(skip_serializing, skip_deserializing)] pub merkle_tree: MerkleTree, pub tree_nodes: Vec, @@ -40,31 +47,25 @@ pub struct GeneratedMerkleTree { pub max_num_nodes: u64, } +pub struct TipPaymentPubkeys { + config_pda: Pubkey, + tip_pdas: Vec, +} + impl GeneratedMerkleTreeCollection { pub fn new_from_stake_meta_collection( stake_meta_coll: StakeMetaCollection, - merkle_root_upload_authority: Pubkey, ) -> Result { let generated_merkle_trees = stake_meta_coll .stake_metas .into_iter() - .filter(|stake_meta| { - if let Some(tip_distribution_meta) = stake_meta.maybe_tip_distribution_meta.as_ref() - { - tip_distribution_meta.merkle_root_upload_authority - == merkle_root_upload_authority - } else { - false - } - }) + .filter(|stake_meta| stake_meta.maybe_tip_distribution_meta.is_some()) .filter_map(|stake_meta| { - // Build the tree let mut tree_nodes = match TreeNode::vec_from_stake_meta(&stake_meta) { Err(e) => return Some(Err(e)), Ok(maybe_tree_nodes) => maybe_tree_nodes, }?; - // Hash the nodes let hashed_nodes: Vec<[u8; 32]> = tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); @@ -80,6 +81,8 @@ impl GeneratedMerkleTreeCollection { Some(Ok(GeneratedMerkleTree { max_num_nodes, tip_distribution_account: tip_distribution_meta.tip_distribution_pubkey, + merkle_root_upload_authority: tip_distribution_meta + .merkle_root_upload_authority, merkle_tree, tree_nodes, max_total_claim: tip_distribution_meta.total_tips, @@ -111,6 +114,25 @@ pub fn get_proof(merkle_tree: &MerkleTree, i: usize) -> Vec<[u8; 32]> { proof } +fn derive_tip_payment_pubkeys(program_id: &Pubkey) -> TipPaymentPubkeys { + let config_pda = Pubkey::find_program_address(&[CONFIG_ACCOUNT_SEED], &program_id).0; + let tip_pda_0 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_0], &program_id).0; + let tip_pda_1 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_1], &program_id).0; + let tip_pda_2 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_2], &program_id).0; + let tip_pda_3 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_3], &program_id).0; + let tip_pda_4 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_4], &program_id).0; + let tip_pda_5 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_5], &program_id).0; + let tip_pda_6 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_6], &program_id).0; + let tip_pda_7 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_7], &program_id).0; + + TipPaymentPubkeys { + config_pda, + tip_pdas: vec![ + tip_pda_0, tip_pda_1, tip_pda_2, tip_pda_3, tip_pda_4, tip_pda_5, tip_pda_6, tip_pda_7, + ], + } +} + #[derive(Clone, Eq, Debug, Hash, PartialEq, Deserialize, Serialize)] pub struct TreeNode { /// The account entitled to redeem. diff --git a/tip-distributor/src/merkle_root_generator_workflow.rs b/tip-distributor/src/merkle_root_generator_workflow.rs index 77d6d38a16..a3e4061157 100644 --- a/tip-distributor/src/merkle_root_generator_workflow.rs +++ b/tip-distributor/src/merkle_root_generator_workflow.rs @@ -63,10 +63,8 @@ pub fn run_workflow( ) -> Result<(), Error> { let stake_meta_coll = read_stake_meta_collection(stake_meta_coll_path)?; - let merkle_tree_coll = GeneratedMerkleTreeCollection::new_from_stake_meta_collection( - stake_meta_coll.clone(), - my_keypair.pubkey(), - )?; + let merkle_tree_coll = + GeneratedMerkleTreeCollection::new_from_stake_meta_collection(stake_meta_coll.clone())?; write_to_json_file(&merkle_tree_coll, out_path)?; @@ -138,6 +136,7 @@ fn upload_roots( let txs = merkle_tree_coll .generated_merkle_trees .into_iter() + .filter(|tree| tree.merkle_root_upload_authority == my_keypair.pubkey()) .filter(|gmt| { if !force_upload_roots { let account = rt diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 9bd8f0f6a3..4319ff8fb0 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -1,12 +1,13 @@ use { crate::{ - derive_tip_distribution_account_address, StakeMeta, StakeMetaCollection, - TipDistributionAccount, TipDistributionAccountWrapper, TipDistributionMeta, + derive_tip_distribution_account_address, derive_tip_payment_pubkeys, Config, StakeMeta, + StakeMetaCollection, TipDistributionAccount, TipDistributionAccountWrapper, + TipDistributionMeta, }, anchor_lang::AccountDeserialize, itertools::Itertools, log::*, - solana_client::{client_error::ClientError, rpc_client::RpcClient}, + solana_client::client_error::ClientError, solana_ledger::{ bank_forks_utils, blockstore::BlockstoreError, @@ -20,7 +21,7 @@ use { vote_account::VoteAccount, }, solana_sdk::{ - account::{AccountSharedData, ReadableAccount}, + account::{ReadableAccount, WritableAccount}, clock::Slot, pubkey::Pubkey, }, @@ -73,14 +74,14 @@ pub fn run_workflow( snapshot_slot: &Slot, tip_distribution_program_id: &Pubkey, out_path: &str, - rpc_client: &RpcClient, + tip_payment_program_id: &Pubkey, ) -> Result<(), Error> { info!("Creating bank from ledger path..."); let bank = create_bank_from_snapshot(ledger_path, snapshot_slot)?; info!("Generating stake_meta_collection object..."); let stake_meta_coll = - generate_stake_meta_collection(&bank, tip_distribution_program_id, rpc_client)?; + generate_stake_meta_collection(&bank, tip_distribution_program_id, tip_payment_program_id)?; info!("Writing stake_meta_collection to JSON {}...", out_path); write_to_json_file(&stake_meta_coll, out_path)?; @@ -133,9 +134,8 @@ fn write_to_json_file(stake_meta_coll: &StakeMetaCollection, out_path: &str) -> /// Creates a collection of [StakeMeta]'s from the given bank. pub fn generate_stake_meta_collection( bank: &Arc, - // Used to derive the PDA and fetch the account data from the Bank. tip_distribution_program_id: &Pubkey, - rpc_client: &RpcClient, + tip_payment_program_id: &Pubkey, ) -> Result { assert!(bank.is_frozen()); @@ -148,57 +148,72 @@ pub fn generate_stake_meta_collection( let l_stakes = bank.stakes_cache.stakes(); let delegations = l_stakes.stake_delegations(); + let voter_pubkey_to_delegations = group_delegations_by_voter_pubkey(delegations, bank); + // the last leader in an epoch may not crank the tip program before the epoch is over, which // would result in MEV rewards for epoch N not being cranked until epoch N + 1. This means that // the account balance in the snapshot could be incorrect. - // We assume that by the time the call to read the balance from the RPC server is made that - // the tip payment program has been cranked and the balance for the tip distribution account - // is reflected accurately by reading the tip distribution amount on-chain. + // We assume that the rewards sitting in the tip program PDAs are cranked out by the time all of + // the rewards are claimed. + let tip_accounts = derive_tip_payment_pubkeys(&tip_payment_program_id); + let tip_receiver = Config::try_deserialize( + &mut bank + .get_account(&tip_accounts.config_pda) + .expect("tip payment config account exists") + .data(), + ) + .expect("tip payment config account deserializes") + .tip_receiver; + + let excess_tip_balances: u64 = tip_accounts + .tip_pdas + .iter() + .map(|pubkey| { + let acc = bank.get_account(pubkey).expect("tip account exists"); + acc.lamports() + .checked_sub(bank.get_minimum_balance_for_rent_exemption(acc.data().len())) + .expect("tip balance underflow") + }) + .sum(); + let vote_pk_and_maybe_tdas: Vec<( (Pubkey, &VoteAccount), Option, )> = epoch_vote_accounts .iter() .map(|(vote_pubkey, (_total_stake, vote_account))| { - // the tip distribution account for a given epoch will exist in the bank, but might - // not have an accurate number of lamports in it, so only fetch the lamport balance - // from RPC if it exists in the bank (checking bank way faster than rpc). let tip_distribution_pubkey = derive_tip_distribution_account_address( tip_distribution_program_id, &vote_pubkey, bank.epoch(), ) .0; - let tda = if bank.get_account(&tip_distribution_pubkey).is_some() { - info!( - "fetching tip distribution account for pubkey: {}", - tip_distribution_pubkey - ); - let account_data: AccountSharedData = rpc_client - .get_account(&tip_distribution_pubkey) - .expect("error reading account from RPC") - .into(); - Some(TipDistributionAccountWrapper { - tip_distribution_account: TipDistributionAccount::try_deserialize( - &mut account_data.data(), - ) - .expect("deserialize tda"), - account_data, - tip_distribution_pubkey, - }) - } else { - info!( - "no tip distribution account for pubkey: {}", - tip_distribution_pubkey - ); - None - }; + let tda = bank + .get_account(&tip_distribution_pubkey) + .map(|mut account_data| { + let tip_distribution_account = + TipDistributionAccount::try_deserialize(&mut account_data.data()) + .expect("deserialized TipDistributionAccount"); + // this snapshot might have tips that weren't claimed by the time the epoch is over + // assume that it will eventually be cranked and credit the excess to this account + if tip_distribution_pubkey == tip_receiver { + account_data.set_lamports( + account_data + .lamports() + .checked_add(excess_tip_balances) + .expect("tip overflow"), + ); + } + TipDistributionAccountWrapper { + tip_distribution_account, + account_data, + tip_distribution_pubkey, + } + }); Ok(((*vote_pubkey, vote_account), tda)) }) .collect::>()?; - let voter_pubkey_to_delegations = group_delegations_by_voter_pubkey(delegations, bank); - let mut stake_metas = vec![]; for ((vote_pubkey, vote_account), maybe_tda) in vote_pk_and_maybe_tdas { if let Some(delegations) = voter_pubkey_to_delegations.get(&vote_pubkey).cloned() { From 449369241faaed3749534605b5f2c19f9bb4f7a7 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 21:11:26 -0500 Subject: [PATCH 08/14] more cleanup --- tip-distributor/src/lib.rs | 58 ++++---- .../src/stake_meta_generator_workflow.rs | 136 +++++++++--------- 2 files changed, 101 insertions(+), 93 deletions(-) diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index ad16828111..b10227ed8f 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -474,21 +474,22 @@ mod tests { #[test] fn test_new_from_stake_meta_collection_happy_path() { - let b58_merkle_root_upload_authority = - bs58::encode(Pubkey::new_unique().as_ref()).into_string(); + let merkle_root_upload_authority = Pubkey::new_unique(); - let (tda_0, tda_1) = ( - bs58::encode(Pubkey::new_unique().as_ref()).into_string(), - bs58::encode(Pubkey::new_unique().as_ref()).into_string(), - ); + let (tda_0, tda_1) = (Pubkey::new_unique(), Pubkey::new_unique()); + + let stake_account_0 = Pubkey::new_unique(); + let stake_account_1 = Pubkey::new_unique(); + let stake_account_2 = Pubkey::new_unique(); + let stake_account_3 = Pubkey::new_unique(); - let stake_account_0 = bs58::encode(Pubkey::new_unique().as_ref()).into_string(); - let stake_account_1 = bs58::encode(Pubkey::new_unique().as_ref()).into_string(); - let stake_account_2 = bs58::encode(Pubkey::new_unique().as_ref()).into_string(); - let stake_account_3 = bs58::encode(Pubkey::new_unique().as_ref()).into_string(); + let staker_account_0 = Pubkey::new_unique(); + let staker_account_1 = Pubkey::new_unique(); + let staker_account_2 = Pubkey::new_unique(); + let staker_account_3 = Pubkey::new_unique(); - let validator_vote_account_0 = bs58::encode(Pubkey::new_unique().as_ref()).into_string(); - let validator_vote_account_1 = bs58::encode(Pubkey::new_unique().as_ref()).into_string(); + let validator_vote_account_0 = Pubkey::new_unique(); + let validator_vote_account_1 = Pubkey::new_unique(); println!("test stake_account {}", stake_account_0); println!("test stake_account {}", stake_account_1); @@ -500,7 +501,7 @@ mod tests { StakeMeta { validator_vote_account: validator_vote_account_0.clone(), maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: b58_merkle_root_upload_authority.clone(), + merkle_root_upload_authority: merkle_root_upload_authority.clone(), tip_distribution_pubkey: tda_0.clone(), total_tips: 1_900_122_111_000, validator_fee_bps: 100, @@ -508,10 +509,14 @@ mod tests { delegations: vec![ Delegation { stake_account_pubkey: stake_account_0.clone(), + staker_pubkey: staker_account_0, + withdrawer_pubkey: staker_account_0, lamports_delegated: 123_999_123_555, }, Delegation { stake_account_pubkey: stake_account_1.clone(), + staker_pubkey: staker_account_1, + withdrawer_pubkey: staker_account_1, lamports_delegated: 144_555_444_556, }, ], @@ -521,7 +526,7 @@ mod tests { StakeMeta { validator_vote_account: validator_vote_account_1.clone(), maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: b58_merkle_root_upload_authority.clone(), + merkle_root_upload_authority: merkle_root_upload_authority.clone(), tip_distribution_pubkey: tda_1.clone(), total_tips: 1_900_122_111_333, validator_fee_bps: 200, @@ -529,10 +534,14 @@ mod tests { delegations: vec![ Delegation { stake_account_pubkey: stake_account_2.clone(), + staker_pubkey: staker_account_2, + withdrawer_pubkey: staker_account_2, lamports_delegated: 224_555_444, }, Delegation { stake_account_pubkey: stake_account_3.clone(), + staker_pubkey: staker_account_3, + withdrawer_pubkey: staker_account_3, lamports_delegated: 700_888_944_555, }, ], @@ -540,7 +549,7 @@ mod tests { commission: 10, }, ], - tip_distribution_program_id: bs58::encode(Pubkey::new_unique().as_ref()).into_string(), + tip_distribution_program_id: Pubkey::new_unique(), bank_hash: solana_sdk::hash::Hash::new_unique().to_string(), epoch: 100, slot: 2_000_000, @@ -548,7 +557,6 @@ mod tests { let merkle_tree_collection = GeneratedMerkleTreeCollection::new_from_stake_meta_collection( stake_meta_collection.clone(), - b58_merkle_root_upload_authority.parse().unwrap(), ) .unwrap(); @@ -565,24 +573,25 @@ mod tests { let tree_nodes = vec![ TreeNode { - claimant: validator_vote_account_0.parse().unwrap(), + claimant: validator_vote_account_0, amount: 19_001_221_110, proof: None, }, TreeNode { - claimant: stake_account_0.parse().unwrap(), + claimant: stake_account_0, amount: 149_992, proof: None, }, TreeNode { - claimant: stake_account_1.parse().unwrap(), + claimant: stake_account_1, amount: 174_858, proof: None, }, ]; let hashed_nodes: Vec<[u8; 32]> = tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); let gmt_0 = GeneratedMerkleTree { - tip_distribution_account: tda_0.parse().unwrap(), + tip_distribution_account: tda_0, + merkle_root_upload_authority, merkle_tree: MerkleTree::new(&hashed_nodes[..], true), tree_nodes, max_total_claim: stake_meta_collection.stake_metas[0] @@ -595,24 +604,25 @@ mod tests { let tree_nodes = vec![ TreeNode { - claimant: validator_vote_account_1.parse().unwrap(), + claimant: validator_vote_account_1, amount: 38_002_442_227, proof: None, }, TreeNode { - claimant: stake_account_2.parse().unwrap(), + claimant: stake_account_2, amount: 163_000, proof: None, }, TreeNode { - claimant: stake_account_3.parse().unwrap(), + claimant: stake_account_3, amount: 508_762_900, proof: None, }, ]; let hashed_nodes: Vec<[u8; 32]> = tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); let gmt_1 = GeneratedMerkleTree { - tip_distribution_account: tda_1.parse().unwrap(), + tip_distribution_account: tda_1, + merkle_root_upload_authority, merkle_tree: MerkleTree::new(&hashed_nodes[..], true), tree_nodes, max_total_claim: stake_meta_collection.stake_metas[1] diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 4319ff8fb0..eda24069e2 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -156,23 +156,23 @@ pub fn generate_stake_meta_collection( // We assume that the rewards sitting in the tip program PDAs are cranked out by the time all of // the rewards are claimed. let tip_accounts = derive_tip_payment_pubkeys(&tip_payment_program_id); - let tip_receiver = Config::try_deserialize( - &mut bank - .get_account(&tip_accounts.config_pda) - .expect("tip payment config account exists") - .data(), - ) - .expect("tip payment config account deserializes") - .tip_receiver; + let account = bank.get_account(&tip_accounts.config_pda); + let maybe_tip_receiver = account + .map(|account| Config::try_deserialize(&mut account.data()).ok()) + .flatten() + .map(|config| config.tip_receiver); let excess_tip_balances: u64 = tip_accounts .tip_pdas .iter() .map(|pubkey| { - let acc = bank.get_account(pubkey).expect("tip account exists"); - acc.lamports() - .checked_sub(bank.get_minimum_balance_for_rent_exemption(acc.data().len())) - .expect("tip balance underflow") + bank.get_account(pubkey) + .map(|acc| { + acc.lamports() + .checked_sub(bank.get_minimum_balance_for_rent_exemption(acc.data().len())) + .expect("tip balance underflow") + }) + .unwrap_or_default() }) .sum(); @@ -196,7 +196,9 @@ pub fn generate_stake_meta_collection( .expect("deserialized TipDistributionAccount"); // this snapshot might have tips that weren't claimed by the time the epoch is over // assume that it will eventually be cranked and credit the excess to this account - if tip_distribution_pubkey == tip_receiver { + if maybe_tip_receiver.is_some() + && tip_distribution_pubkey == maybe_tip_receiver.unwrap() + { account_data.set_lamports( account_data .lamports() @@ -317,7 +319,6 @@ mod tests { transaction::Transaction, }, solana_stake_program::stake_state, - std::str::FromStr, tip_distribution::state::TipDistributionAccount, }; @@ -395,10 +396,9 @@ mod tests { /* 3. Delegate some stake to the initial set of validators */ let mut validator_0_delegations = vec![crate::Delegation { - stake_account_pubkey: bs58::encode( - validator_keypairs_0.stake_keypair.pubkey().as_ref(), - ) - .into_string(), + stake_account_pubkey: validator_keypairs_0.stake_keypair.pubkey(), + staker_pubkey: validator_keypairs_0.node_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_0.node_keypair.pubkey(), lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( @@ -408,7 +408,9 @@ mod tests { 30_000_000_000, ); validator_0_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_0.pubkey(), + withdrawer_pubkey: delegator_0.pubkey(), lamports_delegated: 30_000_000_000, }); let stake_account = delegate_stake_helper( @@ -418,7 +420,9 @@ mod tests { 3_000_000_000, ); validator_0_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_1.pubkey(), + withdrawer_pubkey: delegator_1.pubkey(), lamports_delegated: 3_000_000_000, }); let stake_account = delegate_stake_helper( @@ -428,15 +432,16 @@ mod tests { 33_000_000_000, ); validator_0_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_2.pubkey(), + withdrawer_pubkey: delegator_2.pubkey(), lamports_delegated: 33_000_000_000, }); let mut validator_1_delegations = vec![crate::Delegation { - stake_account_pubkey: bs58::encode( - validator_keypairs_1.stake_keypair.pubkey().as_ref(), - ) - .into_string(), + stake_account_pubkey: validator_keypairs_1.stake_keypair.pubkey(), + staker_pubkey: validator_keypairs_1.node_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_1.node_keypair.pubkey(), lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( @@ -446,7 +451,9 @@ mod tests { 4_222_364_000, ); validator_1_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_3.pubkey(), + withdrawer_pubkey: delegator_3.pubkey(), lamports_delegated: 4_222_364_000, }); let stake_account = delegate_stake_helper( @@ -456,15 +463,16 @@ mod tests { 6_000_000_527, ); validator_1_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_4.pubkey(), + withdrawer_pubkey: delegator_4.pubkey(), lamports_delegated: 6_000_000_527, }); let mut validator_2_delegations = vec![crate::Delegation { - stake_account_pubkey: bs58::encode( - validator_keypairs_2.stake_keypair.pubkey().as_ref(), - ) - .into_string(), + stake_account_pubkey: validator_keypairs_2.stake_keypair.pubkey(), + staker_pubkey: validator_keypairs_2.node_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_2.node_keypair.pubkey(), lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( @@ -474,7 +482,9 @@ mod tests { 1_300_123_156, ); validator_2_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_0.pubkey(), + withdrawer_pubkey: delegator_0.pubkey(), lamports_delegated: 1_300_123_156, }); let stake_account = delegate_stake_helper( @@ -484,7 +494,9 @@ mod tests { 1_610_565_420, ); validator_2_delegations.push(crate::Delegation { - stake_account_pubkey: bs58::encode(stake_account.as_ref()).into_string(), + stake_account_pubkey: stake_account, + staker_pubkey: delegator_4.pubkey(), + withdrawer_pubkey: delegator_4.pubkey(), lamports_delegated: 1_610_565_420, }); @@ -524,17 +536,17 @@ mod tests { let mut bank = Arc::new(bank); let mut stake_pubkeys = validator_0_delegations .iter() - .map(|v| Pubkey::from_str(&*v.stake_account_pubkey).unwrap()) + .map(|v| v.stake_account_pubkey) .collect::>(); stake_pubkeys.extend( validator_1_delegations .iter() - .map(|v| Pubkey::from_str(&*v.stake_account_pubkey).unwrap()), + .map(|v| v.stake_account_pubkey), ); stake_pubkeys.extend( validator_2_delegations .iter() - .map(|v| Pubkey::from_str(&*v.stake_account_pubkey).unwrap()), + .map(|v| v.stake_account_pubkey), ); loop { if warmed_up(&bank, &stake_pubkeys[..]) { @@ -608,23 +620,24 @@ mod tests { bank.store_accounts((bank.slot(), &accounts[..])); bank.freeze(); - let stake_meta_collection = - generate_stake_meta_collection(&bank, tip_distribution_program_id, None).unwrap(); + let stake_meta_collection = generate_stake_meta_collection( + &bank, + &tip_distribution_program_id, + &Pubkey::new_unique(), + ) + .unwrap(); assert_eq!( stake_meta_collection.tip_distribution_program_id, - bs58::encode(tip_distribution_program_id.as_ref()).into_string() + tip_distribution_program_id ); assert_eq!(stake_meta_collection.slot, bank.slot()); assert_eq!(stake_meta_collection.epoch, bank.epoch()); let mut expected_stake_metas = HashMap::new(); expected_stake_metas.insert( - bs58::encode(validator_keypairs_0.vote_keypair.pubkey()).into_string(), + validator_keypairs_0.vote_keypair.pubkey(), StakeMeta { - validator_vote_account: bs58::encode( - validator_keypairs_0.vote_keypair.pubkey().as_ref(), - ) - .into_string(), + validator_vote_account: validator_keypairs_0.vote_keypair.pubkey(), delegations: validator_0_delegations.clone(), total_delegated: validator_0_delegations .iter() @@ -632,11 +645,8 @@ mod tests { sum.checked_add(delegation.lamports_delegated).unwrap() }), maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: bs58::encode( - merkle_root_upload_authority.as_ref(), - ) - .into_string(), - tip_distribution_pubkey: bs58::encode(tda_0_fields.0.as_ref()).into_string(), + merkle_root_upload_authority, + tip_distribution_pubkey: tda_0_fields.0, total_tips: tip_distro_0_tips .checked_sub( bank.get_minimum_balance_for_rent_exemption( @@ -650,12 +660,9 @@ mod tests { }, ); expected_stake_metas.insert( - bs58::encode(validator_keypairs_1.vote_keypair.pubkey().as_ref()).into_string(), + validator_keypairs_1.vote_keypair.pubkey(), StakeMeta { - validator_vote_account: bs58::encode( - validator_keypairs_1.vote_keypair.pubkey().as_ref(), - ) - .into_string(), + validator_vote_account: validator_keypairs_1.vote_keypair.pubkey(), delegations: validator_1_delegations.clone(), total_delegated: validator_1_delegations .iter() @@ -663,11 +670,8 @@ mod tests { sum.checked_add(delegation.lamports_delegated).unwrap() }), maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: bs58::encode( - merkle_root_upload_authority.as_ref(), - ) - .into_string(), - tip_distribution_pubkey: bs58::encode(tda_1_fields.0.as_ref()).into_string(), + merkle_root_upload_authority, + tip_distribution_pubkey: tda_1_fields.0, total_tips: tip_distro_1_tips .checked_sub( bank.get_minimum_balance_for_rent_exemption( @@ -681,12 +685,9 @@ mod tests { }, ); expected_stake_metas.insert( - bs58::encode(validator_keypairs_2.vote_keypair.pubkey().as_ref()).into_string(), + validator_keypairs_2.vote_keypair.pubkey(), StakeMeta { - validator_vote_account: bs58::encode( - validator_keypairs_2.vote_keypair.pubkey().as_ref(), - ) - .into_string(), + validator_vote_account: validator_keypairs_2.vote_keypair.pubkey(), delegations: validator_2_delegations.clone(), total_delegated: validator_2_delegations .iter() @@ -694,11 +695,8 @@ mod tests { sum.checked_add(delegation.lamports_delegated).unwrap() }), maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: bs58::encode( - merkle_root_upload_authority.as_ref(), - ) - .into_string(), - tip_distribution_pubkey: bs58::encode(tda_2_fields.0.as_ref()).into_string(), + merkle_root_upload_authority, + tip_distribution_pubkey: tda_2_fields.0, total_tips: tip_distro_2_tips .checked_sub( bank.get_minimum_balance_for_rent_exemption( From 4306ef5fc5dcb109f7fb737f2370834d7aad4f55 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 21:17:45 -0500 Subject: [PATCH 09/14] ci --- tip-distributor/src/lib.rs | 41 +++++++++---------- .../src/merkle_root_generator_workflow.rs | 2 +- .../src/stake_meta_generator_workflow.rs | 11 +++-- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index b10227ed8f..03b14355b3 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -115,15 +115,15 @@ pub fn get_proof(merkle_tree: &MerkleTree, i: usize) -> Vec<[u8; 32]> { } fn derive_tip_payment_pubkeys(program_id: &Pubkey) -> TipPaymentPubkeys { - let config_pda = Pubkey::find_program_address(&[CONFIG_ACCOUNT_SEED], &program_id).0; - let tip_pda_0 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_0], &program_id).0; - let tip_pda_1 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_1], &program_id).0; - let tip_pda_2 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_2], &program_id).0; - let tip_pda_3 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_3], &program_id).0; - let tip_pda_4 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_4], &program_id).0; - let tip_pda_5 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_5], &program_id).0; - let tip_pda_6 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_6], &program_id).0; - let tip_pda_7 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_7], &program_id).0; + let config_pda = Pubkey::find_program_address(&[CONFIG_ACCOUNT_SEED], program_id).0; + let tip_pda_0 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_0], program_id).0; + let tip_pda_1 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_1], program_id).0; + let tip_pda_2 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_2], program_id).0; + let tip_pda_3 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_3], program_id).0; + let tip_pda_4 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_4], program_id).0; + let tip_pda_5 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_5], program_id).0; + let tip_pda_6 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_6], program_id).0; + let tip_pda_7 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_7], program_id).0; TipPaymentPubkeys { config_pda, @@ -420,8 +420,7 @@ mod pubkey_string_conversion { where S: Serializer, { - let s = format!("{}", pubkey.to_string()); - serializer.serialize_str(&s) + serializer.serialize_str(&pubkey.to_string()) } pub fn deserialize<'de, D>(deserializer: D) -> Result @@ -499,22 +498,22 @@ mod tests { let stake_meta_collection = StakeMetaCollection { stake_metas: vec![ StakeMeta { - validator_vote_account: validator_vote_account_0.clone(), + validator_vote_account: validator_vote_account_0, maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: merkle_root_upload_authority.clone(), - tip_distribution_pubkey: tda_0.clone(), + merkle_root_upload_authority, + tip_distribution_pubkey: tda_0, total_tips: 1_900_122_111_000, validator_fee_bps: 100, }), delegations: vec![ Delegation { - stake_account_pubkey: stake_account_0.clone(), + stake_account_pubkey: stake_account_0, staker_pubkey: staker_account_0, withdrawer_pubkey: staker_account_0, lamports_delegated: 123_999_123_555, }, Delegation { - stake_account_pubkey: stake_account_1.clone(), + stake_account_pubkey: stake_account_1, staker_pubkey: staker_account_1, withdrawer_pubkey: staker_account_1, lamports_delegated: 144_555_444_556, @@ -524,22 +523,22 @@ mod tests { commission: 100, }, StakeMeta { - validator_vote_account: validator_vote_account_1.clone(), + validator_vote_account: validator_vote_account_1, maybe_tip_distribution_meta: Some(TipDistributionMeta { - merkle_root_upload_authority: merkle_root_upload_authority.clone(), - tip_distribution_pubkey: tda_1.clone(), + merkle_root_upload_authority, + tip_distribution_pubkey: tda_1, total_tips: 1_900_122_111_333, validator_fee_bps: 200, }), delegations: vec![ Delegation { - stake_account_pubkey: stake_account_2.clone(), + stake_account_pubkey: stake_account_2, staker_pubkey: staker_account_2, withdrawer_pubkey: staker_account_2, lamports_delegated: 224_555_444, }, Delegation { - stake_account_pubkey: stake_account_3.clone(), + stake_account_pubkey: stake_account_3, staker_pubkey: staker_account_3, withdrawer_pubkey: staker_account_3, lamports_delegated: 700_888_944_555, diff --git a/tip-distributor/src/merkle_root_generator_workflow.rs b/tip-distributor/src/merkle_root_generator_workflow.rs index a3e4061157..38b3a1cdfe 100644 --- a/tip-distributor/src/merkle_root_generator_workflow.rs +++ b/tip-distributor/src/merkle_root_generator_workflow.rs @@ -115,7 +115,7 @@ fn write_to_json_file( let file = File::create(file_path)?; let mut writer = BufWriter::new(file); let json = serde_json::to_string_pretty(&merkle_tree_coll).unwrap(); - writer.write(json.as_bytes())?; + let _ = writer.write(json.as_bytes())?; writer.flush()?; Ok(()) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index eda24069e2..c75e619a6e 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -125,7 +125,7 @@ fn write_to_json_file(stake_meta_coll: &StakeMetaCollection, out_path: &str) -> let file = File::create(out_path)?; let mut writer = BufWriter::new(file); let json = serde_json::to_string_pretty(&stake_meta_coll).unwrap(); - writer.write(json.as_bytes())?; + let _ = writer.write(json.as_bytes())?; writer.flush()?; Ok(()) @@ -155,12 +155,11 @@ pub fn generate_stake_meta_collection( // the account balance in the snapshot could be incorrect. // We assume that the rewards sitting in the tip program PDAs are cranked out by the time all of // the rewards are claimed. - let tip_accounts = derive_tip_payment_pubkeys(&tip_payment_program_id); + let tip_accounts = derive_tip_payment_pubkeys(tip_payment_program_id); let account = bank.get_account(&tip_accounts.config_pda); let maybe_tip_receiver = account - .map(|account| Config::try_deserialize(&mut account.data()).ok()) - .flatten() - .map(|config| config.tip_receiver); + .and_then(|account| Config::try_deserialize(&mut account.data()).ok()) + .and_then(|config| Some(config.tip_receiver)); let excess_tip_balances: u64 = tip_accounts .tip_pdas @@ -184,7 +183,7 @@ pub fn generate_stake_meta_collection( .map(|(vote_pubkey, (_total_stake, vote_account))| { let tip_distribution_pubkey = derive_tip_distribution_account_address( tip_distribution_program_id, - &vote_pubkey, + vote_pubkey, bank.epoch(), ) .0; From d00bc4395434fc31408cbcbb978b0a344c0a2548 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 21:26:46 -0500 Subject: [PATCH 10/14] tsts passing --- .../src/stake_meta_generator_workflow.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index c75e619a6e..a2fbae7a2e 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -396,8 +396,8 @@ mod tests { /* 3. Delegate some stake to the initial set of validators */ let mut validator_0_delegations = vec![crate::Delegation { stake_account_pubkey: validator_keypairs_0.stake_keypair.pubkey(), - staker_pubkey: validator_keypairs_0.node_keypair.pubkey(), - withdrawer_pubkey: validator_keypairs_0.node_keypair.pubkey(), + staker_pubkey: validator_keypairs_0.stake_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_0.stake_keypair.pubkey(), lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( @@ -439,8 +439,8 @@ mod tests { let mut validator_1_delegations = vec![crate::Delegation { stake_account_pubkey: validator_keypairs_1.stake_keypair.pubkey(), - staker_pubkey: validator_keypairs_1.node_keypair.pubkey(), - withdrawer_pubkey: validator_keypairs_1.node_keypair.pubkey(), + staker_pubkey: validator_keypairs_1.stake_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_1.stake_keypair.pubkey(), lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( @@ -470,8 +470,8 @@ mod tests { let mut validator_2_delegations = vec![crate::Delegation { stake_account_pubkey: validator_keypairs_2.stake_keypair.pubkey(), - staker_pubkey: validator_keypairs_2.node_keypair.pubkey(), - withdrawer_pubkey: validator_keypairs_2.node_keypair.pubkey(), + staker_pubkey: validator_keypairs_2.stake_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_2.stake_keypair.pubkey(), lamports_delegated: INITIAL_VALIDATOR_STAKES, }]; let stake_account = delegate_stake_helper( @@ -794,7 +794,7 @@ mod tests { &from_keypair.pubkey(), &stake_keypair.pubkey(), vote_account, - &Authorized::auto(&stake_keypair.pubkey()), + &Authorized::auto(&from_keypair.pubkey()), &Lockup::default(), delegation_amount, ); From 09c4e7920ad5a9dfdb2880edbc818573220f07f9 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 21:30:11 -0500 Subject: [PATCH 11/14] ci --- tip-distributor/src/stake_meta_generator_workflow.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index a2fbae7a2e..15ec382cc3 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -159,7 +159,7 @@ pub fn generate_stake_meta_collection( let account = bank.get_account(&tip_accounts.config_pda); let maybe_tip_receiver = account .and_then(|account| Config::try_deserialize(&mut account.data()).ok()) - .and_then(|config| Some(config.tip_receiver)); + .map(|config| Some(config.tip_receiver)); let excess_tip_balances: u64 = tip_accounts .tip_pdas From d34c02df0d344384c7aafcc069c93fb26e22cddb Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 21:34:01 -0500 Subject: [PATCH 12/14] werd --- tip-distributor/src/stake_meta_generator_workflow.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index 15ec382cc3..bbc9353026 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -157,9 +157,9 @@ pub fn generate_stake_meta_collection( // the rewards are claimed. let tip_accounts = derive_tip_payment_pubkeys(tip_payment_program_id); let account = bank.get_account(&tip_accounts.config_pda); - let maybe_tip_receiver = account + let maybe_tip_receiver: Option = account .and_then(|account| Config::try_deserialize(&mut account.data()).ok()) - .map(|config| Some(config.tip_receiver)); + .and_then(|config| Some(config.tip_receiver)); let excess_tip_balances: u64 = tip_accounts .tip_pdas From 63da172d4d2a622c1b243b6ec49b0a28de3bebcf Mon Sep 17 00:00:00 2001 From: Lucas B Date: Thu, 13 Oct 2022 21:37:36 -0500 Subject: [PATCH 13/14] ok --- tip-distributor/src/stake_meta_generator_workflow.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs index bbc9353026..8bcdf30045 100644 --- a/tip-distributor/src/stake_meta_generator_workflow.rs +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -159,7 +159,7 @@ pub fn generate_stake_meta_collection( let account = bank.get_account(&tip_accounts.config_pda); let maybe_tip_receiver: Option = account .and_then(|account| Config::try_deserialize(&mut account.data()).ok()) - .and_then(|config| Some(config.tip_receiver)); + .map(|config| config.tip_receiver); let excess_tip_balances: u64 = tip_accounts .tip_pdas From 35bb34070d3b8c9cbaec7d53f0afe92042c10ed7 Mon Sep 17 00:00:00 2001 From: Lucas B Date: Fri, 14 Oct 2022 13:49:15 -0500 Subject: [PATCH 14/14] werd --- tip-distributor/src/lib.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs index 03b14355b3..14a00c811f 100644 --- a/tip-distributor/src/lib.rs +++ b/tip-distributor/src/lib.rs @@ -135,10 +135,16 @@ fn derive_tip_payment_pubkeys(program_id: &Pubkey) -> TipPaymentPubkeys { #[derive(Clone, Eq, Debug, Hash, PartialEq, Deserialize, Serialize)] pub struct TreeNode { - /// The account entitled to redeem. + /// The stake account entitled to redeem. #[serde(with = "pubkey_string_conversion")] pub claimant: Pubkey, + #[serde(with = "pubkey_string_conversion")] + pub staker_pubkey: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub withdrawer_pubkey: Pubkey, + /// The amount this account is entitled to. pub amount: u64, @@ -155,6 +161,8 @@ impl TreeNode { ); let mut tree_nodes = vec![TreeNode { claimant: stake_meta.validator_vote_account, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: validator_fee, proof: None, }]; @@ -206,6 +214,8 @@ impl TreeNode { Ok(TreeNode { claimant: delegation.stake_account_pubkey, + staker_pubkey: delegation.staker_pubkey, + withdrawer_pubkey: delegation.withdrawer_pubkey, amount: amount.to_u64().unwrap(), proof: None }) @@ -445,11 +455,15 @@ mod tests { let tree_nodes = vec![ TreeNode { claimant: acct_0.parse().unwrap(), + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 151_507, proof: None, }, TreeNode { claimant: acct_1.parse().unwrap(), + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 176_624, proof: None, }, @@ -573,16 +587,22 @@ mod tests { let tree_nodes = vec![ TreeNode { claimant: validator_vote_account_0, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 19_001_221_110, proof: None, }, TreeNode { claimant: stake_account_0, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 149_992, proof: None, }, TreeNode { claimant: stake_account_1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 174_858, proof: None, }, @@ -604,16 +624,22 @@ mod tests { let tree_nodes = vec![ TreeNode { claimant: validator_vote_account_1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 38_002_442_227, proof: None, }, TreeNode { claimant: stake_account_2, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 163_000, proof: None, }, TreeNode { claimant: stake_account_3, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), amount: 508_762_900, proof: None, },