diff --git a/Cargo.lock b/Cargo.lock index d6e93fd1d5d3a..3594fc270a33e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1309,6 +1309,7 @@ dependencies = [ "aptos-network", "aptos-reliable-broadcast", "aptos-runtimes", + "aptos-safety-rules", "aptos-time-service", "aptos-types", "aptos-validator-transaction-pool", @@ -2354,6 +2355,7 @@ dependencies = [ "aptos-network", "aptos-reliable-broadcast", "aptos-runtimes", + "aptos-safety-rules", "aptos-time-service", "aptos-types", "aptos-validator-transaction-pool", @@ -3046,7 +3048,6 @@ dependencies = [ "aptos-peer-monitoring-service-server", "aptos-peer-monitoring-service-types", "aptos-runtimes", - "aptos-safety-rules", "aptos-state-sync-driver", "aptos-storage-interface", "aptos-storage-service-client", @@ -3515,7 +3516,6 @@ dependencies = [ name = "aptos-safety-rules" version = "0.1.0" dependencies = [ - "anyhow", "aptos-config", "aptos-consensus-types", "aptos-crypto", @@ -3530,6 +3530,7 @@ dependencies = [ "aptos-vault-client", "claims", "criterion", + "hex", "once_cell", "proptest", "rand 0.7.3", diff --git a/aptos-node/Cargo.toml b/aptos-node/Cargo.toml index d411df696e306..61acb392e7e4b 100644 --- a/aptos-node/Cargo.toml +++ b/aptos-node/Cargo.toml @@ -51,7 +51,6 @@ aptos-peer-monitoring-service-client = { workspace = true } aptos-peer-monitoring-service-server = { workspace = true } aptos-peer-monitoring-service-types = { workspace = true } aptos-runtimes = { workspace = true } -aptos-safety-rules = { workspace = true } aptos-state-sync-driver = { workspace = true } aptos-storage-interface = { workspace = true } aptos-storage-service-client = { workspace = true } diff --git a/aptos-node/src/consensus.rs b/aptos-node/src/consensus.rs index 6f3c9ceb0bd14..ed503a69c5efe 100644 --- a/aptos-node/src/consensus.rs +++ b/aptos-node/src/consensus.rs @@ -25,10 +25,8 @@ use aptos_event_notifications::{ DbBackedOnChainConfig, EventNotificationListener, ReconfigNotificationListener, }; use aptos_jwk_consensus::{start_jwk_consensus_runtime, types::JWKConsensusMsg}; -use aptos_logger::debug; use aptos_mempool::QuorumStoreRequest; use aptos_network::application::interface::{NetworkClient, NetworkServiceEvents}; -use aptos_safety_rules::safety_rules_manager::load_consensus_key_from_secure_storage; use aptos_storage_interface::DbReaderWriter; use aptos_validator_transaction_pool::VTxnPoolState; use futures::channel::mpsc::Sender; @@ -73,13 +71,9 @@ pub fn create_dkg_runtime( )>, dkg_network_interfaces: Option>, ) -> (VTxnPoolState, Option) { - let maybe_dkg_dealer_sk = - load_consensus_key_from_secure_storage(&node_config.consensus.safety_rules); - debug!("maybe_dkg_dealer_sk={:?}", maybe_dkg_dealer_sk); - let vtxn_pool = VTxnPoolState::default(); - let dkg_runtime = match (dkg_network_interfaces, maybe_dkg_dealer_sk) { - (Some(interfaces), Ok(dkg_dealer_sk)) => { + let dkg_runtime = match dkg_network_interfaces { + Some(interfaces) => { let ApplicationNetworkInterfaces { network_client, network_service_events, @@ -90,7 +84,7 @@ pub fn create_dkg_runtime( let rb_config = node_config.consensus.rand_rb_config.clone(); let dkg_runtime = start_dkg_runtime( my_addr, - dkg_dealer_sk, + &node_config.consensus.safety_rules, network_client, network_service_events, reconfig_events, @@ -117,15 +111,8 @@ pub fn create_jwk_consensus_runtime( jwk_consensus_network_interfaces: Option>, vtxn_pool: &VTxnPoolState, ) -> Option { - let maybe_jwk_consensus_key = - load_consensus_key_from_secure_storage(&node_config.consensus.safety_rules); - debug!( - "jwk_consensus_key_err={:?}", - maybe_jwk_consensus_key.as_ref().err() - ); - - let jwk_consensus_runtime = match (jwk_consensus_network_interfaces, maybe_jwk_consensus_key) { - (Some(interfaces), Ok(consensus_key)) => { + let jwk_consensus_runtime = match jwk_consensus_network_interfaces { + Some(interfaces) => { let ApplicationNetworkInterfaces { network_client, network_service_events, @@ -136,7 +123,7 @@ pub fn create_jwk_consensus_runtime( let my_addr = node_config.validator_network.as_ref().unwrap().peer_id(); let jwk_consensus_runtime = start_jwk_consensus_runtime( my_addr, - consensus_key, + &node_config.consensus.safety_rules, network_client, network_service_events, reconfig_events, diff --git a/aptos-node/src/lib.rs b/aptos-node/src/lib.rs index 28cd85091a9ae..4634c22c3efb7 100644 --- a/aptos-node/src/lib.rs +++ b/aptos-node/src/lib.rs @@ -20,9 +20,7 @@ use anyhow::anyhow; use aptos_admin_service::AdminService; use aptos_api::bootstrap as bootstrap_api; use aptos_build_info::build_information; -use aptos_config::config::{ - merge_node_config, InitialSafetyRulesConfig, NodeConfig, PersistableConfig, -}; +use aptos_config::config::{merge_node_config, NodeConfig, PersistableConfig}; use aptos_framework::ReleaseBundle; use aptos_logger::{prelude::*, telemetry_log_writer::TelemetryLog, Level, LoggerFilterUpdater}; use aptos_state_sync_driver::driver_factory::StateSyncRuntimes; @@ -703,17 +701,6 @@ pub fn setup_environment_and_start_node( peers_and_metadata, ); - // Ensure consensus key in secure DB. - if !matches!( - node_config - .consensus - .safety_rules - .initial_safety_rules_config, - InitialSafetyRulesConfig::None - ) { - aptos_safety_rules::safety_rules_manager::storage(&node_config.consensus.safety_rules); - } - // Create the DKG runtime and get the VTxn pool let (vtxn_pool, dkg_runtime) = consensus::create_dkg_runtime(&mut node_config, dkg_subscriptions, dkg_network_interfaces); diff --git a/config/Cargo.toml b/config/Cargo.toml index 1bdf814cd7b5c..cc9e9b5855cb0 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -46,5 +46,6 @@ tempfile = { workspace = true } default = [] failpoints = [] fuzzing = ["aptos-crypto/fuzzing", "aptos-types/fuzzing"] +smoke-test = [] testing = [] tokio-console = [] diff --git a/config/src/config/safety_rules_config.rs b/config/src/config/safety_rules_config.rs index e5e19db1c17e5..9f7af088f0278 100644 --- a/config/src/config/safety_rules_config.rs +++ b/config/src/config/safety_rules_config.rs @@ -123,15 +123,22 @@ impl ConfigSanitizer for SafetyRulesConfig { pub enum InitialSafetyRulesConfig { FromFile { identity_blob_path: PathBuf, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + overriding_identity_paths: Vec, waypoint: WaypointConfig, }, None, } impl InitialSafetyRulesConfig { - pub fn from_file(identity_blob_path: PathBuf, waypoint: WaypointConfig) -> Self { + pub fn from_file( + identity_blob_path: PathBuf, + overriding_identity_paths: Vec, + waypoint: WaypointConfig, + ) -> Self { Self::FromFile { identity_blob_path, + overriding_identity_paths, waypoint, } } @@ -160,6 +167,38 @@ impl InitialSafetyRulesConfig { }, } } + + pub fn overriding_identity_blobs(&self) -> anyhow::Result> { + match self { + InitialSafetyRulesConfig::FromFile { + overriding_identity_paths, + .. + } => { + let mut blobs = vec![]; + for path in overriding_identity_paths { + let blob = IdentityBlob::from_file(path)?; + blobs.push(blob); + } + Ok(blobs) + }, + InitialSafetyRulesConfig::None => { + bail!("loading overriding identity blobs failed with missing initial safety rules config") + }, + } + } + + #[cfg(feature = "smoke-test")] + pub fn overriding_identity_blob_paths_mut(&mut self) -> &mut Vec { + match self { + InitialSafetyRulesConfig::FromFile { + overriding_identity_paths, + .. + } => overriding_identity_paths, + InitialSafetyRulesConfig::None => { + unreachable!() + }, + } + } } /// Defines how safety rules should be executed diff --git a/consensus/safety-rules/Cargo.toml b/consensus/safety-rules/Cargo.toml index 3132a07dba70d..962b14c1ceabc 100644 --- a/consensus/safety-rules/Cargo.toml +++ b/consensus/safety-rules/Cargo.toml @@ -13,7 +13,6 @@ repository = { workspace = true } rust-version = { workspace = true } [dependencies] -anyhow = { workspace = true } aptos-config = { workspace = true } aptos-consensus-types = { workspace = true } aptos-crypto = { workspace = true } @@ -25,6 +24,7 @@ aptos-secure-net = { workspace = true } aptos-secure-storage = { workspace = true } aptos-types = { workspace = true } aptos-vault-client = { workspace = true } +hex = { workspace = true } once_cell = { workspace = true } proptest = { workspace = true, optional = true } rand = { workspace = true } diff --git a/consensus/safety-rules/src/persistent_safety_storage.rs b/consensus/safety-rules/src/persistent_safety_storage.rs index 0f3c03b9daf88..532ae6b2c8cbf 100644 --- a/consensus/safety-rules/src/persistent_safety_storage.rs +++ b/consensus/safety-rules/src/persistent_safety_storage.rs @@ -96,16 +96,32 @@ impl PersistentSafetyStorage { Ok(self.internal_store.get(OWNER_ACCOUNT).map(|v| v.value)?) } - pub fn consensus_key_for_version( + pub fn consensus_sk_by_pk( &self, - version: bls12381::PublicKey, + pk: bls12381::PublicKey, ) -> Result { let _timer = counters::start_timer("get", CONSENSUS_KEY); - let key: bls12381::PrivateKey = self.internal_store.get(CONSENSUS_KEY).map(|v| v.value)?; - if key.public_key() != version { + let pk_hex = hex::encode(pk.to_bytes()); + let explicit_storage_key = format!("{}_{}", CONSENSUS_KEY, pk_hex); + let explicit_sk = self + .internal_store + .get::(explicit_storage_key.as_str()) + .map(|v| v.value); + let default_sk = self + .internal_store + .get::(CONSENSUS_KEY) + .map(|v| v.value); + let key = match (explicit_sk, default_sk) { + (Ok(sk_0), _) => sk_0, + (Err(_), Ok(sk_1)) => sk_1, + (Err(_), Err(_)) => { + return Err(Error::ValidatorKeyNotFound("not found!".to_string())); + }, + }; + if key.public_key() != pk { return Err(Error::SecureStorageMissingDataError(format!( - "PrivateKey for {:?} not found", - version + "Incorrect sk saved for {:?} the expected pk", + pk ))); } Ok(key) @@ -164,7 +180,6 @@ impl PersistentSafetyStorage { Ok(()) } - #[cfg(any(test, feature = "testing"))] pub fn internal_store(&mut self) -> &mut Storage { &mut self.internal_store } diff --git a/consensus/safety-rules/src/safety_rules.rs b/consensus/safety-rules/src/safety_rules.rs index 4072dd6aa65c1..c0b94133bfc61 100644 --- a/consensus/safety-rules/src/safety_rules.rs +++ b/consensus/safety-rules/src/safety_rules.rs @@ -32,7 +32,7 @@ use aptos_types::{ waypoint::Waypoint, }; use serde::Serialize; -use std::cmp::Ordering; +use std::{cmp::Ordering, sync::Arc}; pub(crate) fn next_round(round: Round) -> Result { u64::checked_add(round, 1).ok_or(Error::IncorrectRound(round)) @@ -316,13 +316,10 @@ impl SafetyRules { Ok(()) } else { // Try to export the consensus key directly from storage. - match self - .persistent_storage - .consensus_key_for_version(expected_key) - { + match self.persistent_storage.consensus_sk_by_pk(expected_key) { Ok(consensus_key) => { self.validator_signer = - Some(ValidatorSigner::new(author, consensus_key)); + Some(ValidatorSigner::new(author, Arc::new(consensus_key))); Ok(()) }, Err(Error::SecureStorageMissingDataError(error)) => { diff --git a/consensus/safety-rules/src/safety_rules_manager.rs b/consensus/safety-rules/src/safety_rules_manager.rs index f0e743d3d9dbb..7d87297789d2b 100644 --- a/consensus/safety-rules/src/safety_rules_manager.rs +++ b/consensus/safety-rules/src/safety_rules_manager.rs @@ -11,13 +11,13 @@ use crate::{ thread::ThreadService, SafetyRules, TSafetyRules, }; -use anyhow::anyhow; use aptos_config::config::{InitialSafetyRulesConfig, SafetyRulesConfig, SafetyRulesService}; -use aptos_crypto::bls12381::PrivateKey; +use aptos_crypto::bls12381::PublicKey; use aptos_global_constants::CONSENSUS_KEY; use aptos_infallible::RwLock; +use aptos_logger::{info, warn}; use aptos_secure_storage::{KVStorage, Storage}; -use std::{net::SocketAddr, sync::Arc}; +use std::{net::SocketAddr, sync::Arc, time::Instant}; pub fn storage(config: &SafetyRulesConfig) -> PersistentSafetyStorage { let backend = &config.backend; @@ -45,8 +45,8 @@ pub fn storage(config: &SafetyRulesConfig) -> PersistentSafetyStorage { } else { let storage = PersistentSafetyStorage::new(internal_storage, config.enable_cached_safety_data); - // If it's initialized, then we can continue - if storage.author().is_ok() { + + let mut storage = if storage.author().is_ok() { storage } else if !matches!( config.initial_safety_rules_config, @@ -75,19 +75,32 @@ pub fn storage(config: &SafetyRulesConfig) -> PersistentSafetyStorage { panic!( "Safety rules storage is not initialized, provide an initial safety rules config" ) + }; + + // Ensuring all the overriding consensus keys are in the storage. + let timer = Instant::now(); + for blob in config + .initial_safety_rules_config + .overriding_identity_blobs() + .unwrap_or_default() + { + if let Some(sk) = blob.consensus_private_key { + let pk_hex = hex::encode(PublicKey::from(&sk).to_bytes()); + let storage_key = format!("{}_{}", CONSENSUS_KEY, pk_hex); + match storage.internal_store().set(storage_key.as_str(), sk) { + Ok(_) => { + info!("Setting {storage_key} succeeded."); + }, + Err(e) => { + warn!("Setting {storage_key} failed with internal store set error: {e}"); + }, + } + } } - } -} + info!("Overriding key work time: {:?}", timer.elapsed()); -pub fn load_consensus_key_from_secure_storage( - config: &SafetyRulesConfig, -) -> anyhow::Result { - let storage: Storage = (&config.backend).into(); - let storage = Box::new(storage); - let response = storage.get::(CONSENSUS_KEY).map_err(|e| { - anyhow!("load_consensus_key_from_secure_storage failed with storage read error: {e}") - })?; - Ok(response.value) + storage + } } enum SafetyRulesWrapper { diff --git a/consensus/src/consensus_observer/observer.rs b/consensus/src/consensus_observer/observer.rs index 4724abd9957eb..ff28bcdc70a54 100644 --- a/consensus/src/consensus_observer/observer.rs +++ b/consensus/src/consensus_observer/observer.rs @@ -1095,15 +1095,14 @@ impl ConsensusObserver { }; // Start the new epoch - let signer = Arc::new(ValidatorSigner::new( - AccountAddress::ZERO, - bls12381::PrivateKey::genesis(), - )); + let sk = Arc::new(bls12381::PrivateKey::genesis()); + let signer = Arc::new(ValidatorSigner::new(AccountAddress::ZERO, sk.clone())); let dummy_signer = Arc::new(DagCommitSigner::new(signer.clone())); let (_, rand_msg_rx) = aptos_channel::new::(QueueStyle::FIFO, 1, None); self.execution_client .start_epoch( + Some(sk), epoch_state.clone(), dummy_signer, payload_manager, diff --git a/consensus/src/epoch_manager.rs b/consensus/src/epoch_manager.rs index df384c5afd48a..186d86b9312ae 100644 --- a/consensus/src/epoch_manager.rs +++ b/consensus/src/epoch_manager.rs @@ -58,7 +58,6 @@ use aptos_bounded_executor::BoundedExecutor; use aptos_channels::{aptos_channel, message_queues::QueueStyle}; use aptos_config::config::{ ConsensusConfig, DagConsensusConfig, ExecutionConfig, NodeConfig, QcAggregatorType, - SafetyRulesConfig, SecureBackend, }; use aptos_consensus_types::{ common::{Author, Round}, @@ -67,19 +66,17 @@ use aptos_consensus_types::{ proof_of_store::ProofCache, utils::PayloadTxnsSize, }; -use aptos_crypto::bls12381; +use aptos_crypto::bls12381::PrivateKey; use aptos_dkg::{ pvss::{traits::Transcript, Player}, weighted_vuf::traits::WeightedVUF, }; use aptos_event_notifications::ReconfigNotificationListener; -use aptos_global_constants::CONSENSUS_KEY; use aptos_infallible::{duration_since_epoch, Mutex}; use aptos_logger::prelude::*; use aptos_mempool::QuorumStoreRequest; use aptos_network::{application::interface::NetworkClient, protocols::network::Event}; -use aptos_safety_rules::SafetyRulesManager; -use aptos_secure_storage::{KVStorage, Storage}; +use aptos_safety_rules::{safety_rules_manager, PersistentSafetyStorage, SafetyRulesManager}; use aptos_types::{ account_address::AccountAddress, dkg::{real_dkg::maybe_dk_from_bls_sk, DKGState, DKGTrait, DefaultDKG}, @@ -94,6 +91,7 @@ use aptos_types::{ }, randomness::{RandKeys, WvufPP, WVUF}, validator_signer::ValidatorSigner, + validator_verifier::ValidatorVerifier, }; use aptos_validator_transaction_pool::VTxnPoolState; use fail::fail_point; @@ -178,6 +176,7 @@ pub struct EpochManager { proof_cache: ProofCache, consensus_publisher: Option>, pending_blocks: Arc>, + key_storage: PersistentSafetyStorage, } impl EpochManager

{ @@ -205,6 +204,7 @@ impl EpochManager

{ let dag_config = node_config.dag_consensus.clone(); let sr_config = &node_config.consensus.safety_rules; let safety_rules_manager = SafetyRulesManager::new(sr_config); + let key_storage = safety_rules_manager::storage(sr_config); Self { author, config, @@ -247,6 +247,7 @@ impl EpochManager

{ .build(), consensus_publisher, pending_blocks: Arc::new(Mutex::new(PendingBlocks::new())), + key_storage, } } @@ -759,6 +760,7 @@ impl EpochManager

{ async fn start_round_manager( &mut self, + consensus_key: Option>, recovery_data: RecoveryData, epoch_state: Arc, onchain_consensus_config: OnChainConsensusConfig, @@ -815,6 +817,7 @@ impl EpochManager

{ self.execution_client .start_epoch( + consensus_key, epoch_state.clone(), safety_rules_container.clone(), payload_manager.clone(), @@ -940,6 +943,7 @@ impl EpochManager

{ fn try_get_rand_config_for_new_epoch( &self, + maybe_consensus_key: Option>, new_epoch_state: &EpochState, onchain_randomness_config: &OnChainRandomnessConfig, maybe_dkg_state: anyhow::Result, @@ -968,8 +972,10 @@ impl EpochManager

{ .copied() .ok_or_else(|| NoRandomnessReason::NotInValidatorSet)?; - let dkg_decrypt_key = load_dkg_decrypt_key(&self.config.safety_rules) - .ok_or_else(|| NoRandomnessReason::DKGDecryptKeyUnavailable)?; + let consensus_key = + maybe_consensus_key.ok_or(NoRandomnessReason::ConsensusKeyUnavailable)?; + let dkg_decrypt_key = maybe_dk_from_bls_sk(consensus_key.as_ref()) + .map_err(NoRandomnessReason::ErrConvertingConsensusKeyToDecryptionKey)?; let transcript = bcs::from_bytes::<::Transcript>( dkg_session.transcript.as_slice(), ) @@ -1009,8 +1015,13 @@ impl EpochManager

{ .map_err(NoRandomnessReason::RandDbNotAvailable)? .filter(|(epoch, _)| *epoch == new_epoch) { + info!(epoch = new_epoch, "Recovering existing augmented key"); bcs::from_bytes(&key_pair).map_err(NoRandomnessReason::KeyPairDeserializationError)? } else { + info!( + epoch = new_epoch_state.epoch, + "Generating a new augmented key" + ); let mut rng = StdRng::from_rng(thread_rng()).map_err(NoRandomnessReason::RngCreationError)?; let augmented_key_pair = WVUF::augment_key_pair(&vuf_pp, sk.main, pk.main, &mut rng); @@ -1133,7 +1144,17 @@ impl EpochManager

{ // `jwk_consensus_config` not yet initialized, falling back to the old configs. Self::equivalent_jwk_consensus_config_from_deprecated_resources(&payload) }); + + let loaded_consensus_key = match self.load_consensus_key(&epoch_state.verifier) { + Ok(k) => Some(Arc::new(k)), + Err(e) => { + warn!("load_consensus_key failed: {e}"); + None + }, + }; + let rand_configs = self.try_get_rand_config_for_new_epoch( + loaded_consensus_key.clone(), &epoch_state, &onchain_randomness_config, dkg_state, @@ -1180,6 +1201,7 @@ impl EpochManager

{ if consensus_config.is_dag_enabled() { self.start_new_epoch_with_dag( epoch_state, + loaded_consensus_key.clone(), consensus_config, execution_config, onchain_randomness_config, @@ -1194,6 +1216,7 @@ impl EpochManager

{ .await } else { self.start_new_epoch_with_joltean( + loaded_consensus_key.clone(), epoch_state, consensus_config, execution_config, @@ -1242,6 +1265,7 @@ impl EpochManager

{ async fn start_new_epoch_with_joltean( &mut self, + consensus_key: Option>, epoch_state: Arc, consensus_config: OnChainConsensusConfig, execution_config: OnChainExecutionConfig, @@ -1258,6 +1282,7 @@ impl EpochManager

{ LivenessStorageData::FullRecoveryData(initial_data) => { self.recovery_mode = false; self.start_round_manager( + consensus_key, initial_data, epoch_state, consensus_config, @@ -1289,6 +1314,7 @@ impl EpochManager

{ async fn start_new_epoch_with_dag( &mut self, epoch_state: Arc, + loaded_consensus_key: Option>, onchain_consensus_config: OnChainConsensusConfig, on_chain_execution_config: OnChainExecutionConfig, onchain_randomness_config: OnChainRandomnessConfig, @@ -1301,9 +1327,12 @@ impl EpochManager

{ rand_msg_rx: aptos_channel::Receiver, ) { let epoch = epoch_state.epoch; - let consensus_key = new_consensus_key_from_storage(&self.config.safety_rules.backend) - .expect("unable to get private key"); - let signer = Arc::new(ValidatorSigner::new(self.author, consensus_key)); + let signer = Arc::new(ValidatorSigner::new( + self.author, + loaded_consensus_key + .clone() + .expect("unable to get private key"), + )); let commit_signer = Arc::new(DagCommitSigner::new(signer.clone())); assert!( @@ -1320,6 +1349,7 @@ impl EpochManager

{ self.execution_client .start_epoch( + loaded_consensus_key, epoch_state.clone(), commit_signer, payload_manager.clone(), @@ -1751,55 +1781,15 @@ impl EpochManager

{ let oidc_providers = payload.get::().ok(); OnChainJWKConsensusConfig::from((features, oidc_providers)) } -} - -fn new_consensus_key_from_storage(backend: &SecureBackend) -> anyhow::Result { - let storage: Storage = backend.into(); - storage - .available() - .map_err(|e| anyhow!("Storage is not available: {e}"))?; - storage - .get(CONSENSUS_KEY) - .map(|v| v.value) - .map_err(|e| anyhow!("storage get and map err: {e}")) -} -fn load_dkg_decrypt_key_from_identity_blob( - config: &SafetyRulesConfig, -) -> anyhow::Result<::NewValidatorDecryptKey> { - let identity_blob = config.initial_safety_rules_config.identity_blob()?; - identity_blob.try_into_dkg_new_validator_decrypt_key() -} - -fn load_dkg_decrypt_key_from_secure_storage( - config: &SafetyRulesConfig, -) -> anyhow::Result<::NewValidatorDecryptKey> { - let consensus_key = new_consensus_key_from_storage(&config.backend)?; - maybe_dk_from_bls_sk(&consensus_key) -} - -fn load_dkg_decrypt_key( - config: &SafetyRulesConfig, -) -> Option<::NewValidatorDecryptKey> { - match load_dkg_decrypt_key_from_secure_storage(config) { - Ok(dk) => { - return Some(dk); - }, - Err(e) => { - warn!("{e}"); - }, + fn load_consensus_key(&self, vv: &ValidatorVerifier) -> anyhow::Result { + let pk = vv + .get_public_key(&self.author) + .ok_or_else(|| anyhow!("i am not in the validator set!"))?; + self.key_storage + .consensus_sk_by_pk(pk) + .map_err(|e| anyhow!("could not find sk by pk: {:?}", e)) } - - match load_dkg_decrypt_key_from_identity_blob(config) { - Ok(dk) => { - return Some(dk); - }, - Err(e) => { - warn!("{e}"); - }, - } - - None } #[derive(Debug)] @@ -1810,7 +1800,8 @@ pub enum NoRandomnessReason { DKGCompletedSessionResourceMissing, CompletedSessionTooOld, NotInValidatorSet, - DKGDecryptKeyUnavailable, + ConsensusKeyUnavailable, + ErrConvertingConsensusKeyToDecryptionKey(anyhow::Error), TranscriptDeserializationError(bcs::Error), SecretShareDecryptionFailed(anyhow::Error), RngCreationError(rand::Error), @@ -1818,5 +1809,5 @@ pub enum NoRandomnessReason { KeyPairDeserializationError(bcs::Error), KeyPairSerializationError(bcs::Error), KeyPairPersistError(anyhow::Error), - // Test only reasons + MyPkNotFoundInValidatorSet, } diff --git a/consensus/src/pipeline/execution_client.rs b/consensus/src/pipeline/execution_client.rs index 95dddba956c2b..efefc4097f2aa 100644 --- a/consensus/src/pipeline/execution_client.rs +++ b/consensus/src/pipeline/execution_client.rs @@ -33,11 +33,11 @@ use aptos_consensus_types::{ common::{Author, Round}, pipelined_block::PipelinedBlock, }; +use aptos_crypto::bls12381::PrivateKey; use aptos_executor_types::ExecutorResult; use aptos_infallible::RwLock; use aptos_logger::prelude::*; use aptos_network::{application::interface::NetworkClient, protocols::network::Event}; -use aptos_safety_rules::safety_rules_manager::load_consensus_key_from_secure_storage; use aptos_types::{ epoch_state::EpochState, ledger_info::LedgerInfoWithSignatures, @@ -58,6 +58,7 @@ pub trait TExecutionClient: Send + Sync { /// Initialize the execution phase for a new epoch. async fn start_epoch( &self, + maybe_consensus_key: Option>, epoch_state: Arc, commit_signer_provider: Arc, payload_manager: Arc, @@ -183,6 +184,7 @@ impl ExecutionProxyClient { fn spawn_decoupled_execution( &self, + maybe_consensus_key: Option>, commit_signer_provider: Arc, epoch_state: Arc, rand_config: Option, @@ -215,10 +217,9 @@ impl ExecutionProxyClient { let (rand_ready_block_tx, rand_ready_block_rx) = unbounded::(); let (reset_tx_to_rand_manager, reset_rand_manager_rx) = unbounded::(); - let consensus_key = - load_consensus_key_from_secure_storage(&self.consensus_config.safety_rules) - .expect("Failed in loading consensus key for ExecutionProxyClient."); - let signer = Arc::new(ValidatorSigner::new(self.author, consensus_key)); + let consensus_sk = maybe_consensus_key + .expect("consensus key unavailable for ExecutionProxyClient"); + let signer = Arc::new(ValidatorSigner::new(self.author, consensus_sk)); let rand_manager = RandManager::::new( self.author, @@ -292,6 +293,7 @@ impl ExecutionProxyClient { impl TExecutionClient for ExecutionProxyClient { async fn start_epoch( &self, + maybe_consensus_key: Option>, epoch_state: Arc, commit_signer_provider: Arc, payload_manager: Arc, @@ -304,6 +306,7 @@ impl TExecutionClient for ExecutionProxyClient { highest_committed_round: Round, ) { let maybe_rand_msg_tx = self.spawn_decoupled_execution( + maybe_consensus_key, commit_signer_provider, epoch_state.clone(), rand_config, @@ -483,6 +486,7 @@ pub struct DummyExecutionClient; impl TExecutionClient for DummyExecutionClient { async fn start_epoch( &self, + _maybe_consensus_key: Option>, _epoch_state: Arc, _commit_signer_provider: Arc, _payload_manager: Arc, diff --git a/consensus/src/test_utils/mock_execution_client.rs b/consensus/src/test_utils/mock_execution_client.rs index 30063920d5d83..2649af9fd3b31 100644 --- a/consensus/src/test_utils/mock_execution_client.rs +++ b/consensus/src/test_utils/mock_execution_client.rs @@ -20,7 +20,7 @@ use aptos_consensus_types::{ common::{Payload, Round}, pipelined_block::PipelinedBlock, }; -use aptos_crypto::HashValue; +use aptos_crypto::{bls12381::PrivateKey, HashValue}; use aptos_executor_types::ExecutorResult; use aptos_infallible::Mutex; use aptos_logger::prelude::*; @@ -94,6 +94,7 @@ impl MockExecutionClient { impl TExecutionClient for MockExecutionClient { async fn start_epoch( &self, + _maybe_consensus_key: Option>, _epoch_state: Arc, _commit_signer_provider: Arc, _payload_manager: Arc, diff --git a/crates/aptos-dkg/src/weighted_vuf/pinkas/mod.rs b/crates/aptos-dkg/src/weighted_vuf/pinkas/mod.rs index fb83f422f0d7a..dd3795830f9ea 100644 --- a/crates/aptos-dkg/src/weighted_vuf/pinkas/mod.rs +++ b/crates/aptos-dkg/src/weighted_vuf/pinkas/mod.rs @@ -138,7 +138,7 @@ impl WeightedVUF for PinkasWUF { [&pks_combined, &pp.g_hat.neg()].into_iter(), ) != Gt::identity() { - bail!("RPKs were not correctly randomized."); + panic!("RPKs were not correctly randomized."); } Ok((delta, pk)) diff --git a/crates/aptos-genesis/src/builder.rs b/crates/aptos-genesis/src/builder.rs index 9a5b8586d7cbd..4117f9a4f2fe5 100644 --- a/crates/aptos-genesis/src/builder.rs +++ b/crates/aptos-genesis/src/builder.rs @@ -166,7 +166,11 @@ impl ValidatorNodeConfig { // Init safety rules let validator_identity_file = self.dir.join(VALIDATOR_IDENTITY); config.consensus.safety_rules.initial_safety_rules_config = - InitialSafetyRulesConfig::from_file(validator_identity_file, waypoint_config.clone()); + InitialSafetyRulesConfig::from_file( + validator_identity_file, + vec![], + waypoint_config.clone(), + ); config.base.waypoint = waypoint_config; } diff --git a/crates/aptos-jwk-consensus/Cargo.toml b/crates/aptos-jwk-consensus/Cargo.toml index b5bf11ad1948d..62f56a242aa74 100644 --- a/crates/aptos-jwk-consensus/Cargo.toml +++ b/crates/aptos-jwk-consensus/Cargo.toml @@ -29,6 +29,7 @@ aptos-metrics-core = { workspace = true } aptos-network = { workspace = true } aptos-reliable-broadcast = { workspace = true } aptos-runtimes = { workspace = true } +aptos-safety-rules = { workspace = true } aptos-time-service = { workspace = true } aptos-types = { workspace = true } aptos-validator-transaction-pool = { workspace = true } diff --git a/crates/aptos-jwk-consensus/src/epoch_manager.rs b/crates/aptos-jwk-consensus/src/epoch_manager.rs index 31609b06504fd..590435f65be77 100644 --- a/crates/aptos-jwk-consensus/src/epoch_manager.rs +++ b/crates/aptos-jwk-consensus/src/epoch_manager.rs @@ -8,11 +8,11 @@ use crate::{ types::JWKConsensusMsg, update_certifier::UpdateCertifier, }; -use anyhow::Result; +use anyhow::{anyhow, Result}; use aptos_bounded_executor::BoundedExecutor; use aptos_channels::{aptos_channel, message_queues::QueueStyle}; +use aptos_config::config::SafetyRulesConfig; use aptos_consensus_types::common::Author; -use aptos_crypto::bls12381::PrivateKey; use aptos_event_notifications::{ EventNotification, EventNotificationListener, ReconfigNotification, ReconfigNotificationListener, @@ -20,6 +20,7 @@ use aptos_event_notifications::{ use aptos_logger::{error, info}; use aptos_network::{application::interface::NetworkClient, protocols::network::Event}; use aptos_reliable_broadcast::ReliableBroadcast; +use aptos_safety_rules::{safety_rules_manager::storage, PersistentSafetyStorage}; use aptos_types::{ account_address::AccountAddress, epoch_state::EpochState, @@ -42,7 +43,7 @@ pub struct EpochManager { epoch_state: Option>, // credential - consensus_key: Arc, + key_storage: PersistentSafetyStorage, // events we subscribe reconfig_events: ReconfigNotificationListener

, @@ -65,7 +66,7 @@ pub struct EpochManager { impl EpochManager

{ pub fn new( my_addr: AccountAddress, - consensus_key: PrivateKey, + safety_rules_config: &SafetyRulesConfig, reconfig_events: ReconfigNotificationListener

, jwk_updated_events: EventNotificationListener, self_sender: aptos_channels::Sender>, @@ -74,7 +75,7 @@ impl EpochManager

{ ) -> Self { Self { my_addr, - consensus_key: Arc::new(consensus_key), + key_storage: storage(safety_rules_config), epoch_state: None, reconfig_events, jwk_updated_events, @@ -144,10 +145,11 @@ impl EpochManager

{ .await .expect("Reconfig sender dropped, unable to start new epoch"); self.start_new_epoch(reconfig_notification.on_chain_configs) - .await; + .await + .unwrap(); } - async fn start_new_epoch(&mut self, payload: OnChainConfigPayload

) { + async fn start_new_epoch(&mut self, payload: OnChainConfigPayload

) -> Result<()> { let validator_set: ValidatorSet = payload .get() .expect("failed to get ValidatorSet from payload"); @@ -210,9 +212,15 @@ impl EpochManager

{ BoundedExecutor::new(8, tokio::runtime::Handle::current()), ); let update_certifier = UpdateCertifier::new(rb); - + let my_pk = epoch_state + .verifier + .get_public_key(&self.my_addr) + .ok_or_else(|| anyhow!("my pk not found in validator set"))?; + let my_sk = self.key_storage.consensus_sk_by_pk(my_pk).map_err(|e| { + anyhow!("jwk-consensus new epoch handling failed with consensus sk lookup err: {e}") + })?; let jwk_consensus_manager = JWKManager::new( - self.consensus_key.clone(), + Arc::new(my_sk), self.my_addr, epoch_state.clone(), Arc::new(update_certifier), @@ -236,12 +244,13 @@ impl EpochManager

{ )); info!(epoch = epoch_state.epoch, "JWKManager spawned.",); } + Ok(()) } async fn on_new_epoch(&mut self, reconfig_notification: ReconfigNotification

) -> Result<()> { self.shutdown_current_processor().await; self.start_new_epoch(reconfig_notification.on_chain_configs) - .await; + .await?; Ok(()) } diff --git a/crates/aptos-jwk-consensus/src/lib.rs b/crates/aptos-jwk-consensus/src/lib.rs index 3e456e2cdcd6a..f2139ed5acbcc 100644 --- a/crates/aptos-jwk-consensus/src/lib.rs +++ b/crates/aptos-jwk-consensus/src/lib.rs @@ -5,7 +5,7 @@ use crate::{ epoch_manager::EpochManager, network::NetworkTask, network_interface::JWKConsensusNetworkClient, types::JWKConsensusMsg, }; -use aptos_crypto::bls12381::PrivateKey; +use aptos_config::config::SafetyRulesConfig; use aptos_event_notifications::{ DbBackedOnChainConfig, EventNotificationListener, ReconfigNotificationListener, }; @@ -17,7 +17,7 @@ use tokio::runtime::Runtime; #[allow(clippy::let_and_return)] pub fn start_jwk_consensus_runtime( my_addr: AccountAddress, - consensus_key: PrivateKey, + safety_rules_config: &SafetyRulesConfig, network_client: NetworkClient, network_service_events: NetworkServiceEvents, reconfig_events: ReconfigNotificationListener, @@ -29,7 +29,7 @@ pub fn start_jwk_consensus_runtime( let jwk_consensus_network_client = JWKConsensusNetworkClient::new(network_client); let epoch_manager = EpochManager::new( my_addr, - consensus_key, + safety_rules_config, reconfig_events, jwk_updated_events, self_sender, diff --git a/crates/aptos/src/test/mod.rs b/crates/aptos/src/test/mod.rs index 3012b0c148e65..2d3fdab434540 100644 --- a/crates/aptos/src/test/mod.rs +++ b/crates/aptos/src/test/mod.rs @@ -536,9 +536,10 @@ impl CliTestFramework { pool_index: Option, consensus_public_key: bls12381::PublicKey, proof_of_possession: bls12381::ProofOfPossession, + gas_options: Option, ) -> CliTypedResult { UpdateConsensusKey { - txn_options: self.transaction_options(operator_index, None), + txn_options: self.transaction_options(operator_index, gas_options), operator_args: self.operator_args(pool_index), operator_config_file_args: OperatorConfigFileArgs { operator_config_file: None, diff --git a/dkg/Cargo.toml b/dkg/Cargo.toml index 94bc2d04b0e5a..20213ec74ad2d 100644 --- a/dkg/Cargo.toml +++ b/dkg/Cargo.toml @@ -28,6 +28,7 @@ aptos-metrics-core = { workspace = true } aptos-network = { workspace = true } aptos-reliable-broadcast = { workspace = true } aptos-runtimes = { workspace = true } +aptos-safety-rules = { workspace = true } aptos-time-service = { workspace = true } aptos-types = { workspace = true } aptos-validator-transaction-pool = { workspace = true } diff --git a/dkg/src/epoch_manager.rs b/dkg/src/epoch_manager.rs index 464b9a98d471c..94ef6309cc89f 100644 --- a/dkg/src/epoch_manager.rs +++ b/dkg/src/epoch_manager.rs @@ -8,10 +8,10 @@ use crate::{ network_interface::DKGNetworkClient, DKGMessage, }; -use anyhow::Result; +use anyhow::{anyhow, Result}; use aptos_bounded_executor::BoundedExecutor; use aptos_channels::{aptos_channel, message_queues::QueueStyle}; -use aptos_config::config::ReliableBroadcastConfig; +use aptos_config::config::{ReliableBroadcastConfig, SafetyRulesConfig}; use aptos_event_notifications::{ EventNotification, EventNotificationListener, ReconfigNotification, ReconfigNotificationListener, @@ -19,9 +19,10 @@ use aptos_event_notifications::{ use aptos_logger::{debug, error, info, warn}; use aptos_network::{application::interface::NetworkClient, protocols::network::Event}; use aptos_reliable_broadcast::ReliableBroadcast; +use aptos_safety_rules::{safety_rules_manager::storage, PersistentSafetyStorage}; use aptos_types::{ account_address::AccountAddress, - dkg::{DKGStartEvent, DKGState, DKGTrait, DefaultDKG}, + dkg::{DKGStartEvent, DKGState, DefaultDKG}, epoch_state::EpochState, on_chain_config::{ OnChainConfigPayload, OnChainConfigProvider, OnChainConsensusConfig, @@ -35,7 +36,6 @@ use std::{sync::Arc, time::Duration}; use tokio_retry::strategy::ExponentialBackoff; pub struct EpochManager { - dkg_dealer_sk: Arc<::DealerPrivateKey>, // Some useful metadata my_addr: AccountAddress, epoch_state: Option>, @@ -58,12 +58,14 @@ pub struct EpochManager { // Randomness overriding. randomness_override_seq_num: u64, + + key_storage: PersistentSafetyStorage, } impl EpochManager

{ pub fn new( + safety_rules_config: &SafetyRulesConfig, my_addr: AccountAddress, - dkg_dealer_sk: ::DealerPrivateKey, reconfig_events: ReconfigNotificationListener

, dkg_start_events: EventNotificationListener, self_sender: aptos_channels::Sender>, @@ -73,7 +75,6 @@ impl EpochManager

{ randomness_override_seq_num: u64, ) -> Self { Self { - dkg_dealer_sk: Arc::new(dkg_dealer_sk), my_addr, epoch_state: None, reconfig_events, @@ -86,6 +87,7 @@ impl EpochManager

{ dkg_start_event_tx: None, rb_config, randomness_override_seq_num, + key_storage: storage(safety_rules_config), } } @@ -148,10 +150,11 @@ impl EpochManager

{ .await .expect("Reconfig sender dropped, unable to start new epoch"); self.start_new_epoch(reconfig_notification.on_chain_configs) - .await; + .await + .unwrap(); } - async fn start_new_epoch(&mut self, payload: OnChainConfigPayload

) { + async fn start_new_epoch(&mut self, payload: OnChainConfigPayload

) -> Result<()> { let validator_set: ValidatorSet = payload .get() .expect("failed to get ValidatorSet from payload"); @@ -231,9 +234,15 @@ impl EpochManager

{ self.dkg_rpc_msg_tx = Some(dkg_rpc_msg_tx); let (dkg_manager_close_tx, dkg_manager_close_rx) = oneshot::channel(); self.dkg_manager_close_tx = Some(dkg_manager_close_tx); - + let my_pk = epoch_state + .verifier + .get_public_key(&self.my_addr) + .ok_or_else(|| anyhow!("my pk not found in validator set"))?; + let dealer_sk = self.key_storage.consensus_sk_by_pk(my_pk).map_err(|e| { + anyhow!("dkg new epoch handling failed with consensus sk lookup err: {e}") + })?; let dkg_manager = DKGManager::::new( - self.dkg_dealer_sk.clone(), + Arc::new(dealer_sk), my_index, self.my_addr, epoch_state, @@ -246,13 +255,14 @@ impl EpochManager

{ dkg_rpc_msg_rx, dkg_manager_close_rx, )); - } + }; + Ok(()) } async fn on_new_epoch(&mut self, reconfig_notification: ReconfigNotification

) -> Result<()> { self.shutdown_current_processor().await; self.start_new_epoch(reconfig_notification.on_chain_configs) - .await; + .await?; Ok(()) } diff --git a/dkg/src/lib.rs b/dkg/src/lib.rs index 819e69bf7a808..3fd39eb80adc6 100644 --- a/dkg/src/lib.rs +++ b/dkg/src/lib.rs @@ -13,12 +13,11 @@ pub mod types; use crate::{ epoch_manager::EpochManager, network::NetworkTask, network_interface::DKGNetworkClient, }; -use aptos_config::config::ReliableBroadcastConfig; +use aptos_config::config::{ReliableBroadcastConfig, SafetyRulesConfig}; use aptos_event_notifications::{ DbBackedOnChainConfig, EventNotificationListener, ReconfigNotificationListener, }; use aptos_network::application::interface::{NetworkClient, NetworkServiceEvents}; -use aptos_types::dkg::{DKGTrait, DefaultDKG}; use aptos_validator_transaction_pool::VTxnPoolState; use move_core_types::account_address::AccountAddress; use tokio::runtime::Runtime; @@ -26,7 +25,7 @@ pub use types::DKGMessage; pub fn start_dkg_runtime( my_addr: AccountAddress, - dkg_dealer_sk: ::DealerPrivateKey, + safety_rules_config: &SafetyRulesConfig, network_client: NetworkClient, network_service_events: NetworkServiceEvents, reconfig_events: ReconfigNotificationListener, @@ -40,8 +39,8 @@ pub fn start_dkg_runtime( let dkg_network_client = DKGNetworkClient::new(network_client); let dkg_epoch_manager = EpochManager::new( + safety_rules_config, my_addr, - dkg_dealer_sk, reconfig_events, dkg_start_events, self_sender, diff --git a/execution/executor-test-helpers/src/integration_test_impl.rs b/execution/executor-test-helpers/src/integration_test_impl.rs index e0ee3a0d1fb1a..8aceb408eaaf5 100644 --- a/execution/executor-test-helpers/src/integration_test_impl.rs +++ b/execution/executor-test-helpers/src/integration_test_impl.rs @@ -69,7 +69,7 @@ pub fn test_execution_with_storage_impl_inner( let parent_block_id = executor.committed_block_id(); let signer = aptos_types::validator_signer::ValidatorSigner::new( validators[0].data.owner_address, - validators[0].consensus_key.clone(), + Arc::new(validators[0].consensus_key.clone()), ); // This generates accounts that do not overlap with genesis diff --git a/execution/executor-test-helpers/src/lib.rs b/execution/executor-test-helpers/src/lib.rs index 0338814134ff3..8bf2adcf0ea5e 100644 --- a/execution/executor-test-helpers/src/lib.rs +++ b/execution/executor-test-helpers/src/lib.rs @@ -22,6 +22,7 @@ use aptos_types::{ waypoint::Waypoint, }; use aptos_vm::VMExecutor; +use std::sync::Arc; /// Helper function for test to blindly bootstrap without waypoint. pub fn bootstrap_genesis( @@ -62,7 +63,7 @@ pub fn extract_signer(config: &mut NodeConfig) -> ValidatorSigner { let sr_test = config.consensus.safety_rules.test.as_ref().unwrap(); ValidatorSigner::new( sr_test.author, - sr_test.consensus_key.as_ref().unwrap().private_key(), + Arc::new(sr_test.consensus_key.as_ref().unwrap().private_key()), ) } diff --git a/execution/executor/tests/db_bootstrapper_test.rs b/execution/executor/tests/db_bootstrapper_test.rs index 8fa93a668e9bf..a279398e49e48 100644 --- a/execution/executor/tests/db_bootstrapper_test.rs +++ b/execution/executor/tests/db_bootstrapper_test.rs @@ -36,6 +36,7 @@ use aptos_types::{ use aptos_vm::AptosVM; use move_core_types::{language_storage::TypeTag, move_resource::MoveStructType}; use rand::SeedableRng; +use std::sync::Arc; #[test] fn test_empty_db() { @@ -189,7 +190,7 @@ fn test_new_genesis() { let waypoint = bootstrap_genesis::(&db, &genesis_txn).unwrap(); let signer = ValidatorSigner::new( genesis.1[0].data.owner_address, - genesis.1[0].consensus_key.clone(), + Arc::new(genesis.1[0].consensus_key.clone()), ); // Mint for 2 demo accounts. diff --git a/execution/executor/tests/internal_indexer_test.rs b/execution/executor/tests/internal_indexer_test.rs index 7e85cc90c9752..397a4bde92c02 100644 --- a/execution/executor/tests/internal_indexer_test.rs +++ b/execution/executor/tests/internal_indexer_test.rs @@ -53,7 +53,7 @@ pub fn create_test_db() -> (Arc, LocalAccount) { let mut rng = ::rand::rngs::StdRng::from_seed(seed); let signer = aptos_types::validator_signer::ValidatorSigner::new( validators[0].data.owner_address, - validators[0].consensus_key.clone(), + Arc::new(validators[0].consensus_key.clone()), ); let account1 = LocalAccount::generate(&mut rng); let account2 = LocalAccount::generate(&mut rng); diff --git a/execution/executor/tests/storage_integration_test.rs b/execution/executor/tests/storage_integration_test.rs index 7a037cb4f83d8..33e077dc06b83 100644 --- a/execution/executor/tests/storage_integration_test.rs +++ b/execution/executor/tests/storage_integration_test.rs @@ -25,6 +25,7 @@ use aptos_types::{ validator_config::ValidatorConfig, validator_signer::ValidatorSigner, }; +use std::sync::Arc; #[test] fn test_genesis() { @@ -79,7 +80,7 @@ fn test_reconfiguration() { let parent_block_id = executor.committed_block_id(); let signer = ValidatorSigner::new( validators[0].data.owner_address, - validators[0].consensus_key.clone(), + Arc::new(validators[0].consensus_key.clone()), ); let validator_account = signer.author(); diff --git a/testsuite/smoke-test/Cargo.toml b/testsuite/smoke-test/Cargo.toml index 16dfafe0f632b..fb92ab47b09c4 100644 --- a/testsuite/smoke-test/Cargo.toml +++ b/testsuite/smoke-test/Cargo.toml @@ -17,7 +17,7 @@ anyhow = { workspace = true } aptos = { workspace = true, features = ["fuzzing"] } aptos-bitvec = { path = "../../crates/aptos-bitvec" } aptos-cached-packages = { workspace = true } -aptos-config = { workspace = true } +aptos-config = { workspace = true, features = ["smoke-test"] } aptos-consensus = { workspace = true } aptos-crypto = { workspace = true } aptos-db = { workspace = true } diff --git a/testsuite/smoke-test/src/aptos_cli/validator.rs b/testsuite/smoke-test/src/aptos_cli/validator.rs index 69e7bd3429074..45c835e2cb1b5 100644 --- a/testsuite/smoke-test/src/aptos_cli/validator.rs +++ b/testsuite/smoke-test/src/aptos_cli/validator.rs @@ -1340,6 +1340,7 @@ async fn test_owner_create_and_delegate_flow() { Some(owner_cli_index), operator_keys.consensus_public_key(), operator_keys.consensus_proof_of_possession(), + None, ) .await .unwrap(), diff --git a/testsuite/smoke-test/src/consensus_key_rotation.rs b/testsuite/smoke-test/src/consensus_key_rotation.rs new file mode 100644 index 0000000000000..160abb5b846d0 --- /dev/null +++ b/testsuite/smoke-test/src/consensus_key_rotation.rs @@ -0,0 +1,192 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use crate::{smoke_test_environment::SwarmBuilder, utils::get_on_chain_resource}; +use anyhow::bail; +use aptos::common::types::GasOptions; +use aptos_config::config::{OverrideNodeConfig, PersistableConfig}; +use aptos_crypto::{bls12381, Uniform}; +use aptos_forge::{NodeExt, Swarm, SwarmExt}; +use aptos_logger::info; +use aptos_rest_client::Client; +use aptos_types::{ + on_chain_config::{ConfigurationResource, OnChainRandomnessConfig, ValidatorSet}, + validator_verifier::ValidatorVerifier, +}; +use rand::{thread_rng, Rng}; +use std::{ + fs::File, + io::Write, + ops::Add, + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, +}; + +#[tokio::test] +async fn consensus_key_rotation() { + let epoch_duration_secs = 60; + let n = 2; + let (mut swarm, mut cli, _faucet) = SwarmBuilder::new_local(n) + .with_aptos() + .with_init_genesis_config(Arc::new(move |conf| { + conf.epoch_duration_secs = epoch_duration_secs; + + // Ensure randomness is enabled. + conf.consensus_config.enable_validator_txns(); + conf.randomness_config_override = Some(OnChainRandomnessConfig::default_enabled()); + })) + .build_with_cli(0) + .await; + + let rest_client = swarm.validators().next().unwrap().rest_client(); + + info!("Wait for epoch 3."); + wait_until_epoch( + &rest_client, + 3, + Duration::from_secs(epoch_duration_secs * 2), + ) + .await + .unwrap(); + info!("Epoch 3 arrived."); + + let (operator_addr, new_pk, pop, operator_idx) = + if let Some(validator) = swarm.validators_mut().nth(n - 1) { + let operator_sk = validator + .account_private_key() + .as_ref() + .unwrap() + .private_key(); + let operator_idx = cli.add_account_to_cli(operator_sk); + info!("Stopping the last node."); + + validator.stop(); + tokio::time::sleep(Duration::from_secs(5)).await; + + let new_identity_path = PathBuf::from( + format!( + "/tmp/{}-new-validator-identity.yaml", + thread_rng().gen::() + ) + .as_str(), + ); + info!( + "Generating and writing new validator identity to {:?}.", + new_identity_path + ); + let new_sk = bls12381::PrivateKey::generate(&mut thread_rng()); + let pop = bls12381::ProofOfPossession::create(&new_sk); + let new_pk = bls12381::PublicKey::from(&new_sk); + let mut validator_identity_blob = validator + .config() + .consensus + .safety_rules + .initial_safety_rules_config + .identity_blob() + .unwrap(); + validator_identity_blob.consensus_private_key = Some(new_sk); + let operator_addr = validator_identity_blob.account_address.unwrap(); + + Write::write_all( + &mut File::create(&new_identity_path).unwrap(), + serde_yaml::to_string(&validator_identity_blob) + .unwrap() + .as_bytes(), + ) + .unwrap(); + + info!("Updating the node config accordingly."); + let config_path = validator.config_path(); + let mut validator_override_config = + OverrideNodeConfig::load_config(config_path.clone()).unwrap(); + validator_override_config + .override_config_mut() + .consensus + .safety_rules + .initial_safety_rules_config + .overriding_identity_blob_paths_mut() + .push(new_identity_path); + validator_override_config.save_config(config_path).unwrap(); + + info!("Restarting the node."); + validator.start().unwrap(); + info!("Let it bake for 5 secs."); + tokio::time::sleep(Duration::from_secs(5)).await; + (operator_addr, new_pk, pop, operator_idx) + } else { + unreachable!() + }; + + info!("Update on-chain. Retry is needed in case randomness is enabled."); + swarm + .chain_info() + .into_aptos_public_info() + .mint(operator_addr, 99999999999) + .await + .unwrap(); + let mut attempts = 10; + while attempts > 0 { + attempts -= 1; + let gas_options = GasOptions { + gas_unit_price: Some(100), + max_gas: Some(200000), + expiration_secs: 60, + }; + let update_result = cli + .update_consensus_key( + operator_idx, + None, + new_pk.clone(), + pop.clone(), + Some(gas_options), + ) + .await; + info!("update_result={:?}", update_result); + if let Ok(txn_smry) = update_result { + if txn_smry.success == Some(true) { + break; + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + + assert!(attempts >= 1); + + info!("Wait for epoch 5."); + wait_until_epoch( + &rest_client, + 5, + Duration::from_secs(epoch_duration_secs * 2), + ) + .await + .unwrap(); + info!("Epoch 5 arrived."); + + info!("All nodes should be alive."); + let liveness_check_result = swarm + .liveness_check(Instant::now().add(Duration::from_secs(30))) + .await; + assert!(liveness_check_result.is_ok()); + + info!("On-chain pk should be updated."); + let validator_set = get_on_chain_resource::(&rest_client).await; + let verifier = ValidatorVerifier::from(&validator_set); + assert_eq!(new_pk, verifier.get_public_key(&operator_addr).unwrap()); +} + +async fn wait_until_epoch( + rest_cli: &Client, + target_epoch: u64, + time_limit: Duration, +) -> anyhow::Result<()> { + let timer = Instant::now(); + while timer.elapsed() < time_limit { + let c = get_on_chain_resource::(rest_cli).await; + if c.epoch() >= target_epoch { + return Ok(()); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + bail!(""); +} diff --git a/testsuite/smoke-test/src/lib.rs b/testsuite/smoke-test/src/lib.rs index fe96bb83e7ea3..2df5effb1d7a3 100644 --- a/testsuite/smoke-test/src/lib.rs +++ b/testsuite/smoke-test/src/lib.rs @@ -13,6 +13,8 @@ mod client; #[cfg(test)] mod consensus; #[cfg(test)] +mod consensus_key_rotation; +#[cfg(test)] mod consensus_observer; #[cfg(test)] mod execution; diff --git a/testsuite/smoke-test/src/randomness/disable_feature_0.rs b/testsuite/smoke-test/src/randomness/disable_feature_0.rs index f004a95ebb187..623b48f0476a8 100644 --- a/testsuite/smoke-test/src/randomness/disable_feature_0.rs +++ b/testsuite/smoke-test/src/randomness/disable_feature_0.rs @@ -2,10 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - randomness::{ - decrypt_key_map, get_on_chain_resource, script_to_disable_main_logic, verify_dkg_transcript, - }, + randomness::{decrypt_key_map, script_to_disable_main_logic, verify_dkg_transcript}, smoke_test_environment::SwarmBuilder, + utils::get_on_chain_resource, }; use aptos_forge::{Node, Swarm, SwarmExt}; use aptos_logger::{debug, info}; diff --git a/testsuite/smoke-test/src/randomness/disable_feature_1.rs b/testsuite/smoke-test/src/randomness/disable_feature_1.rs index efcfb5c5f3c53..6b34550f59077 100644 --- a/testsuite/smoke-test/src/randomness/disable_feature_1.rs +++ b/testsuite/smoke-test/src/randomness/disable_feature_1.rs @@ -2,12 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - randomness::{ - decrypt_key_map, get_on_chain_resource, script_to_update_consensus_config, - verify_dkg_transcript, - }, + randomness::{decrypt_key_map, script_to_update_consensus_config, verify_dkg_transcript}, smoke_test_environment::SwarmBuilder, - utils::get_current_consensus_config, + utils::{get_current_consensus_config, get_on_chain_resource}, }; use aptos_forge::{Node, Swarm, SwarmExt}; use aptos_logger::{debug, info}; diff --git a/testsuite/smoke-test/src/randomness/e2e_correctness.rs b/testsuite/smoke-test/src/randomness/e2e_correctness.rs index eef1cb35770fe..3a329d273c5f1 100644 --- a/testsuite/smoke-test/src/randomness/e2e_correctness.rs +++ b/testsuite/smoke-test/src/randomness/e2e_correctness.rs @@ -2,11 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - randomness::{ - decrypt_key_map, get_current_version, get_on_chain_resource, verify_dkg_transcript, - verify_randomness, - }, + randomness::{decrypt_key_map, get_current_version, verify_dkg_transcript, verify_randomness}, smoke_test_environment::SwarmBuilder, + utils::get_on_chain_resource, }; use aptos_forge::{NodeExt, SwarmExt}; use aptos_logger::info; diff --git a/testsuite/smoke-test/src/randomness/enable_feature_0.rs b/testsuite/smoke-test/src/randomness/enable_feature_0.rs index 1f7e47d1d1445..96621901ede5b 100644 --- a/testsuite/smoke-test/src/randomness/enable_feature_0.rs +++ b/testsuite/smoke-test/src/randomness/enable_feature_0.rs @@ -3,11 +3,11 @@ use crate::{ randomness::{ - decrypt_key_map, get_on_chain_resource, script_to_enable_main_logic, - script_to_update_consensus_config, verify_dkg_transcript, + decrypt_key_map, script_to_enable_main_logic, script_to_update_consensus_config, + verify_dkg_transcript, }, smoke_test_environment::SwarmBuilder, - utils::get_current_consensus_config, + utils::{get_current_consensus_config, get_on_chain_resource}, }; use aptos_forge::{Node, Swarm, SwarmExt}; use aptos_logger::{debug, info}; diff --git a/testsuite/smoke-test/src/randomness/enable_feature_1.rs b/testsuite/smoke-test/src/randomness/enable_feature_1.rs index 2288f1f16b57d..97a26e6fbb961 100644 --- a/testsuite/smoke-test/src/randomness/enable_feature_1.rs +++ b/testsuite/smoke-test/src/randomness/enable_feature_1.rs @@ -3,11 +3,11 @@ use crate::{ randomness::{ - decrypt_key_map, get_on_chain_resource, script_to_enable_main_logic, - script_to_update_consensus_config, verify_dkg_transcript, + decrypt_key_map, script_to_enable_main_logic, script_to_update_consensus_config, + verify_dkg_transcript, }, smoke_test_environment::SwarmBuilder, - utils::get_current_consensus_config, + utils::{get_current_consensus_config, get_on_chain_resource}, }; use aptos_forge::{Node, Swarm, SwarmExt}; use aptos_logger::{debug, info}; diff --git a/testsuite/smoke-test/src/randomness/enable_feature_2.rs b/testsuite/smoke-test/src/randomness/enable_feature_2.rs index 3f008e9fe9c78..f9f291b8d0901 100644 --- a/testsuite/smoke-test/src/randomness/enable_feature_2.rs +++ b/testsuite/smoke-test/src/randomness/enable_feature_2.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - randomness::{decrypt_key_map, get_on_chain_resource, verify_dkg_transcript}, + randomness::{decrypt_key_map, verify_dkg_transcript}, smoke_test_environment::SwarmBuilder, - utils::get_current_consensus_config, + utils::{get_current_consensus_config, get_on_chain_resource}, }; use aptos_forge::{Node, Swarm, SwarmExt}; use aptos_logger::{debug, info}; diff --git a/testsuite/smoke-test/src/randomness/mod.rs b/testsuite/smoke-test/src/randomness/mod.rs index 5796ee7a31106..b7ef35055c397 100644 --- a/testsuite/smoke-test/src/randomness/mod.rs +++ b/testsuite/smoke-test/src/randomness/mod.rs @@ -1,6 +1,7 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 +use crate::utils; use anyhow::{anyhow, ensure, Result}; use aptos_crypto::{compat::Sha3_256, Uniform}; use aptos_dkg::weighted_vuf::traits::WeightedVUF; @@ -42,14 +43,6 @@ async fn get_current_version(rest_client: &Client) -> u64 { .version } -async fn get_on_chain_resource(rest_client: &Client) -> T { - let maybe_response = rest_client - .get_account_resource_bcs::(CORE_CODE_ADDRESS, T::struct_tag().to_string().as_str()) - .await; - let response = maybe_response.unwrap(); - response.into_inner() -} - #[allow(dead_code)] async fn get_on_chain_resource_at_version( rest_client: &Client, @@ -74,7 +67,7 @@ async fn wait_for_dkg_finish( target_epoch: Option, time_limit_secs: u64, ) -> DKGSessionState { - let mut dkg_state = get_on_chain_resource::(client).await; + let mut dkg_state = utils::get_on_chain_resource::(client).await; let timer = Instant::now(); while timer.elapsed().as_secs() < time_limit_secs && !(dkg_state.in_progress.is_none() @@ -87,7 +80,7 @@ async fn wait_for_dkg_finish( == target_epoch)) { tokio::time::sleep(Duration::from_secs(1)).await; - dkg_state = get_on_chain_resource::(client).await; + dkg_state = utils::get_on_chain_resource::(client).await; } assert!(timer.elapsed().as_secs() < time_limit_secs); dkg_state.last_complete().clone() diff --git a/testsuite/smoke-test/src/randomness/randomness_stall_recovery.rs b/testsuite/smoke-test/src/randomness/randomness_stall_recovery.rs index aa71e069bd3d4..2691c426c3f62 100644 --- a/testsuite/smoke-test/src/randomness/randomness_stall_recovery.rs +++ b/testsuite/smoke-test/src/randomness/randomness_stall_recovery.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - genesis::enable_sync_only_mode, randomness::get_on_chain_resource, - smoke_test_environment::SwarmBuilder, + genesis::enable_sync_only_mode, smoke_test_environment::SwarmBuilder, + utils::get_on_chain_resource, }; use aptos::common::types::GasOptions; use aptos_config::config::{OverrideNodeConfig, PersistableConfig}; diff --git a/testsuite/smoke-test/src/randomness/validator_restart_during_dkg.rs b/testsuite/smoke-test/src/randomness/validator_restart_during_dkg.rs index 8f437f01769c7..e4309b6b1d1c9 100644 --- a/testsuite/smoke-test/src/randomness/validator_restart_during_dkg.rs +++ b/testsuite/smoke-test/src/randomness/validator_restart_during_dkg.rs @@ -2,10 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - randomness::{ - decrypt_key_map, get_on_chain_resource, verify_dkg_transcript, wait_for_dkg_finish, - }, + randomness::{decrypt_key_map, verify_dkg_transcript, wait_for_dkg_finish}, smoke_test_environment::SwarmBuilder, + utils::get_on_chain_resource, }; use aptos_forge::{NodeExt, SwarmExt}; use aptos_logger::{debug, info}; diff --git a/testsuite/smoke-test/src/utils.rs b/testsuite/smoke-test/src/utils.rs index dc64bdb686b1c..397bbfb310978 100644 --- a/testsuite/smoke-test/src/utils.rs +++ b/testsuite/smoke-test/src/utils.rs @@ -4,12 +4,12 @@ use aptos_cached_packages::aptos_stdlib; use aptos_forge::{reconfig, LocalSwarm, NodeExt, Swarm, SwarmExt}; -use aptos_rest_client::Client as RestClient; +use aptos_rest_client::{Client as RestClient, Client}; use aptos_sdk::{ transaction_builder::TransactionFactory, types::{transaction::SignedTransaction, LocalAccount}, }; -use aptos_types::on_chain_config::{OnChainConsensusConfig, OnChainExecutionConfig}; +use aptos_types::on_chain_config::{OnChainConfig, OnChainConsensusConfig, OnChainExecutionConfig}; use move_core_types::language_storage::CORE_CODE_ADDRESS; use rand::random; use std::{sync::Arc, time::Duration}; @@ -220,6 +220,14 @@ pub async fn get_current_version(rest_client: &RestClient) -> u64 { .version } +pub async fn get_on_chain_resource(rest_client: &Client) -> T { + let maybe_response = rest_client + .get_account_resource_bcs::(CORE_CODE_ADDRESS, T::struct_tag().to_string().as_str()) + .await; + let response = maybe_response.unwrap(); + response.into_inner() +} + #[cfg(test)] pub mod swarm_utils { use aptos_config::config::{NodeConfig, SecureBackend, WaypointConfig}; diff --git a/types/src/proptest_types.rs b/types/src/proptest_types.rs index 3605e6e88f8f3..44b82289992d6 100644 --- a/types/src/proptest_types.rs +++ b/types/src/proptest_types.rs @@ -54,6 +54,7 @@ use serde_json::Value; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, iter::Iterator, + sync::Arc, }; impl WriteOp { @@ -193,7 +194,7 @@ impl AccountInfoUniverse { accounts.sort_by(|a, b| a.address.cmp(&b.address)); let validator_signer = ValidatorSigner::new( accounts[0].address, - accounts[0].consensus_private_key.clone(), + Arc::new(accounts[0].consensus_private_key.clone()), ); let validator_set_by_epoch = vec![(0, vec![validator_signer])].into_iter().collect(); @@ -1017,7 +1018,10 @@ impl ValidatorSetGen { .get_account_infos_dedup(&self.validators) .iter() .map(|account| { - ValidatorSigner::new(account.address, account.consensus_private_key.clone()) + ValidatorSigner::new( + account.address, + Arc::new(account.consensus_private_key.clone()), + ) }) .collect() } diff --git a/types/src/validator_signer.rs b/types/src/validator_signer.rs index 8414012654705..7248e66992f2f 100644 --- a/types/src/validator_signer.rs +++ b/types/src/validator_signer.rs @@ -9,7 +9,7 @@ use aptos_crypto::{ }; use rand::{rngs::StdRng, SeedableRng}; use serde::ser::Serialize; -use std::convert::TryFrom; +use std::{convert::TryFrom, sync::Arc}; /// ValidatorSigner associates an author with public and private keys with helpers for signing and /// validating. This struct can be used for all signing operations including block and network @@ -18,11 +18,11 @@ use std::convert::TryFrom; #[cfg_attr(any(test, feature = "fuzzing"), derive(Clone))] pub struct ValidatorSigner { author: AccountAddress, - private_key: bls12381::PrivateKey, + private_key: Arc, } impl ValidatorSigner { - pub fn new(author: AccountAddress, private_key: bls12381::PrivateKey) -> Self { + pub fn new(author: AccountAddress, private_key: Arc) -> Self { ValidatorSigner { author, private_key, @@ -50,7 +50,7 @@ impl ValidatorSigner { /// Returns the private key associated with this signer. Only available for testing purposes. #[cfg(any(test, feature = "fuzzing"))] pub fn private_key(&self) -> &bls12381::PrivateKey { - &self.private_key + self.private_key.as_ref() } } @@ -63,7 +63,7 @@ impl ValidatorSigner { let mut rng = StdRng::from_seed(opt_rng_seed.into().unwrap_or(TEST_SEED)); Self::new( AccountAddress::random(), - bls12381::PrivateKey::generate(&mut rng), + Arc::new(bls12381::PrivateKey::generate(&mut rng)), ) } @@ -73,7 +73,10 @@ impl ValidatorSigner { let mut address = [0; AccountAddress::LENGTH]; address[0] = num; let private_key = bls12381::PrivateKey::generate_for_testing(); - Self::new(AccountAddress::try_from(&address[..]).unwrap(), private_key) + Self::new( + AccountAddress::try_from(&address[..]).unwrap(), + Arc::new(private_key), + ) } } @@ -99,7 +102,7 @@ pub mod proptests { signing_key_strategy.prop_map(|signing_key| { ValidatorSigner::new( AccountAddress::from_bytes(&signing_key.public_key().to_bytes()[0..32]).unwrap(), - signing_key, + Arc::new(signing_key), ) }) } @@ -115,7 +118,7 @@ pub mod proptests { rand_signer(), LazyJust::new(|| { let genesis_key = bls12381::PrivateKey::genesis(); - ValidatorSigner::new(AccountAddress::random(), genesis_key) + ValidatorSigner::new(AccountAddress::random(), Arc::new(genesis_key)) }) ] } @@ -139,7 +142,7 @@ pub mod proptests { #[test] fn test_new_signer(signing_key in arb_signing_key()){ let public_key = signing_key.public_key(); - let signer = ValidatorSigner::new(AccountAddress::random(), signing_key); + let signer = ValidatorSigner::new(AccountAddress::random(), Arc::new(signing_key)); prop_assert_eq!(public_key, signer.public_key()); }