diff --git a/Cargo.lock b/Cargo.lock index 0cf0541fa5f7..04eb1f7b1fea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7254,9 +7254,11 @@ name = "polkadot-approval-distribution" version = "0.9.43" dependencies = [ "assert_matches", + "bitvec", "env_logger 0.9.3", "futures", "futures-timer", + "itertools", "log", "polkadot-node-jaeger", "polkadot-node-metrics", @@ -7559,10 +7561,13 @@ dependencies = [ "async-trait", "bitvec", "derive_more", + "env_logger 0.9.3", "futures", "futures-timer", + "itertools", "kvdb", "kvdb-memorydb", + "log", "lru 0.11.0", "merlin 2.0.1", "parity-scale-codec", @@ -8005,6 +8010,7 @@ dependencies = [ name = "polkadot-node-primitives" version = "0.9.43" dependencies = [ + "bitvec", "bounded-vec", "futures", "parity-scale-codec", @@ -8055,6 +8061,7 @@ name = "polkadot-node-subsystem-types" version = "0.9.43" dependencies = [ "async-trait", + "bitvec", "derive_more", "futures", "orchestra", diff --git a/Cargo.toml b/Cargo.toml index dc42123a9f20..e544c4aa6075 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -226,6 +226,8 @@ fast-runtime = [ "polkadot-cli/fast-runtime" ] runtime-metrics = [ "polkadot-cli/runtime-metrics" ] pyroscope = ["polkadot-cli/pyroscope"] jemalloc-allocator = ["polkadot-node-core-pvf-prepare-worker/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator"] +network-protocol-staging = ["polkadot-cli/network-protocol-staging"] + # Enables timeout-based tests supposed to be run only in CI environment as they may be flaky # when run locally depending on system load ci-only-tests = ["polkadot-node-core-pvf/ci-only-tests"] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index df1a22d8c18d..ae9cf51e1282 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -75,3 +75,4 @@ rococo-native = ["service/rococo-native"] malus = ["full-node", "service/malus"] runtime-metrics = ["service/runtime-metrics", "polkadot-node-metrics/runtime-metrics"] +network-protocol-staging = ["service/network-protocol-staging"] \ No newline at end of file diff --git a/node/core/approval-voting/Cargo.toml b/node/core/approval-voting/Cargo.toml index f31c7d0a1694..8748eaa0bec4 100644 --- a/node/core/approval-voting/Cargo.toml +++ b/node/core/approval-voting/Cargo.toml @@ -17,6 +17,7 @@ schnorrkel = "0.9.1" kvdb = "0.13.0" derive_more = "0.99.17" thiserror = "1.0.31" +itertools = "0.10.5" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } @@ -30,6 +31,9 @@ sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "mast sp-consensus-slots = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = ["full_crypto"] } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +# Needed for migration test helpers +test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +rand_core = "0.5.1" [dev-dependencies] async-trait = "0.1.57" @@ -43,3 +47,5 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } assert_matches = "1.4.0" kvdb-memorydb = "0.13.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +log = "0.4.17" +env_logger = "0.9.0" diff --git a/node/core/approval-voting/src/approval_checking.rs b/node/core/approval-voting/src/approval_checking.rs index f345b57029b5..2f0ea4160ae1 100644 --- a/node/core/approval-voting/src/approval_checking.rs +++ b/node/core/approval-voting/src/approval_checking.rs @@ -17,7 +17,7 @@ //! Utilities for checking whether a candidate has been approved under a given block. use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use polkadot_primitives::ValidatorIndex; use crate::{ @@ -64,7 +64,7 @@ pub enum RequiredTranches { } /// The result of a check. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum Check { /// The candidate is unapproved. Unapproved, @@ -372,19 +372,22 @@ pub fn tranches_to_approve( block_tick: Tick, no_show_duration: Tick, needed_approvals: usize, -) -> RequiredTranches { +) -> (RequiredTranches, usize) { let tick_now = tranche_now as Tick + block_tick; let n_validators = approval_entry.n_validators(); - let initial_state = State { - assignments: 0, - depth: 0, - covered: 0, - covering: needed_approvals, - uncovered: 0, - next_no_show: None, - last_assignment_tick: None, - }; + let initial_state = ( + State { + assignments: 0, + depth: 0, + covered: 0, + covering: needed_approvals, + uncovered: 0, + next_no_show: None, + last_assignment_tick: None, + }, + 0usize, + ); // The `ApprovalEntry` doesn't have any data for empty tranches. We still want to iterate over // these empty tranches, so we create an iterator to fill the gaps. @@ -395,7 +398,7 @@ pub fn tranches_to_approve( tranches_with_gaps_filled .scan(Some(initial_state), |state, (tranche, assignments)| { // The `Option` here is used for early exit. - let s = state.take()?; + let (s, prev_no_shows) = state.take()?; let clock_drift = s.clock_drift(no_show_duration); let drifted_tick_now = tick_now.saturating_sub(clock_drift); @@ -444,11 +447,11 @@ pub fn tranches_to_approve( RequiredTranches::Pending { .. } => { // Pending results are only interesting when they are the last result of the iterator // i.e. we never achieve a satisfactory level of assignment. - Some(s) + Some((s, no_shows + prev_no_shows)) } }; - Some(output) + Some((output, no_shows + prev_no_shows)) }) .last() .expect("the underlying iterator is infinite, starts at 0, and never exits early before tranche 1; qed") @@ -472,9 +475,9 @@ mod tests { } .into(); - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: BitVec::default(), + assigned_validators: BitVec::default(), our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -509,22 +512,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -581,22 +584,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -647,9 +650,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 5], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -675,7 +678,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -691,9 +695,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -715,7 +719,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: Some(block_tick + no_show_duration), @@ -731,9 +736,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -759,7 +764,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 11, next_no_show: None, @@ -776,9 +782,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -807,7 +813,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -826,7 +833,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -843,9 +851,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -879,7 +887,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -898,7 +907,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -917,7 +927,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -934,9 +945,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -970,7 +981,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -992,7 +1004,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -1013,7 +1026,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 3, tolerated_missing: 2, @@ -1041,15 +1055,15 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ // Assignments with invalid validator indexes. - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1068,7 +1082,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 10, next_no_show: None, @@ -1094,12 +1109,12 @@ mod tests { ]; for test_tranche in test_tranches { - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), backing_group: GroupIndex(0), our_assignment: None, our_approval_sig: None, - assignments: bitvec![u8, BitOrderLsb0; 0; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3], approved: false, } .into(); diff --git a/node/core/approval-voting/src/approval_db/mod.rs b/node/core/approval-voting/src/approval_db/mod.rs index f0ace95613da..20fb6aa82d8d 100644 --- a/node/core/approval-voting/src/approval_db/mod.rs +++ b/node/core/approval-voting/src/approval_db/mod.rs @@ -31,3 +31,4 @@ //! time being we share the same DB with the rest of Substrate. pub mod v1; +pub mod v2; diff --git a/node/core/approval-voting/src/approval_db/v1/mod.rs b/node/core/approval-voting/src/approval_db/v1/mod.rs index c31389269d2e..011d0a559c02 100644 --- a/node/core/approval-voting/src/approval_db/v1/mod.rs +++ b/node/core/approval-voting/src/approval_db/v1/mod.rs @@ -23,144 +23,15 @@ //! require a db migration (check `node/service/src/parachains_db/upgrade.rs`). use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche}; -use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; -use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_node_primitives::approval::v1::{AssignmentCert, DelayTranche}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; +use std::collections::BTreeMap; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use std::{collections::BTreeMap, sync::Arc}; - -use crate::{ - backend::{Backend, BackendWriteOp}, - persisted_entries, -}; - -const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; - -#[cfg(test)] -pub mod tests; - -/// `DbBackend` is a concrete implementation of the higher-level Backend trait -pub struct DbBackend { - inner: Arc, - config: Config, -} - -impl DbBackend { - /// Create a new [`DbBackend`] with the supplied key-value store and - /// config. - pub fn new(db: Arc, config: Config) -> Self { - DbBackend { inner: db, config } - } -} - -impl Backend for DbBackend { - fn load_block_entry( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } - - fn load_candidate_entry( - &self, - candidate_hash: &CandidateHash, - ) -> SubsystemResult> { - load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) - } - - fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { - load_blocks_at_height(&*self.inner, &self.config, block_height) - } - - fn load_all_blocks(&self) -> SubsystemResult> { - load_all_blocks(&*self.inner, &self.config) - } - - fn load_stored_blocks(&self) -> SubsystemResult> { - load_stored_blocks(&*self.inner, &self.config) - } - - /// Atomically write the list of operations, with later operations taking precedence over prior. - fn write(&mut self, ops: I) -> SubsystemResult<()> - where - I: IntoIterator, - { - let mut tx = DBTransaction::new(); - for op in ops { - match op { - BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { - tx.put_vec( - self.config.col_approval_data, - &STORED_BLOCKS_KEY, - stored_block_range.encode(), - ); - }, - BackendWriteOp::DeleteStoredBlockRange => { - tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); - }, - BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { - tx.put_vec( - self.config.col_approval_data, - &blocks_at_height_key(h), - blocks.encode(), - ); - }, - BackendWriteOp::DeleteBlocksAtHeight(h) => { - tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); - }, - BackendWriteOp::WriteBlockEntry(block_entry) => { - let block_entry: BlockEntry = block_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &block_entry_key(&block_entry.block_hash), - block_entry.encode(), - ); - }, - BackendWriteOp::DeleteBlockEntry(hash) => { - tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); - }, - BackendWriteOp::WriteCandidateEntry(candidate_entry) => { - let candidate_entry: CandidateEntry = candidate_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &candidate_entry_key(&candidate_entry.candidate.hash()), - candidate_entry.encode(), - ); - }, - BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { - tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); - }, - } - } - - self.inner.write(tx).map_err(|e| e.into()) - } -} - -/// A range from earliest..last block number stored within the DB. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); - -// slot_duration * 2 + DelayTranche gives the number of delay tranches since the -// unix epoch. -#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] -pub struct Tick(u64); - -/// Convenience type definition -pub type Bitfield = BitVec; - -/// The database config. -#[derive(Debug, Clone, Copy)] -pub struct Config { - /// The column family in the database where data is stored. - pub col_approval_data: u32, -} +use super::v2::Bitfield; /// Details pertaining to our assignment on a block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -171,15 +42,7 @@ pub struct OurAssignment { // Whether the assignment has been triggered already. pub triggered: bool, } - -/// Metadata regarding a specific tranche of assignments for a specific candidate. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct TrancheEntry { - pub tranche: DelayTranche, - // Assigned validators, and the instant we received their assignment, rounded - // to the nearest tick. - pub assignments: Vec<(ValidatorIndex, Tick)>, -} +use super::v2::TrancheEntry; /// Metadata regarding approval of a particular candidate within the context of some /// particular block. @@ -226,126 +89,3 @@ pub struct BlockEntry { pub approved_bitfield: Bitfield, pub children: Vec, } - -impl From for Tick { - fn from(tick: crate::Tick) -> Tick { - Tick(tick) - } -} - -impl From for crate::Tick { - fn from(tick: Tick) -> crate::Tick { - tick.0 - } -} - -/// Errors while accessing things from the DB. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub enum Error { - Io(std::io::Error), - InvalidDecoding(parity_scale_codec::Error), -} - -impl std::error::Error for Error {} - -/// Result alias for DB errors. -pub type Result = std::result::Result; - -pub(crate) fn load_decode( - store: &dyn Database, - col_approval_data: u32, - key: &[u8], -) -> Result> { - match store.get(col_approval_data, key)? { - None => Ok(None), - Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), - } -} - -/// The key a given block entry is stored under. -pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { - const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(block_hash.as_ref()); - - key -} - -/// The key a given candidate entry is stored under. -pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { - const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); - - key -} - -/// The key a set of block hashes corresponding to a block number is stored under. -pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { - const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; - - let mut key = [0u8; 12 + 4]; - key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); - block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); - - key -} - -/// Return all blocks which have entries in the DB, ascending, by height. -pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { - let mut hashes = Vec::new(); - if let Some(stored_blocks) = load_stored_blocks(store, config)? { - for height in stored_blocks.0..stored_blocks.1 { - let blocks = load_blocks_at_height(store, config, &height)?; - hashes.extend(blocks); - } - } - - Ok(hashes) -} - -/// Load the stored-blocks key from the state. -pub fn load_stored_blocks( - store: &dyn Database, - config: &Config, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a blocks-at-height entry for a given block number. -pub fn load_blocks_at_height( - store: &dyn Database, - config: &Config, - block_number: &BlockNumber, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) - .map(|x| x.unwrap_or_default()) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a block entry from the aux store. -pub fn load_block_entry( - store: &dyn Database, - config: &Config, - block_hash: &Hash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a candidate entry from the aux store. -pub fn load_candidate_entry( - store: &dyn Database, - config: &Config, - candidate_hash: &CandidateHash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} diff --git a/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs new file mode 100644 index 000000000000..e0aea5d7367b --- /dev/null +++ b/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -0,0 +1,241 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Approval DB migration helpers. +use super::{StoredBlockRange, *}; +use crate::backend::Backend; +use polkadot_node_primitives::approval::v1::{ + AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +}; +use polkadot_node_subsystem_util::database::Database; +use std::{collections::HashSet, sync::Arc}; + +use ::test_helpers::dummy_candidate_receipt; + +fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + +fn make_block_entry_v1( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> crate::approval_db::v1::BlockEntry { + crate::approval_db::v1::BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +/// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2. +/// Returns on any error. +/// Must only be used in parachains DB migration code - `polkadot-service` crate. +pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { + let mut backend = crate::DbBackend::new(db, config); + let all_blocks = backend + .load_all_blocks() + .map_err(|e| Error::InternalError(e))? + .iter() + .filter_map(|block_hash| { + backend + .load_block_entry_v1(block_hash) + .map_err(|e| Error::InternalError(e)) + .ok()? + }) + .collect::>(); + + gum::info!( + target: crate::LOG_TARGET, + "Migrating candidate entries on top of {} blocks", + all_blocks.len() + ); + + let mut overlay = crate::OverlayedBackend::new(&backend); + let mut counter = 0; + // Get all candidate entries, approval entries and convert each of them. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and return + // that represantation. + if let Some(candidate_entry) = backend + .load_candidate_entry_v1(&candidate_hash) + .map_err(|e| Error::InternalError(e))? + { + // Write the updated representation. + overlay.write_candidate_entry(candidate_entry); + counter += 1; + } + } + overlay.write_block_entry(block); + } + + gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter); + + // Commit all changes to DB. + let write_ops = overlay.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(()) +} + +// Checks if the migration doesn't leave the DB in an unsane state. +// This function is to be used in tests. +pub fn v1_to_v2_sanity_check( + db: Arc, + config: Config, + expected_candidates: HashSet, +) -> Result<()> { + let backend = crate::DbBackend::new(db, config); + + let all_blocks = backend + .load_all_blocks() + .unwrap() + .iter() + .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) + .collect::>(); + + let mut candidates = HashSet::new(); + + // Iterate all blocks and approval entries. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and return + // that represantation. + if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { + candidates.insert(candidate_entry.candidate.hash()); + } + } + } + + assert_eq!(candidates, expected_candidates); + + Ok(()) +} + +// Fills the db with dummy data in v1 scheme. +pub fn v1_to_v2_fill_test_data( + db: Arc, + config: Config, +) -> Result> { + let mut backend = crate::DbBackend::new(db.clone(), config); + let mut overlay_db = crate::OverlayedBackend::new(&backend); + let mut expected_candidates = HashSet::new(); + + const RELAY_BLOCK_COUNT: u32 = 10; + + let range = StoredBlockRange(1, 11); + overlay_db.write_stored_block_range(range.clone()); + + for relay_number in 1..=RELAY_BLOCK_COUNT { + let relay_hash = Hash::repeat_byte(relay_number as u8); + let assignment_core_index = CoreIndex(relay_number); + let candidate = dummy_candidate_receipt(relay_hash); + let candidate_hash = candidate.hash(); + + let at_height = vec![relay_hash]; + + let block_entry = make_block_entry_v1( + relay_hash, + Default::default(), + relay_number, + vec![(assignment_core_index, candidate_hash)], + ); + + let dummy_assignment = crate::approval_db::v1::OurAssignment { + cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + }; + + let candidate_entry = crate::approval_db::v1::CandidateEntry { + candidate, + session: 123, + block_assignments: vec![( + relay_hash, + crate::approval_db::v1::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: Some(dummy_assignment), + our_approval_sig: None, + assignments: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + overlay_db.write_blocks_at_height(relay_number, at_height.clone()); + expected_candidates.insert(candidate_entry.candidate.hash()); + + db.write(write_candidate_entry_v1(candidate_entry, config)).unwrap(); + db.write(write_block_entry_v1(block_entry, config)).unwrap(); + } + + let write_ops = overlay_db.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(expected_candidates) +} + +// Low level DB helper to write a candidate entry in v1 scheme. +fn write_candidate_entry_v1( + candidate_entry: crate::approval_db::v1::CandidateEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + tx +} + +// Low level DB helper to write a block entry in v1 scheme. +fn write_block_entry_v1( + block_entry: crate::approval_db::v1::BlockEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + tx +} diff --git a/node/core/approval-voting/src/approval_db/v2/mod.rs b/node/core/approval-voting/src/approval_db/v2/mod.rs new file mode 100644 index 000000000000..88650a539130 --- /dev/null +++ b/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -0,0 +1,420 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Version 2 of the DB schema. + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_primitives::approval::{ + v1::DelayTranche, + v2::{AssignmentCertV2, CandidateBitfield}, +}; +use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, +}; + +use sp_consensus_slots::Slot; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use std::{collections::BTreeMap, sync::Arc}; + +use crate::{ + backend::{Backend, BackendWriteOp, V1ReadBackend}, + persisted_entries, +}; + +const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; + +pub mod migration_helpers; +#[cfg(test)] +pub mod tests; + +/// `DbBackend` is a concrete implementation of the higher-level Backend trait +pub struct DbBackend { + inner: Arc, + config: Config, +} + +impl DbBackend { + /// Create a new [`DbBackend`] with the supplied key-value store and + /// config. + pub fn new(db: Arc, config: Config) -> Self { + DbBackend { inner: db, config } + } +} + +impl V1ReadBackend for DbBackend { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(Into::into)) + } + + fn load_block_entry_v1( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +impl Backend for DbBackend { + fn load_block_entry( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } + + fn load_candidate_entry( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) + } + + fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { + load_blocks_at_height(&*self.inner, &self.config, block_height) + } + + fn load_all_blocks(&self) -> SubsystemResult> { + load_all_blocks(&*self.inner, &self.config) + } + + fn load_stored_blocks(&self) -> SubsystemResult> { + load_stored_blocks(&*self.inner, &self.config) + } + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> SubsystemResult<()> + where + I: IntoIterator, + { + let mut tx = DBTransaction::new(); + for op in ops { + match op { + BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { + tx.put_vec( + self.config.col_approval_data, + &STORED_BLOCKS_KEY, + stored_block_range.encode(), + ); + }, + BackendWriteOp::DeleteStoredBlockRange => { + tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); + }, + BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { + tx.put_vec( + self.config.col_approval_data, + &blocks_at_height_key(h), + blocks.encode(), + ); + }, + BackendWriteOp::DeleteBlocksAtHeight(h) => { + tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); + }, + BackendWriteOp::WriteBlockEntry(block_entry) => { + let block_entry: BlockEntry = block_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + }, + BackendWriteOp::DeleteBlockEntry(hash) => { + tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); + }, + BackendWriteOp::WriteCandidateEntry(candidate_entry) => { + let candidate_entry: CandidateEntry = candidate_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + }, + BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { + tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); + }, + } + } + + self.inner.write(tx).map_err(|e| e.into()) + } +} + +/// A range from earliest..last block number stored within the DB. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); + +// slot_duration * 2 + DelayTranche gives the number of delay tranches since the +// unix epoch. +#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] +pub struct Tick(u64); + +/// Convenience type definition +pub type Bitfield = BitVec; + +/// The database config. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The column family in the database where data is stored. + pub col_approval_data: u32, +} + +/// Details pertaining to our assignment on a block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurAssignment { + /// Our assignment certificate. + pub cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + pub tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + pub validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + pub triggered: bool, +} + +/// Metadata regarding a specific tranche of assignments for a specific candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct TrancheEntry { + pub tranche: DelayTranche, + // Assigned validators, and the instant we received their assignment, rounded + // to the nearest tick. + pub assignments: Vec<(ValidatorIndex, Tick)>, +} + +/// Metadata about our approval signature +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval, an empty value means only + /// the candidate referred by this approval entry was signed. + pub signed_candidates_indices: Option, +} + +/// Metadata regarding approval of a particular candidate within the context of some +/// particular block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct ApprovalEntry { + pub tranches: Vec, + pub backing_group: GroupIndex, + pub our_assignment: Option, + pub our_approval_sig: Option, + // `n_validators` bits. + pub assigned_validators: Bitfield, + pub approved: bool, +} + +/// Metadata regarding approval of a particular candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct CandidateEntry { + pub candidate: CandidateReceipt, + pub session: SessionIndex, + // Assignments are based on blocks, so we need to track assignments separately + // based on the block we are looking at. + pub block_assignments: BTreeMap, + pub approvals: Bitfield, +} + +/// Metadata regarding approval of a particular block, by way of approval of the +/// candidates contained within it. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct BlockEntry { + pub block_hash: Hash, + pub block_number: BlockNumber, + pub parent_hash: Hash, + pub session: SessionIndex, + pub slot: Slot, + /// Random bytes derived from the VRF submitted within the block by the block + /// author as a credential and used as input to approval assignment criteria. + pub relay_vrf_story: [u8; 32], + // The candidates included as-of this block and the index of the core they are + // leaving. Sorted ascending by core index. + pub candidates: Vec<(CoreIndex, CandidateHash)>, + // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. + // The i'th bit is `true` iff the candidate has been approved in the context of this + // block. The block can be considered approved if the bitfield has all bits set to `true`. + pub approved_bitfield: Bitfield, + pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + pub candidates_pending_signature: BTreeMap, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + pub distributed_assignments: Bitfield, +} + +#[derive(Encode, Decode, Debug, Clone, PartialEq)] + +/// Context needed for creating an approval signature for a given candidate. +pub struct CandidateSigningContext { + /// The candidate hash, to be included in the signature. + pub candidate_hash: CandidateHash, + /// The latest tick we have to create and send the approval. + pub send_no_later_than_tick: Tick, +} + +impl From for Tick { + fn from(tick: crate::Tick) -> Tick { + Tick(tick) + } +} + +impl From for crate::Tick { + fn from(tick: Tick) -> crate::Tick { + tick.0 + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + Io(std::io::Error), + InvalidDecoding(parity_scale_codec::Error), + InternalError(SubsystemError), +} + +impl std::error::Error for Error {} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +pub(crate) fn load_decode( + store: &dyn Database, + col_approval_data: u32, + key: &[u8], +) -> Result> { + match store.get(col_approval_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), + } +} + +/// The key a given block entry is stored under. +pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { + const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(block_hash.as_ref()); + + key +} + +/// The key a given candidate entry is stored under. +pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { + const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); + + key +} + +/// The key a set of block hashes corresponding to a block number is stored under. +pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { + const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; + + let mut key = [0u8; 12 + 4]; + key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); + block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); + + key +} + +/// Return all blocks which have entries in the DB, ascending, by height. +pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { + let mut hashes = Vec::new(); + if let Some(stored_blocks) = load_stored_blocks(store, config)? { + for height in stored_blocks.0..stored_blocks.1 { + let blocks = load_blocks_at_height(store, config, &height)?; + hashes.extend(blocks); + } + } + + Ok(hashes) +} + +/// Load the stored-blocks key from the state. +pub fn load_stored_blocks( + store: &dyn Database, + config: &Config, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a blocks-at-height entry for a given block number. +pub fn load_blocks_at_height( + store: &dyn Database, + config: &Config, + block_number: &BlockNumber, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) + .map(|x| x.unwrap_or_default()) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store. +pub fn load_block_entry( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in current version format. +pub fn load_candidate_entry( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in v1 format. +pub fn load_candidate_entry_v1( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store in v1 format. +pub fn load_block_entry_v1( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/node/core/approval-voting/src/approval_db/v1/tests.rs b/node/core/approval-voting/src/approval_db/v2/tests.rs similarity index 99% rename from node/core/approval-voting/src/approval_db/v1/tests.rs rename to node/core/approval-voting/src/approval_db/v2/tests.rs index 07d8242b772e..31b66d88577f 100644 --- a/node/core/approval-voting/src/approval_db/v1/tests.rs +++ b/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -40,10 +40,6 @@ fn make_db() -> (DbBackend, Arc) { (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_block_entry( block_hash: Hash, parent_hash: Hash, @@ -60,9 +56,15 @@ fn make_block_entry( approved_bitfield: make_bitvec(candidates.len()), candidates, children: Vec::new(), + candidates_pending_signature: Default::default(), + distributed_assignments: Default::default(), } } +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { let mut c = dummy_candidate_receipt(dummy_hash()); @@ -96,7 +98,7 @@ fn read_write() { backing_group: GroupIndex(1), our_assignment: None, our_approval_sig: None, - assignments: Default::default(), + assigned_validators: Default::default(), approved: false, }, )] diff --git a/node/core/approval-voting/src/backend.rs b/node/core/approval-voting/src/backend.rs index 87d67c52c467..374e7a826d19 100644 --- a/node/core/approval-voting/src/backend.rs +++ b/node/core/approval-voting/src/backend.rs @@ -27,7 +27,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, Hash}; use std::collections::HashMap; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, persisted_entries::{BlockEntry, CandidateEntry}, }; @@ -44,6 +44,7 @@ pub enum BackendWriteOp { } /// An abstraction over backend storage for the logic of this subsystem. +/// Implementation must always target latest storage version. pub trait Backend { /// Load a block entry from the DB. fn load_block_entry(&self, hash: &Hash) -> SubsystemResult>; @@ -52,6 +53,7 @@ pub trait Backend { &self, candidate_hash: &CandidateHash, ) -> SubsystemResult>; + /// Load all blocks at a specific height. fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult>; /// Load all block from the DB. @@ -64,6 +66,18 @@ pub trait Backend { I: IntoIterator; } +/// A read only backed to enable db migration from version 1 of DB. +pub trait V1ReadBackend: Backend { + /// Load a candidate entry from the DB with scheme version 1. + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult>; + + /// Load a block entry from the DB with scheme version 1. + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult>; +} + // Status of block range in the `OverlayedBackend`. #[derive(PartialEq)] enum BlockRangeStatus { diff --git a/node/core/approval-voting/src/criteria.rs b/node/core/approval-voting/src/criteria.rs index 0e1d18198c21..db74302f0225 100644 --- a/node/core/approval-voting/src/criteria.rs +++ b/node/core/approval-voting/src/criteria.rs @@ -18,7 +18,9 @@ use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::{ - self as approval_types, AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory, + self as approval_types, + v1::{AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory}, + v2::{AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, VrfOutput, VrfProof, VrfSignature}, }; use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, @@ -30,6 +32,7 @@ use sp_application_crypto::ByteArray; use merlin::Transcript; use schnorrkel::vrf::VRFInOut; +use itertools::Itertools; use std::collections::{hash_map::Entry, HashMap}; use super::LOG_TARGET; @@ -37,7 +40,7 @@ use super::LOG_TARGET; /// Details pertaining to our assignment on a block. #[derive(Debug, Clone, Encode, Decode, PartialEq)] pub struct OurAssignment { - cert: AssignmentCert, + cert: AssignmentCertV2, tranche: DelayTranche, validator_index: ValidatorIndex, // Whether the assignment has been triggered already. @@ -45,7 +48,7 @@ pub struct OurAssignment { } impl OurAssignment { - pub(crate) fn cert(&self) -> &AssignmentCert { + pub(crate) fn cert(&self) -> &AssignmentCertV2 { &self.cert } @@ -66,8 +69,8 @@ impl OurAssignment { } } -impl From for OurAssignment { - fn from(entry: crate::approval_db::v1::OurAssignment) -> Self { +impl From for OurAssignment { + fn from(entry: crate::approval_db::v2::OurAssignment) -> Self { OurAssignment { cert: entry.cert, tranche: entry.tranche, @@ -77,7 +80,7 @@ impl From for OurAssignment { } } -impl From for crate::approval_db::v1::OurAssignment { +impl From for crate::approval_db::v2::OurAssignment { fn from(entry: OurAssignment) -> Self { Self { cert: entry.cert, @@ -88,17 +91,97 @@ impl From for crate::approval_db::v1::OurAssignment { } } -fn relay_vrf_modulo_transcript(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { - // combine the relay VRF story with a sample number. - let mut t = Transcript::new(approval_types::RELAY_VRF_MODULO_CONTEXT); - t.append_message(b"RC-VRF", &relay_vrf_story.0); - sample.using_encoded(|s| t.append_message(b"sample", s)); +// Combines the relay VRF story with a sample number if any. +fn relay_vrf_modulo_transcript_inner( + mut transcript: Transcript, + relay_vrf_story: RelayVRFStory, + sample: Option, +) -> Transcript { + transcript.append_message(b"RC-VRF", &relay_vrf_story.0); - t + if let Some(sample) = sample { + sample.using_encoded(|s| transcript.append_message(b"sample", s)); + } + + transcript +} + +fn relay_vrf_modulo_transcript_v1(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v1::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + Some(sample), + ) +} + +fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v2::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + None, + ) +} + +/// A hard upper bound on num_cores * target_checkers / num_validators +const MAX_MODULO_SAMPLES: usize = 40; + +use std::convert::AsMut; + +fn clone_into_array(slice: &[T]) -> A +where + A: Default + AsMut<[T]>, + T: Clone, +{ + let mut a = A::default(); + >::as_mut(&mut a).clone_from_slice(slice); + a +} + +struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); + +impl Default for BigArray { + fn default() -> Self { + BigArray([0u8; MAX_MODULO_SAMPLES * 4]) + } +} + +impl AsMut<[u8]> for BigArray { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} + +/// Takes the VRF output as input and returns a Vec of cores the validator is assigned +/// to as a tranche0 checker. +fn relay_vrf_modulo_cores( + vrf_in_out: &VRFInOut, + // Configuration - `relay_vrf_modulo_samples`. + num_samples: u32, + // Configuration - `n_cores`. + max_cores: u32, +) -> Vec { + if num_samples as usize > MAX_MODULO_SAMPLES { + gum::warn!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + ); + } + + vrf_in_out + .make_bytes::(approval_types::v2::CORE_RANDOMNESS_CONTEXT) + .0 + .chunks_exact(4) + .take(num_samples as usize) + .map(move |sample| CoreIndex(u32::from_le_bytes(clone_into_array(&sample)) % max_cores)) + .unique() + .collect::>() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::CORE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::CORE_RANDOMNESS_CONTEXT); // interpret as little-endian u32. let random_core = u32::from_le_bytes(bytes) % n_cores; @@ -106,7 +189,7 @@ fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { } fn relay_vrf_delay_transcript(relay_vrf_story: RelayVRFStory, core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::RELAY_VRF_DELAY_CONTEXT); + let mut t = Transcript::new(approval_types::v1::RELAY_VRF_DELAY_CONTEXT); t.append_message(b"RC-VRF", &relay_vrf_story.0); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t @@ -117,7 +200,7 @@ fn relay_vrf_delay_tranche( num_delay_tranches: u32, zeroth_delay_tranche_width: u32, ) -> DelayTranche { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::TRANCHE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::TRANCHE_RANDOMNESS_CONTEXT); // interpret as little-endian u32 and reduce by the number of tranches. let wide_tranche = @@ -128,13 +211,13 @@ fn relay_vrf_delay_tranche( } fn assigned_core_transcript(core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::ASSIGNED_CORE_CONTEXT); + let mut t = Transcript::new(approval_types::v1::ASSIGNED_CORE_CONTEXT); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t } /// Information about the world assignments are being produced in. -#[derive(Clone)] +#[derive(Clone, Debug)] pub(crate) struct Config { /// The assignment public keys for validators. assignment_keys: Vec, @@ -175,12 +258,13 @@ pub(crate) trait AssignmentCriteria { fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + // Backing groups for each "leaving core". + backing_groups: Vec, ) -> Result; } @@ -194,25 +278,25 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { check_assignment_cert( - claimed_core_index, + claimed_core_bitfield, validator_index, config, relay_vrf_story, assignment, - backing_group, + backing_groups, ) } } @@ -233,6 +317,7 @@ pub(crate) fn compute_assignments( relay_vrf_story: RelayVRFStory, config: &Config, leaving_cores: impl IntoIterator + Clone, + enable_v2_assignments: bool, ) -> HashMap { if config.n_cores == 0 || config.assignment_keys.is_empty() || @@ -291,14 +376,25 @@ pub(crate) fn compute_assignments( let mut assignments = HashMap::new(); // First run `RelayVRFModulo` for each sample. - compute_relay_vrf_modulo_assignments( - &assignments_key, - index, - config, - relay_vrf_story.clone(), - leaving_cores.iter().cloned(), - &mut assignments, - ); + if enable_v2_assignments { + compute_relay_vrf_modulo_assignments_v2( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } else { + compute_relay_vrf_modulo_assignments_v1( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } // Then run `RelayVRFDelay` once for the whole block. compute_relay_vrf_delay_assignments( @@ -313,7 +409,7 @@ pub(crate) fn compute_assignments( assignments } -fn compute_relay_vrf_modulo_assignments( +fn compute_relay_vrf_modulo_assignments_v1( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, config: &Config, @@ -329,7 +425,7 @@ fn compute_relay_vrf_modulo_assignments( // into closure. let core = &mut core; assignments_key.vrf_sign_extra_after_check( - relay_vrf_modulo_transcript(relay_vrf_story.clone(), rvm_sample), + relay_vrf_modulo_transcript_v1(relay_vrf_story.clone(), rvm_sample), |vrf_in_out| { *core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); if let Some((candidate_hash, _)) = @@ -357,15 +453,15 @@ fn compute_relay_vrf_modulo_assignments( // has been executed. let cert = AssignmentCert { kind: AssignmentCertKind::RelayVRFModulo { sample: rvm_sample }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; // All assignments of type RelayVRFModulo have tranche 0. assignments.entry(core).or_insert(OurAssignment { - cert, + cert: cert.into(), tranche: 0, validator_index, triggered: false, @@ -374,6 +470,84 @@ fn compute_relay_vrf_modulo_assignments( } } +fn assigned_cores_transcript(core_bitfield: &CoreBitfield) -> Transcript { + let mut t = Transcript::new(approval_types::v2::ASSIGNED_CORE_CONTEXT); + core_bitfield.using_encoded(|s| t.append_message(b"cores", s)); + t +} + +fn compute_relay_vrf_modulo_assignments_v2( + assignments_key: &schnorrkel::Keypair, + validator_index: ValidatorIndex, + config: &Config, + relay_vrf_story: RelayVRFStory, + leaving_cores: Vec<(CandidateHash, CoreIndex)>, + assignments: &mut HashMap, +) { + let mut assigned_cores = Vec::new(); + let leaving_cores = leaving_cores.iter().map(|(_, core)| core).collect::>(); + + let maybe_assignment = { + let assigned_cores = &mut assigned_cores; + assignments_key.vrf_sign_extra_after_check( + relay_vrf_modulo_transcript_v2(relay_vrf_story.clone()), + |vrf_in_out| { + *assigned_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ) + .into_iter() + .filter(|core| leaving_cores.contains(&core)) + .collect::>(); + + if !assigned_cores.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?assigned_cores, + ?validator_index, + tranche = 0, + "RelayVRFModuloCompact Assignment." + ); + + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + Some(assigned_cores_transcript(&assignment_bitfield)) + } else { + None + } + }, + ) + }; + + if let Some(assignment) = maybe_assignment.map(|(vrf_in_out, vrf_proof, _)| { + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: assignment_bitfield.clone(), + }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), + }, + }; + + // All assignments of type RelayVRFModulo have tranche 0. + OurAssignment { cert, tranche: 0, validator_index, triggered: false } + }) { + for core_index in assigned_cores { + assignments.insert(core_index, assignment.clone()); + } + } +} + fn compute_relay_vrf_delay_assignments( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, @@ -392,11 +566,11 @@ fn compute_relay_vrf_delay_assignments( config.zeroth_delay_tranche_width, ); - let cert = AssignmentCert { - kind: AssignmentCertKind::RelayVRFDelay { core_index: core }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFDelay { core_index: core }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; @@ -453,12 +627,15 @@ pub(crate) enum InvalidAssignmentReason { VRFModuloOutputMismatch, VRFDelayCoreIndexMismatch, VRFDelayOutputMismatch, + InvalidArguments, + /// Assignment vrf check resulted in 0 assigned cores. + NullAssignment, } /// Checks the crypto of an assignment cert. Failure conditions: /// * Validator index out of bounds /// * VRF signature check fails -/// * VRF output doesn't match assigned core +/// * VRF output doesn't match assigned cores /// * Core is not covered by extra data in signature /// * Core index out of bounds /// * Sample is out of bounds @@ -467,12 +644,12 @@ pub(crate) enum InvalidAssignmentReason { /// This function does not check whether the core is actually a valid assignment or not. That should /// be done outside the scope of this function. pub(crate) fn check_assignment_cert( - claimed_core_index: CoreIndex, + claimed_core_indices: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { use InvalidAssignmentReason as Reason; @@ -484,52 +661,133 @@ pub(crate) fn check_assignment_cert( let public = schnorrkel::PublicKey::from_bytes(validator_public.as_slice()) .map_err(|_| InvalidAssignment(Reason::InvalidAssignmentKey))?; - if claimed_core_index.0 >= config.n_cores { - return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + // Check that we have all backing groups for claimed cores. + if claimed_core_indices.count_ones() == 0 || + claimed_core_indices.count_ones() != backing_groups.len() + { + return Err(InvalidAssignment(Reason::InvalidArguments)) } // Check that the validator was not part of the backing group // and not already assigned. - let is_in_backing = - is_in_backing_group(&config.validator_groups, validator_index, backing_group); + for (claimed_core, backing_group) in claimed_core_indices.iter_ones().zip(backing_groups.iter()) + { + if claimed_core >= config.n_cores as usize { + return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + } - if is_in_backing { - return Err(InvalidAssignment(Reason::IsInBackingGroup)) + let is_in_backing = + is_in_backing_group(&config.validator_groups, validator_index, *backing_group); + + if is_in_backing { + return Err(InvalidAssignment(Reason::IsInBackingGroup)) + } } - let vrf_signature = &assignment.vrf; - match assignment.kind { - AssignmentCertKind::RelayVRFModulo { sample } => { - if sample >= config.relay_vrf_modulo_samples { + let vrf_output = &assignment.vrf.output; + let vrf_proof = &assignment.vrf.proof; + let first_claimed_core_index = + claimed_core_indices.first_one().expect("Checked above; qed") as u32; + + match &assignment.kind { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => { + // Check that claimed core bitfield match the one from certificate. + if &claimed_core_indices != core_bitfield { + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + + let (vrf_in_out, _) = public + .vrf_verify_extra( + relay_vrf_modulo_transcript_v2(relay_vrf_story), + &vrf_output.0, + &vrf_proof.0, + assigned_cores_transcript(core_bitfield), + ) + .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + + let resulting_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ); + + // Currently validators can opt out of checking specific cores. + // This is the same issue to how validator can opt out and not send their assignments in + // the first place. Ensure that the `vrf_in_out` actually includes all of the claimed + // cores. + for claimed_core_index in claimed_core_indices.iter_ones() { + if !resulting_cores.contains(&CoreIndex(claimed_core_index as u32)) { + gum::debug!( + target: LOG_TARGET, + ?resulting_cores, + ?claimed_core_indices, + vrf_modulo_cores = ?resulting_cores, + "Assignment claimed cores mismatch", + ); + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + } + + Ok(0) + }, + AssignmentCertKindV2::RelayVRFModulo { sample } => { + if *sample >= config.relay_vrf_modulo_samples { return Err(InvalidAssignment(Reason::SampleOutOfBounds)) } + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::warn!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFModulo` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + let (vrf_in_out, _) = public .vrf_verify_extra( - relay_vrf_modulo_transcript(relay_vrf_story, sample), - &vrf_signature.output.0, - &vrf_signature.proof.0, - assigned_core_transcript(claimed_core_index), + relay_vrf_modulo_transcript_v1(relay_vrf_story, *sample), + &vrf_output.0, + &vrf_proof.0, + assigned_core_transcript(CoreIndex(first_claimed_core_index)), ) .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + let core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); // ensure that the `vrf_in_out` actually gives us the claimed core. - if relay_vrf_modulo_core(&vrf_in_out, config.n_cores) == claimed_core_index { + if core.0 == first_claimed_core_index { Ok(0) } else { + gum::debug!( + target: LOG_TARGET, + ?core, + ?claimed_core_indices, + "Assignment claimed cores mismatch", + ); Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) } }, - AssignmentCertKind::RelayVRFDelay { core_index } => { - if core_index != claimed_core_index { + AssignmentCertKindV2::RelayVRFDelay { core_index } => { + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::debug!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFDelay` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + + if core_index.0 != first_claimed_core_index { return Err(InvalidAssignment(Reason::VRFDelayCoreIndexMismatch)) } let (vrf_in_out, _) = public .vrf_verify( - relay_vrf_delay_transcript(relay_vrf_story, core_index), - &vrf_signature.output.0, - &vrf_signature.proof.0, + relay_vrf_delay_transcript(relay_vrf_story, *core_index), + &vrf_output.0, + &vrf_proof.0, ) .map_err(|_| InvalidAssignment(Reason::VRFDelayOutputMismatch))?; @@ -550,6 +808,19 @@ fn is_in_backing_group( validator_groups.get(group).map_or(false, |g| g.contains(&validator)) } +/// Migration helpers. +impl From for OurAssignment { + fn from(value: crate::approval_db::v1::OurAssignment) -> Self { + Self { + cert: value.cert.into(), + tranche: value.tranche, + validator_index: value.validator_index, + // Whether the assignment has been triggered already. + triggered: value.triggered, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -630,10 +901,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(1)), (c_b, CoreIndex(1), GroupIndex(0))], + false, ); // Note that alice is in group 0, which was the backing group for core 1. @@ -665,10 +937,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(0)), (c_b, CoreIndex(1), GroupIndex(1))], + false, ); assert_eq!(assignments.len(), 1); @@ -692,19 +965,21 @@ mod tests { validator_groups: Default::default(), n_cores: 0, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![], + false, ); assert!(assignments.is_empty()); } + #[derive(Debug)] struct MutatedAssignment { - core: CoreIndex, - cert: AssignmentCert, - group: GroupIndex, + cores: CoreBitfield, + cert: AssignmentCertV2, + groups: Vec, own_group: GroupIndex, val_index: ValidatorIndex, config: Config, @@ -729,12 +1004,12 @@ mod tests { validator_groups: basic_groups(n_validators, n_cores), n_cores: n_cores as u32, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 15, n_delay_tranches: 40, }; let relay_vrf_story = RelayVRFStory([42u8; 32]); - let assignments = compute_assignments( + let mut assignments = compute_assignments( &keystore, relay_vrf_story.clone(), &config, @@ -747,19 +1022,42 @@ mod tests { ) }) .collect::>(), + false, ); + // Extend with v2 assignments as well + assignments.extend(compute_assignments( + &keystore, + relay_vrf_story.clone(), + &config, + (0..n_cores) + .map(|i| { + ( + CandidateHash(Hash::repeat_byte(i as u8)), + CoreIndex(i as u32), + group_for_core(i), + ) + }) + .collect::>(), + true, + )); + let mut counted = 0; for (core, assignment) in assignments { + let cores = match assignment.cert.kind.clone() { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => core_bitfield, + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => core.into(), + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.into(), + }; + let mut mutated = MutatedAssignment { - core, - group: group_for_core(core.0 as _), + cores: cores.clone(), + groups: cores.iter_ones().map(|core| group_for_core(core)).collect(), cert: assignment.cert, own_group: GroupIndex(0), val_index: ValidatorIndex(0), config: config.clone(), }; - let expected = match f(&mut mutated) { None => continue, Some(e) => e, @@ -768,16 +1066,16 @@ mod tests { counted += 1; let is_good = check_assignment_cert( - mutated.core, + mutated.cores, mutated.val_index, &mutated.config, relay_vrf_story.clone(), &mutated.cert, - mutated.group, + mutated.groups, ) .is_ok(); - assert_eq!(expected, is_good) + assert_eq!(expected, is_good); } assert!(counted > 0); @@ -791,7 +1089,7 @@ mod tests { #[test] fn check_rejects_claimed_core_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { - m.core.0 += 100; + m.cores = CoreIndex(100).into(); Some(false) }); } @@ -799,7 +1097,7 @@ mod tests { #[test] fn check_rejects_in_backing_group() { check_mutated_assignments(200, 100, 25, |m| { - m.group = m.own_group; + m.groups[0] = m.own_group; Some(false) }); } @@ -815,9 +1113,10 @@ mod tests { #[test] fn check_rejects_delay_bad_vrf() { check_mutated_assignments(40, 10, 8, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -828,9 +1127,14 @@ mod tests { #[test] fn check_rejects_modulo_bad_vrf() { check_mutated_assignments(200, 100, 25, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFModulo { .. } => { + m.cert.vrf = vrf_signature; + Some(false) + }, + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -842,10 +1146,11 @@ mod tests { fn check_rejects_modulo_sample_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { sample } => { + AssignmentCertKindV2::RelayVRFModulo { sample } => { m.config.relay_vrf_modulo_samples = sample; Some(false) }, + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield: _ } => Some(true), _ => None, // skip everything else. } }); @@ -855,8 +1160,11 @@ mod tests { fn check_rejects_delay_claimed_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + // for core in &mut m.cores { + // core.0 = (core.0 + 1) % 100; + // } + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); Some(false) }, _ => None, // skip everything else. @@ -868,8 +1176,10 @@ mod tests { fn check_rejects_modulo_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFModulo { .. } | + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); + Some(false) }, _ => None, // skip everything else. diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index c504ba71b3c2..019a74a51a74 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -30,7 +30,10 @@ use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ - approval::{self as approval_types, BlockApprovalMeta, RelayVRFStory}, + approval::{ + self as approval_types, + v1::{BlockApprovalMeta, RelayVRFStory}, + }, MAX_FINALITY_LAG, }; use polkadot_node_subsystem::{ @@ -53,7 +56,7 @@ use futures::{channel::oneshot, prelude::*}; use std::collections::HashMap; -use super::approval_db::v1; +use super::approval_db::v2; use crate::{ backend::{Backend, OverlayedBackend}, criteria::{AssignmentCriteria, OurAssignment}, @@ -92,7 +95,7 @@ enum ImportedBlockInfoError { FutureCancelled(&'static str, futures::channel::oneshot::Canceled), #[error(transparent)] - ApprovalError(approval_types::ApprovalError), + ApprovalError(approval_types::v1::ApprovalError), #[error("block is already finalized")] BlockAlreadyFinalized, @@ -216,7 +219,7 @@ async fn imported_block_info( .ok_or(ImportedBlockInfoError::SessionInfoUnavailable)?; let (assignments, slot, relay_vrf_story) = { - let unsafe_vrf = approval_types::babe_unsafe_vrf_info(&block_header); + let unsafe_vrf = approval_types::v1::babe_unsafe_vrf_info(&block_header); match unsafe_vrf { Some(unsafe_vrf) => { @@ -497,7 +500,7 @@ pub(crate) async fn handle_new_head( ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; } - let block_entry = v1::BlockEntry { + let block_entry = v2::BlockEntry { block_hash, parent_hash: block_header.parent_hash, block_number: block_header.number, @@ -510,6 +513,8 @@ pub(crate) async fn handle_new_head( .collect(), approved_bitfield, children: Vec::new(), + candidates_pending_signature: Default::default(), + distributed_assignments: Default::default(), }; gum::trace!( @@ -588,11 +593,11 @@ pub(crate) async fn handle_new_head( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{approval_db::v1::DbBackend, RuntimeInfo, RuntimeInfoConfig}; + use crate::{approval_db::v2::DbBackend, RuntimeInfo, RuntimeInfoConfig}; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_primitives::{ - approval::{VrfSignature, VrfTranscript}, + approval::v1::{VrfSignature, VrfTranscript}, DISPUTE_WINDOW, }; use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage}; @@ -608,7 +613,7 @@ pub(crate) mod tests { pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; - use crate::{approval_db::v1::Config as DatabaseConfig, criteria, BlockEntry}; + use crate::{approval_db::v2::Config as DatabaseConfig, criteria, BlockEntry}; const DATA_COL: u32 = 0; @@ -635,6 +640,7 @@ pub(crate) mod tests { clock: Box::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria), spans: HashMap::new(), + approval_voting_params_cache: None, } } @@ -654,7 +660,7 @@ pub(crate) mod tests { fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -667,13 +673,14 @@ pub(crate) mod tests { fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, _validator_index: polkadot_primitives::ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result + { Ok(0) } } @@ -1252,7 +1259,7 @@ pub(crate) mod tests { let (state, mut session_info_provider) = single_session_state(); overlay_db.write_block_entry( - v1::BlockEntry { + v2::BlockEntry { block_hash: parent_hash, parent_hash: Default::default(), block_number: 4, @@ -1262,6 +1269,8 @@ pub(crate) mod tests { candidates: Vec::new(), approved_bitfield: Default::default(), children: Vec::new(), + candidates_pending_signature: Default::default(), + distributed_assignments: Default::default(), } .into(), ); @@ -1294,7 +1303,7 @@ pub(crate) mod tests { // the first candidate should be insta-approved // the second should not let entry: BlockEntry = - v1::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) + v2::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) .unwrap() .unwrap() .into(); diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index b29e47b4c435..1a84040dacbf 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -21,11 +21,17 @@ //! of others. It uses this information to determine when candidates and blocks have //! been sufficiently approved to finalize. +use itertools::Itertools; use jaeger::{hash_to_trace_identifier, PerLeafSpan}; +use lru::LruCache; use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ approval::{ - BlockApprovalMeta, DelayTranche, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{BlockApprovalMeta, DelayTranche}, + v2::{ + AssignmentCertKindV2, BitfieldError, CandidateBitfield, CoreBitfield, + IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, + }, }, ValidationResult, DISPUTE_WINDOW, }; @@ -49,9 +55,10 @@ use polkadot_node_subsystem_util::{ TimeoutExt, }; use polkadot_primitives::{ - ApprovalVote, BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, - GroupIndex, Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, - ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, + vstaging::{ApprovalVoteMultipleCandidates, ApprovalVotingParams}, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, GroupIndex, + Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, + ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; use sp_application_crypto::Pair; @@ -63,9 +70,11 @@ use futures::{ future::{BoxFuture, RemoteHandle}, prelude::*, stream::FuturesUnordered, + StreamExt, }; use std::{ + cmp::min, collections::{ btree_map::Entry as BTMEntry, hash_map::Entry as HMEntry, BTreeMap, HashMap, HashSet, }, @@ -75,12 +84,13 @@ use std::{ }; use approval_checking::RequiredTranches; +use bitvec::{order::Lsb0, vec::BitVec}; use criteria::{AssignmentCriteria, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; -use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; +use time::{slot_number_to_tick, Clock, ClockExt, DelayedApprovalTimer, SystemClock, Tick}; mod approval_checking; -mod approval_db; +pub mod approval_db; mod backend; mod criteria; mod import; @@ -89,8 +99,11 @@ mod persisted_entries; mod time; use crate::{ - approval_db::v1::{Config as DatabaseConfig, DbBackend}, + approval_checking::Check, + approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, + criteria::InvalidAssignmentReason, + persisted_entries::OurApproval, }; #[cfg(test)] @@ -109,7 +122,17 @@ const APPROVAL_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(1024) { const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; -const LOG_TARGET: &str = "parachain::approval-voting"; +pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; + +// The max number of ticks we delay sending the approval after we are ready to issue the approval +const MAX_APPROVAL_COALESCE_WAIT_TICKS: Tick = 2; + +// The maximum approval params we cache locally in the subsytem, so that we don't have +// to do the back and forth to the runtime subsystem api. +const APPROVAL_PARAMS_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(128) { + Some(cap) => cap, + None => panic!("Approval params cache size must be non-zero."), +}; /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] @@ -146,6 +169,9 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, + // Store approval-voting params, so that we don't to always go to RuntimeAPI + // to ask this information which requires a task context switch. + approval_voting_params_cache: Option>, } #[derive(Clone)] @@ -154,7 +180,13 @@ struct MetricsInner { assignments_produced: prometheus::Histogram, approvals_produced_total: prometheus::CounterVec, no_shows_total: prometheus::Counter, + // The difference from `no_shows_total` is that this counts all observed no-shows at any + // moment in time. While `no_shows_total` catches that the no-shows at the moment the candidate + // is approved, approvals might arrive late and `no_shows_total` wouldn't catch that number. + observed_no_shows: prometheus::Counter, + approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, + coalesced_approvals_buckets: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -180,6 +212,16 @@ impl Metrics { } } + fn on_approval_coalesce(&self, num_coalesced: u32) { + if let Some(metrics) = &self.0 { + // Count how many candidates we covered with this coalesced approvals, + // so that the heat-map really gives a good understanding of the scales. + for _ in 0..num_coalesced { + metrics.coalesced_approvals_buckets.observe(num_coalesced as f64) + } + } + } + fn on_approval_stale(&self) { if let Some(metrics) = &self.0 { metrics.approvals_produced_total.with_label_values(&["stale"]).inc() @@ -216,6 +258,18 @@ impl Metrics { } } + fn on_observed_no_shows(&self, n: usize) { + if let Some(metrics) = &self.0 { + metrics.observed_no_shows.inc_by(n as u64); + } + } + + fn on_approved_by_one_third(&self) { + if let Some(metrics) = &self.0 { + metrics.approved_by_one_third.inc(); + } + } + fn on_wakeup(&self) { if let Some(metrics) = &self.0 { metrics.wakeups_triggered_total.inc(); @@ -293,6 +347,13 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + observed_no_shows: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_approvals_observed_no_shows_total", + "Number of observed no shows at any moment in time", + )?, + registry, + )?, wakeups_triggered_total: prometheus::register( prometheus::Counter::new( "polkadot_parachain_approvals_wakeups_total", @@ -309,6 +370,22 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + coalesced_approvals_buckets: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_approvals_coalesced_approvals_buckets", + "Number of coalesced approvals.", + ).buckets(vec![1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5]), + )?, + registry, + )?, + approved_by_one_third: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_approved_by_one_third", + "Number of candidates where more than one third had to vote ", + )?, + registry, + )?, block_approval_time_ticks: prometheus::register( prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( @@ -364,6 +441,24 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, + ) -> Self { + Self::with_config_and_cache( + config, + db, + keystore, + sync_oracle, + metrics, + Some(LruCache::new(APPROVAL_PARAMS_CACHE_SIZE)), + ) + } + + pub fn with_config_and_cache( + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, + approval_voting_params_cache: Option>, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -372,6 +467,7 @@ impl ApprovalVotingSubsystem { db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, mode: Mode::Syncing(sync_oracle), metrics, + approval_voting_params_cache, } } @@ -379,8 +475,8 @@ impl ApprovalVotingSubsystem { /// The operation is not allowed for blocks older than the last finalized one. pub fn revert_to(&self, hash: Hash) -> Result<(), SubsystemError> { let config = - approval_db::v1::Config { col_approval_data: self.db_config.col_approval_data }; - let mut backend = approval_db::v1::DbBackend::new(self.db.clone(), config); + approval_db::v2::Config { col_approval_data: self.db_config.col_approval_data }; + let mut backend = approval_db::v2::DbBackend::new(self.db.clone(), config); let mut overlay = OverlayedBackend::new(&backend); ops::revert_to(&mut overlay, hash)?; @@ -555,6 +651,7 @@ struct ApprovalStatus { required_tranches: RequiredTranches, tranche_now: DelayTranche, block_tick: Tick, + last_no_shows: usize, } #[derive(Copy, Clone)] @@ -676,6 +773,9 @@ struct State { clock: Box, assignment_criteria: Box, spans: HashMap, + // Store approval-voting params, so that we don't to always go to RuntimeAPI + // to ask this information which requires a task context switch. + approval_voting_params_cache: Option>, } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] @@ -714,7 +814,7 @@ impl State { ); if let Some(approval_entry) = candidate_entry.approval_entry(&block_hash) { - let required_tranches = approval_checking::tranches_to_approve( + let (required_tranches, last_no_shows) = approval_checking::tranches_to_approve( approval_entry, candidate_entry.approvals(), tranche_now, @@ -723,13 +823,56 @@ impl State { session_info.needed_approvals as _, ); - let status = ApprovalStatus { required_tranches, block_tick, tranche_now }; + let status = + ApprovalStatus { required_tranches, block_tick, tranche_now, last_no_shows }; Some((approval_entry, status)) } else { None } } + + // Returns the approval voting from the RuntimeApi. + // To avoid crossing the subsystem boundary every-time we are caching locally the values. + #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] + async fn get_approval_voting_params_or_default( + &mut self, + _ctx: &mut Context, + block_hash: Hash, + ) -> ApprovalVotingParams { + if let Some(params) = self + .approval_voting_params_cache + .as_mut() + .and_then(|cache| cache.get(&block_hash)) + { + *params + } else { + // let (s_tx, s_rx) = oneshot::channel(); + + // ctx.send_message(RuntimeApiMessage::Request( + // block_hash, + // RuntimeApiRequest::ApprovalVotingParams(s_tx), + // )) + // .await; + + // match s_rx.await { + // Ok(Ok(params)) => { + // self.approval_voting_params_cache + // .as_mut() + // .map(|cache| cache.put(block_hash, params)); + // params + // }, + // _ => { + // gum::error!( + // target: LOG_TARGET, + // "Could not request approval voting params from runtime using defaults" + // ); + // ApprovalVotingParams { max_approval_coalesce_count: 1 } + // }, + // } + ApprovalVotingParams { max_approval_coalesce_count: 6 } + } + } } #[derive(Debug, Clone)] @@ -741,14 +884,15 @@ enum Action { tick: Tick, }, LaunchApproval { + claimed_candidate_indices: CandidateBitfield, candidate_hash: CandidateHash, - indirect_cert: IndirectAssignmentCert, + indirect_cert: IndirectAssignmentCertV2, assignment_tranche: DelayTranche, relay_block_hash: Hash, - candidate_index: CandidateIndex, session: SessionIndex, candidate: CandidateReceipt, backing_group: GroupIndex, + distribute_assignment: bool, }, NoteApprovedInChainSelection(Hash), IssueApproval(CandidateHash, ApprovalVoteRequest), @@ -777,6 +921,7 @@ where clock, assignment_criteria, spans: HashMap::new(), + approval_voting_params_cache: subsystem.approval_voting_params_cache.take(), }; // `None` on start-up. Gets initialized/updated on leaf update @@ -787,6 +932,7 @@ where let mut wakeups = Wakeups::default(); let mut currently_checking_set = CurrentlyCheckingSet::default(); let mut approvals_cache = lru::LruCache::new(APPROVAL_CACHE_SIZE); + let mut delayed_approvals_timers = DelayedApprovalTimer::default(); let mut last_finalized_height: Option = { let (tx, rx) = oneshot::channel(); @@ -864,18 +1010,50 @@ where } actions + }, + (block_hash, validator_index) = delayed_approvals_timers.select_next_some() => { + gum::debug!( + target: LOG_TARGET, + "Sign approval for multiple candidates", + ); + + let approval_params = state.get_approval_voting_params_or_default(&mut ctx, block_hash).await; + + match maybe_create_signature( + &mut overlayed_db, + &mut session_info_provider, + approval_params, + &state, &mut ctx, + block_hash, + validator_index, + &subsystem.metrics, + ).await { + Ok(Some(next_wakeup)) => { + delayed_approvals_timers.maybe_arm_timer(next_wakeup, state.clock.as_ref(), block_hash, validator_index); + }, + Ok(None) => {} + Err(err) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Failed to create signature", + ); + } + } + vec![] } }; if handle_actions( &mut ctx, - &state, + &mut state, &mut overlayed_db, &mut session_info_provider, &subsystem.metrics, &mut wakeups, &mut currently_checking_set, &mut approvals_cache, + &mut delayed_approvals_timers, &mut subsystem.mode, actions, ) @@ -916,13 +1094,14 @@ where #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn handle_actions( ctx: &mut Context, - state: &State, + state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, wakeups: &mut Wakeups, currently_checking_set: &mut CurrentlyCheckingSet, approvals_cache: &mut lru::LruCache, + delayed_approvals_timers: &mut DelayedApprovalTimer, mode: &mut Mode, actions: Vec, ) -> SubsystemResult { @@ -952,6 +1131,7 @@ async fn handle_actions( session_info_provider, metrics, candidate_hash, + delayed_approvals_timers, approval_request, ) .await? @@ -963,14 +1143,15 @@ async fn handle_actions( actions_iter = next_actions.into_iter(); }, Action::LaunchApproval { + claimed_candidate_indices, candidate_hash, indirect_cert, assignment_tranche, relay_block_hash, - candidate_index, session, candidate, backing_group, + distribute_assignment, } => { // Don't launch approval work if the node is syncing. if let Mode::Syncing(_) = *mode { @@ -991,10 +1172,12 @@ async fn handle_actions( launch_approval_span.add_string_tag("block-hash", format!("{:?}", block_hash)); let validator_index = indirect_cert.validator; - ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( - indirect_cert, - candidate_index, - )); + if distribute_assignment { + ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( + indirect_cert, + claimed_candidate_indices, + )); + } match approvals_cache.get(&candidate_hash) { Some(ApprovalOutcome::Approved) => { @@ -1048,7 +1231,11 @@ async fn handle_actions( Action::BecomeActive => { *mode = Mode::Active; - let messages = distribution_messages_for_activation(overlayed_db, state)?; + let messages = distribution_messages_for_activation( + overlayed_db, + state, + delayed_approvals_timers, + )?; ctx.send_messages(messages.into_iter()).await; }, @@ -1061,9 +1248,53 @@ async fn handle_actions( Ok(conclude) } +fn cores_to_candidate_indices( + core_indices: &CoreBitfield, + block_entry: &BlockEntry, +) -> Result { + let mut candidate_indices = Vec::new(); + + // Map from core index to candidate index. + for claimed_core_index in core_indices.iter_ones() { + if let Some(candidate_index) = block_entry + .candidates() + .iter() + .position(|(core_index, _)| core_index.0 == claimed_core_index as u32) + { + candidate_indices.push(candidate_index as _); + } + } + + CandidateBitfield::try_from(candidate_indices) +} + +// Returns the claimed core bitfield from the assignment cert, the candidate hash and a +// `BlockEntry`. Can fail only for VRF Delay assignments for which we cannot find the candidate hash +// in the block entry which indicates a bug or corrupted storage. +fn get_assignment_core_indices( + assignment: &AssignmentCertKindV2, + candidate_hash: &CandidateHash, + block_entry: &BlockEntry, +) -> Option { + match &assignment { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + Some(core_bitfield.clone()), + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => block_entry + .candidates() + .iter() + .find(|(_core_index, h)| candidate_hash == h) + .map(|(core_index, _candidate_hash)| { + CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed") + }), + AssignmentCertKindV2::RelayVRFDelay { core_index } => + Some(CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed")), + } +} + fn distribution_messages_for_activation( db: &OverlayedBackend<'_, impl Backend>, state: &State, + delayed_approvals_timers: &mut DelayedApprovalTimer, ) -> SubsystemResult> { let all_blocks: Vec = db.load_all_blocks()?; @@ -1102,7 +1333,7 @@ fn distribution_messages_for_activation( slot: block_entry.slot(), session: block_entry.session(), }); - + let mut signatures_queued = HashSet::new(); for (i, (_, candidate_hash)) in block_entry.candidates().iter().enumerate() { let _candidate_span = distribution_message_span.child("candidate").with_candidate(*candidate_hash); @@ -1125,33 +1356,108 @@ fn distribution_messages_for_activation( match approval_entry.local_statements() { (None, None) | (None, Some(_)) => {}, // second is impossible case. (Some(assignment), None) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + if block_entry.has_candidates_pending_signature() { + delayed_approvals_timers.maybe_arm_timer( + state.clock.tick_now(), + state.clock.as_ref(), + block_entry.block_hash(), + assignment.validator_index(), + ) + } + + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + // Should never happen. If we fail here it means the + // assignment is null (no cores claimed). + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + }, + } + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, (Some(assignment), Some(approval_sig)) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); - - messages.push(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: i as _, - validator: assignment.validator_index(), - signature: approval_sig, - }, - )) + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + // If we didn't send assignment, we don't send approval. + continue + }, + } + let candidate_indices = approval_sig + .signed_candidates_indices + .unwrap_or((i as CandidateIndex).into()); + if signatures_queued.insert(candidate_indices.clone()) { + messages.push(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices, + validator: assignment.validator_index(), + signature: approval_sig.signature, + }, + )) + }; + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, } }, @@ -1271,14 +1577,14 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_core, res) => { + ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, res) => { let (check_outcome, actions) = check_and_import_assignment( ctx.sender(), state, db, session_info_provider, a, - claimed_core, + claimed_cores, ) .await?; let _ = res.send(check_outcome); @@ -1349,7 +1655,7 @@ async fn get_approval_signatures_for_candidate( ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, - tx: oneshot::Sender>, + tx: oneshot::Sender, ValidatorSignature)>>, ) -> SubsystemResult<()> { let send_votes = |votes| { if let Err(_) = tx.send(votes) { @@ -1375,6 +1681,11 @@ async fn get_approval_signatures_for_candidate( let relay_hashes = entry.block_assignments.keys(); let mut candidate_indices = HashSet::new(); + let mut candidate_indices_to_candidate_hashes: HashMap< + Hash, + HashMap, + > = HashMap::new(); + // Retrieve `CoreIndices`/`CandidateIndices` as required by approval-distribution: for hash in relay_hashes { let entry = match db.load_block_entry(hash)? { @@ -1392,8 +1703,11 @@ async fn get_approval_signatures_for_candidate( for (candidate_index, (_core_index, c_hash)) in entry.candidates().iter().enumerate() { if c_hash == &candidate_hash { candidate_indices.insert((*hash, candidate_index as u32)); - break } + candidate_indices_to_candidate_hashes + .entry(*hash) + .or_default() + .insert(candidate_index as _, *c_hash); } } @@ -1418,7 +1732,24 @@ async fn get_approval_signatures_for_candidate( target: LOG_TARGET, "Request for approval signatures got cancelled by `approval-distribution`." ), - Some(Ok(votes)) => send_votes(votes), + Some(Ok(votes)) => { + let votes = votes + .into_iter() + .map(|(validator_index, (hash, signed_candidates_indices, signature))| { + let candidates_hashes = + candidate_indices_to_candidate_hashes.get(&hash).expect("Can't fail because it is the same hash we sent to approval-distribution; qed"); + let signed_candidates_hashes: Vec = + signed_candidates_indices + .into_iter() + .map(|candidate_index| { + *(candidates_hashes.get(&candidate_index).expect("Can't fail because we already checked the signature was valid, so we should be able to find the hash; qed")) + }) + .collect(); + (validator_index, (signed_candidates_hashes, signature)) + }) + .collect(); + send_votes(votes) + }, } }; @@ -1445,10 +1776,10 @@ async fn handle_approved_ancestor( const MAX_TRACING_WINDOW: usize = 200; const ABNORMAL_DEPTH_THRESHOLD: usize = 5; const LOGGING_DEPTH_THRESHOLD: usize = 10; + let mut span = span .child("handle-approved-ancestor") .with_stage(jaeger::Stage::ApprovalChecking); - use bitvec::{order::Lsb0, vec::BitVec}; let mut all_approved_max = None; @@ -1787,8 +2118,8 @@ async fn check_and_import_assignment( state: &State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - assignment: IndirectAssignmentCert, - candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + candidate_indices: CandidateBitfield, ) -> SubsystemResult<(AssignmentCheckResult, Vec)> where Sender: SubsystemSender, @@ -1801,9 +2132,12 @@ where .map(|span| span.child("check-and-import-assignment")) .unwrap_or_else(|| jaeger::Span::new(assignment.block_hash, "check-and-import-assignment")) .with_relay_parent(assignment.block_hash) - .with_uint_tag("candidate-index", candidate_index as u64) .with_stage(jaeger::Stage::ApprovalChecking); + for candidate_index in candidate_indices.iter_ones() { + check_and_import_assignment_span.add_uint_tag("candidate-index", candidate_index as u64); + } + let block_entry = match db.load_block_entry(&assignment.block_hash)? { Some(b) => b, None => @@ -1833,39 +2167,64 @@ where )), }; - let (claimed_core_index, assigned_candidate_hash) = - match block_entry.candidate(candidate_index as usize) { - Some((c, h)) => (*c, *h), + let n_cores = session_info.n_cores as usize; + + // Early check the candidate bitfield and core bitfields lengths < `n_cores`. + // Core bitfield length is checked later in `check_assignment_cert`. + if candidate_indices.len() > n_cores { + gum::debug!( + target: LOG_TARGET, + validator = assignment.validator.0, + n_cores, + candidate_bitfield_len = ?candidate_indices.len(), + "Oversized bitfield", + ); + + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_indices.len(), + )), + Vec::new(), + )) + } + + // The Compact VRF modulo assignment cert has multiple core assignments. + let mut backing_groups = Vec::new(); + let mut claimed_core_indices = Vec::new(); + let mut assigned_candidate_hashes = Vec::new(); + + for candidate_index in candidate_indices.iter_ones() { + let (claimed_core_index, assigned_candidate_hash) = + match block_entry.candidate(candidate_index) { + Some((c, h)) => (*c, *h), + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( + candidate_index as _, + )), + Vec::new(), + )), // no candidate at core. + }; + + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, None => return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( - candidate_index, + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + assigned_candidate_hash, )), Vec::new(), )), // no candidate at core. }; - check_and_import_assignment_span - .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); - check_and_import_assignment_span.add_string_tag( - "traceID", - format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), - ); - - let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { - Some(c) => c, - None => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( - candidate_index, - assigned_candidate_hash, - )), - Vec::new(), - )), - }; + check_and_import_assignment_span + .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); + check_and_import_assignment_span.add_string_tag( + "traceID", + format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), + ); - let res = { - // import the assignment. let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { Some(a) => a, None => @@ -1878,79 +2237,144 @@ where )), }; - let res = state.assignment_criteria.check_assignment_cert( - claimed_core_index, - assignment.validator, - &criteria::Config::from(session_info), - block_entry.relay_vrf_story(), - &assignment.cert, - approval_entry.backing_group(), - ); + backing_groups.push(approval_entry.backing_group()); + claimed_core_indices.push(claimed_core_index); + assigned_candidate_hashes.push(assigned_candidate_hash); + } - let tranche = match res { - Err(crate::criteria::InvalidAssignment(reason)) => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( - assignment.validator, - format!("{:?}", reason), - )), - Vec::new(), + // Error on null assignments. + if claimed_core_indices.is_empty() { + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", InvalidAssignmentReason::NullAssignment), + )), + Vec::new(), + )) + } + + // Check the assignment certificate. + let res = state.assignment_criteria.check_assignment_cert( + claimed_core_indices + .clone() + .try_into() + .expect("Checked for null assignment above; qed"), + assignment.validator, + &criteria::Config::from(session_info), + block_entry.relay_vrf_story(), + &assignment.cert, + backing_groups, + ); + + let tranche = match res { + Err(crate::criteria::InvalidAssignment(reason)) => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", reason), )), - Ok(tranche) => { - let current_tranche = - state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); + Vec::new(), + )), + Ok(tranche) => { + let current_tranche = + state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); - let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; + let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - if tranche >= too_far_in_future { - return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) - } + if tranche >= too_far_in_future { + return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) + } - tranche - }, - }; + tranche + }, + }; + + let mut actions = Vec::new(); + let res = { + let mut is_duplicate = true; + // Import the assignments for all cores in the cert. + for (assigned_candidate_hash, candidate_index) in + assigned_candidate_hashes.iter().zip(candidate_indices.iter_ones()) + { + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; - check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { + Some(a) => a, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::Internal( + assignment.block_hash, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; + is_duplicate &= approval_entry.is_assigned(assignment.validator); + approval_entry.import_assignment(tranche, assignment.validator, tick_now); + check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + + // We've imported a new assignment, so we need to schedule a wake-up for when that might + // no-show. + if let Some((approval_entry, status)) = state + .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) + .await + { + actions.extend(schedule_wakeup_action( + approval_entry, + block_entry.block_hash(), + block_entry.block_number(), + *assigned_candidate_hash, + status.block_tick, + tick_now, + status.required_tranches, + )); + } - let is_duplicate = approval_entry.is_assigned(assignment.validator); - approval_entry.import_assignment(tranche, assignment.validator, tick_now); + // We also write the candidate entry as it now contains the new candidate. + db.write_candidate_entry(candidate_entry.into()); + } + // Since we don't account for tranche in distribution message fingerprinting, some + // validators can be assigned to the same core (VRF modulo vs VRF delay). These can be + // safely ignored ignored. However, if an assignment is for multiple cores (these are only + // tranche0), we cannot ignore it, because it would mean ignoring other non duplicate + // assignments. if is_duplicate { AssignmentCheckResult::AcceptedDuplicate + } else if candidate_indices.count_ones() > 1 { + gum::trace!( + target: LOG_TARGET, + validator = assignment.validator.0, + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + ?tranche, + "Imported assignments for multiple cores.", + ); + + AssignmentCheckResult::Accepted } else { gum::trace!( target: LOG_TARGET, validator = assignment.validator.0, - candidate_hash = ?assigned_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Imported assignment.", + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + "Imported assignment for a single core.", ); AssignmentCheckResult::Accepted } }; - let mut actions = Vec::new(); - - // We've imported a new approval, so we need to schedule a wake-up for when that might no-show. - if let Some((approval_entry, status)) = state - .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) - .await - { - actions.extend(schedule_wakeup_action( - approval_entry, - block_entry.block_hash(), - block_entry.block_number(), - assigned_candidate_hash, - status.block_tick, - tick_now, - status.required_tranches, - )); - } - - // We also write the candidate entry as it now contains the new candidate. - db.write_candidate_entry(candidate_entry.into()); - Ok((res, actions)) } @@ -1960,7 +2384,7 @@ async fn check_and_import_approval( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, - approval: IndirectSignedApprovalVote, + approval: IndirectSignedApprovalVoteV2, with_response: impl FnOnce(ApprovalCheckResult) -> T, ) -> SubsystemResult<(Vec, T)> where @@ -1972,13 +2396,12 @@ where return Ok((Vec::new(), t)) }}; } - let mut span = state .spans .get(&approval.block_hash) .map(|span| span.child("check-and-import-approval")) .unwrap_or_else(|| jaeger::Span::new(approval.block_hash, "check-and-import-approval")) - .with_uint_tag("candidate-index", approval.candidate_index as u64) + .with_string_fmt_debug_tag("candidate-index", approval.candidate_indices.clone()) .with_relay_parent(approval.block_hash) .with_stage(jaeger::Stage::ApprovalChecking); @@ -1991,105 +2414,157 @@ where }, }; - let session_info = match get_session_info( - session_info_provider, - sender, - approval.block_hash, - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( - block_entry.session() - ),)) - }, - }; + let approved_candidates_info: Result, ApprovalCheckError> = + approval + .candidate_indices + .iter_ones() + .map(|candidate_index| { + block_entry + .candidate(candidate_index) + .ok_or(ApprovalCheckError::InvalidCandidateIndex(candidate_index as _)) + .map(|candidate| (candidate_index as _, candidate.1)) + }) + .collect(); - let approved_candidate_hash = match block_entry.candidate(approval.candidate_index as usize) { - Some((_, h)) => *h, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidCandidateIndex(approval.candidate_index), - )), + let approved_candidates_info = match approved_candidates_info { + Ok(approved_candidates_info) => approved_candidates_info, + Err(err) => { + respond_early!(ApprovalCheckResult::Bad(err)) + }, }; - span.add_string_tag("candidate-hash", format!("{:?}", approved_candidate_hash)); + span.add_string_tag("candidate-hashes", format!("{:?}", approved_candidates_info)); span.add_string_tag( - "traceID", - format!("{:?}", hash_to_trace_identifier(approved_candidate_hash.0)), + "traceIDs", + format!( + "{:?}", + approved_candidates_info + .iter() + .map(|(_, approved_candidate_hash)| hash_to_trace_identifier( + approved_candidate_hash.0 + )) + .collect_vec() + ), ); - let pubkey = match session_info.validators.get(approval.validator) { - Some(k) => k, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidValidatorIndex(approval.validator), - )), - }; + { + let session_info = match get_session_info( + session_info_provider, + sender, + approval.block_hash, + block_entry.session(), + ) + .await + { + Some(s) => s, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( + block_entry.session() + ),)) + }, + }; - // Signature check: - match DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking).check_signature( - &pubkey, - approved_candidate_hash, - block_entry.session(), - &approval.signature, - ) { - Err(_) => respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( - approval.validator - ),)), - Ok(()) => {}, - }; + let pubkey = match session_info.validators.get(approval.validator) { + Some(k) => k, + None => respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidValidatorIndex(approval.validator), + )), + }; - let candidate_entry = match db.load_candidate_entry(&approved_candidate_hash)? { - Some(c) => c, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate( - approval.candidate_index, - approved_candidate_hash - ),)) - }, - }; + gum::debug!( + target: LOG_TARGET, + "Received approval for num_candidates {:}", + approval.candidate_indices.count_ones() + ); - // Don't accept approvals until assignment. - match candidate_entry.approval_entry(&approval.block_hash) { - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::Internal( - approval.block_hash, - approved_candidate_hash - ),)) - }, - Some(e) if !e.is_assigned(approval.validator) => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment( - approval.validator - ),)) - }, - _ => {}, + let candidate_hashes: Vec = + approved_candidates_info.iter().map(|candidate| candidate.1).collect(); + // Signature check: + match DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()), + ) + .check_signature( + &pubkey, + *candidate_hashes.first().expect("Checked above this is not empty; qed"), + block_entry.session(), + &approval.signature, + ) { + Err(_) => { + gum::error!( + target: LOG_TARGET, + "Error while checking signature {:}", + approval.candidate_indices.count_ones() + ); + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( + approval.validator + ),)) + }, + Ok(()) => {}, + }; } - // importing the approval can be heavy as it may trigger acceptance for a series of blocks. - let t = with_response(ApprovalCheckResult::Accepted); + let mut actions = Vec::new(); + for (approval_candidate_index, approved_candidate_hash) in approved_candidates_info { + let block_entry = match db.load_block_entry(&approval.block_hash)? { + Some(b) => b, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownBlock( + approval.block_hash + ),)) + }, + }; - gum::trace!( - target: LOG_TARGET, - validator_index = approval.validator.0, - validator = ?pubkey, - candidate_hash = ?approved_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Importing approval vote", - ); + let candidate_entry = match db.load_candidate_entry(&approved_candidate_hash)? { + Some(c) => c, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate( + approval_candidate_index, + approved_candidate_hash + ),)) + }, + }; - let actions = advance_approval_state( - sender, - state, - db, - session_info_provider, - &metrics, - block_entry, - approved_candidate_hash, - candidate_entry, - ApprovalStateTransition::RemoteApproval(approval.validator), - ) - .await; + // Don't accept approvals until assignment. + match candidate_entry.approval_entry(&approval.block_hash) { + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::Internal( + approval.block_hash, + approved_candidate_hash + ),)) + }, + Some(e) if !e.is_assigned(approval.validator) => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment( + approval.validator + ),)) + }, + _ => {}, + } + + gum::debug!( + target: LOG_TARGET, + validator_index = approval.validator.0, + candidate_hash = ?approved_candidate_hash, + para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, + "Importing approval vote", + ); + + let new_actions = advance_approval_state( + sender, + state, + db, + session_info_provider, + &metrics, + block_entry, + approved_candidate_hash, + candidate_entry, + ApprovalStateTransition::RemoteApproval(approval.validator), + ) + .await; + actions.extend(new_actions); + } + + // importing the approval can be heavy as it may trigger acceptance for a series of blocks. + let t = with_response(ApprovalCheckResult::Accepted); Ok((actions, t)) } @@ -2097,7 +2572,7 @@ where #[derive(Debug)] enum ApprovalStateTransition { RemoteApproval(ValidatorIndex), - LocalApproval(ValidatorIndex, ValidatorSignature), + LocalApproval(ValidatorIndex), WakeupProcessed, } @@ -2105,7 +2580,7 @@ impl ApprovalStateTransition { fn validator_index(&self) -> Option { match *self { ApprovalStateTransition::RemoteApproval(v) | - ApprovalStateTransition::LocalApproval(v, _) => Some(v), + ApprovalStateTransition::LocalApproval(v) => Some(v), ApprovalStateTransition::WakeupProcessed => None, } } @@ -2113,7 +2588,7 @@ impl ApprovalStateTransition { fn is_local_approval(&self) -> bool { match *self { ApprovalStateTransition::RemoteApproval(_) => false, - ApprovalStateTransition::LocalApproval(_, _) => true, + ApprovalStateTransition::LocalApproval(_) => true, ApprovalStateTransition::WakeupProcessed => false, } } @@ -2180,7 +2655,16 @@ where // assignment tick of `now - APPROVAL_DELAY` - that is, that // all counted assignments are at least `APPROVAL_DELAY` ticks old. let is_approved = check.is_approved(tick_now.saturating_sub(APPROVAL_DELAY)); - + if status.last_no_shows != 0 { + metrics.on_observed_no_shows(status.last_no_shows); + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + last_no_shows = ?status.last_no_shows, + "Observed no_shows", + ); + } if is_approved { gum::trace!( target: LOG_TARGET, @@ -2198,6 +2682,12 @@ where if no_shows != 0 { metrics.on_no_shows(no_shows); } + if check == Check::ApprovedOneThird { + // No-shows are not counted when more than one third of validators approve a + // candidate, so count candidates where more than one third of validators had to + // approve it, this is indicative of something breaking. + metrics.on_approved_by_one_third() + } metrics.on_candidate_approved(status.tranche_now as _); @@ -2206,6 +2696,10 @@ where actions.push(Action::NoteApprovedInChainSelection(block_hash)); } + db.write_block_entry(block_entry.into()); + } else if transition.is_local_approval() { + // Local approvals always update the block_entry, so we need to flush it to + // the database. db.write_block_entry(block_entry.into()); } @@ -2234,10 +2728,6 @@ where approval_entry.mark_approved(); } - if let ApprovalStateTransition::LocalApproval(_, ref sig) = transition { - approval_entry.import_approval_sig(sig.clone()); - } - actions.extend(schedule_wakeup_action( &approval_entry, block_hash, @@ -2324,7 +2814,7 @@ async fn process_wakeup( let candidate_entry = db.load_candidate_entry(&candidate_hash)?; // If either is not present, we have nothing to wakeup. Might have lost a race with finality - let (block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { + let (mut block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { (Some(b), Some(c)) => (b, c), _ => return Ok(Vec::new()), }; @@ -2362,7 +2852,7 @@ async fn process_wakeup( None => return Ok(Vec::new()), }; - let tranches_to_approve = approval_checking::tranches_to_approve( + let (tranches_to_approve, _last_no_shows) = approval_checking::tranches_to_approve( &approval_entry, candidate_entry.approvals(), tranche_now, @@ -2404,31 +2894,58 @@ async fn process_wakeup( if let Some((cert, val_index, tranche)) = maybe_cert { let indirect_cert = - IndirectAssignmentCert { block_hash: relay_block, validator: val_index, cert }; + IndirectAssignmentCertV2 { block_hash: relay_block, validator: val_index, cert }; - let index_in_candidate = - block_entry.candidates().iter().position(|(_, h)| &candidate_hash == h); + gum::trace!( + target: LOG_TARGET, + ?candidate_hash, + para_id = ?candidate_receipt.descriptor.para_id, + block_hash = ?relay_block, + "Launching approval work.", + ); - if let Some(i) = index_in_candidate { - gum::trace!( + if let Some(claimed_core_indices) = + get_assignment_core_indices(&indirect_cert.cert.kind, &candidate_hash, &block_entry) + { + match cores_to_candidate_indices(&claimed_core_indices, &block_entry) { + Ok(claimed_candidate_indices) => { + // Ensure we distribute multiple core assignments just once. + let distribute_assignment = if claimed_candidate_indices.count_ones() > 1 { + !block_entry.mark_assignment_distributed(claimed_candidate_indices.clone()) + } else { + true + }; + db.write_block_entry(block_entry.clone()); + + actions.push(Action::LaunchApproval { + claimed_candidate_indices, + candidate_hash, + indirect_cert, + assignment_tranche: tranche, + relay_block_hash: relay_block, + session: block_entry.session(), + candidate: candidate_receipt, + backing_group, + distribute_assignment, + }); + }, + Err(err) => { + // Never happens, it should only happen if no cores are claimed, which is a bug. + gum::warn!( + target: LOG_TARGET, + block_hash = ?relay_block, + ?err, + "Failed to create assignment bitfield" + ); + }, + }; + } else { + gum::warn!( target: LOG_TARGET, - ?candidate_hash, - para_id = ?candidate_receipt.descriptor.para_id, block_hash = ?relay_block, - "Launching approval work.", + ?candidate_hash, + "Cannot get assignment claimed core indices", ); - - // sanity: should always be present. - actions.push(Action::LaunchApproval { - candidate_hash, - indirect_cert, - assignment_tranche: tranche, - relay_block_hash: relay_block, - candidate_index: i as _, - session: block_entry.session(), - candidate: candidate_receipt, - backing_group, - }); } } // Although we checked approval earlier in this function, @@ -2669,11 +3186,12 @@ async fn launch_approval( #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn issue_approval( ctx: &mut Context, - state: &State, + state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, candidate_hash: CandidateHash, + delayed_approvals_timers: &mut DelayedApprovalTimer, ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, ) -> SubsystemResult> { let mut issue_approval_span = state @@ -2687,7 +3205,7 @@ async fn issue_approval( .with_validator_index(validator_index) .with_stage(jaeger::Stage::ApprovalChecking); - let block_entry = match db.load_block_entry(&block_hash)? { + let mut block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, None => { // not a cause for alarm - just lost a race with pruning, most likely. @@ -2713,21 +3231,6 @@ async fn issue_approval( }; issue_approval_span.add_int_tag("candidate_index", candidate_index as i64); - let session_info = match get_session_info( - session_info_provider, - ctx.sender(), - block_entry.parent_hash(), - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - metrics.on_approval_error(); - return Ok(Vec::new()) - }, - }; - let candidate_hash = match block_entry.candidate(candidate_index as usize) { Some((_, h)) => *h, None => { @@ -2758,10 +3261,150 @@ async fn issue_approval( }, }; + let session_info = match get_session_info( + session_info_provider, + ctx.sender(), + block_entry.parent_hash(), + block_entry.session(), + ) + .await + { + Some(s) => s, + None => return Ok(Vec::new()), + }; + + if block_entry + .defer_candidate_signature( + candidate_index as _, + candidate_hash, + compute_delayed_approval_sending_tick(state, &block_entry, session_info), + ) + .is_some() + { + gum::error!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + validator_index = validator_index.0, + "Possible bug, we shouldn't have to defer a candidate more than once", + ); + } + + gum::info!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + validator_index = validator_index.0, + "Ready to issue approval vote", + ); + + let approval_params = state.get_approval_voting_params_or_default(ctx, block_hash).await; + + let actions = advance_approval_state( + ctx.sender(), + state, + db, + session_info_provider, + metrics, + block_entry, + candidate_hash, + candidate_entry, + ApprovalStateTransition::LocalApproval(validator_index as _), + ) + .await; + + if let Some(next_wakeup) = maybe_create_signature( + db, + session_info_provider, + approval_params, + state, + ctx, + block_hash, + validator_index, + metrics, + ) + .await? + { + delayed_approvals_timers.maybe_arm_timer( + next_wakeup, + state.clock.as_ref(), + block_hash, + validator_index, + ); + } + Ok(actions) +} + +// Create signature for the approved candidates pending signatures +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] +async fn maybe_create_signature( + db: &mut OverlayedBackend<'_, impl Backend>, + session_info_provider: &mut RuntimeInfo, + approval_params: ApprovalVotingParams, + state: &State, + ctx: &mut Context, + block_hash: Hash, + validator_index: ValidatorIndex, + metrics: &Metrics, +) -> SubsystemResult> { + let mut block_entry = match db.load_block_entry(&block_hash)? { + Some(b) => b, + None => { + // not a cause for alarm - just lost a race with pruning, most likely. + metrics.on_approval_stale(); + gum::info!( + target: LOG_TARGET, + "Could not find block that needs signature {:}", block_hash + ); + return Ok(None) + }, + }; + + gum::debug!( + target: LOG_TARGET, + "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() + ); + + let oldest_candidate_to_sign = match block_entry.longest_waiting_candidate_signature() { + Some(candidate) => candidate, + // No cached candidates, nothing to do here, this just means the timer fired, + // but the signatures were already sent because we gathered more than + // max_approval_coalesce_count. + None => return Ok(None), + }; + + let tick_now = state.clock.tick_now(); + + if oldest_candidate_to_sign.send_no_later_than_tick > tick_now && + (block_entry.num_candidates_pending_signature() as u32) < + approval_params.max_approval_coalesce_count + { + return Ok(Some(oldest_candidate_to_sign.send_no_later_than_tick)) + } + + let session_info = match get_session_info( + session_info_provider, + ctx.sender(), + block_entry.parent_hash(), + block_entry.session(), + ) + .await + { + Some(s) => s, + None => { + metrics.on_approval_error(); + gum::error!( + target: LOG_TARGET, + "Could not retrieve the session" + ); + return Ok(None) + }, + }; + let validator_pubkey = match session_info.validators.get(validator_index) { Some(p) => p, None => { - gum::warn!( + gum::error!( target: LOG_TARGET, "Validator index {} out of bounds in session {}", validator_index.0, @@ -2769,72 +3412,93 @@ async fn issue_approval( ); metrics.on_approval_error(); - return Ok(Vec::new()) + return Ok(None) }, }; - let session = block_entry.session(); - let sig = match sign_approval(&state.keystore, &validator_pubkey, candidate_hash, session) { + let candidate_hashes = block_entry.candidate_hashes_pending_signature(); + + let signature = match sign_approval( + &state.keystore, + &validator_pubkey, + candidate_hashes.clone(), + block_entry.session(), + ) { Some(sig) => sig, None => { - gum::warn!( + gum::error!( target: LOG_TARGET, validator_index = ?validator_index, - session, + session = ?block_entry.session(), "Could not issue approval signature. Assignment key present but not validator key?", ); metrics.on_approval_error(); - return Ok(Vec::new()) + return Ok(None) }, }; - gum::trace!( - target: LOG_TARGET, - ?candidate_hash, - ?block_hash, - validator_index = validator_index.0, - "Issuing approval vote", - ); + let candidate_entries = candidate_hashes + .iter() + .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) + .collect::>>>()?; - let actions = advance_approval_state( - ctx.sender(), - state, - db, - session_info_provider, - metrics, - block_entry, - candidate_hash, - candidate_entry, - ApprovalStateTransition::LocalApproval(validator_index as _, sig.clone()), - ) - .await; + let candidate_indices = block_entry.candidate_indices_pending_signature(); + + for candidate_entry in candidate_entries { + let mut candidate_entry = candidate_entry + .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + let approval_entry = candidate_entry + .approval_entry_mut(&block_entry.block_hash()) + .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + approval_entry.import_approval_sig(OurApproval { + signature: signature.clone(), + signed_candidates_indices: Some( + candidate_indices + .clone() + .try_into() + .expect("Fails only of array empty, it can't be, qed"), + ), + }); + db.write_candidate_entry(candidate_entry); + } + + metrics.on_approval_coalesce(candidate_indices.len() as u32); metrics.on_approval_produced(); - // dispatch to approval distribution. ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: candidate_index as _, + IndirectSignedApprovalVoteV2 { + block_hash: block_entry.block_hash(), + candidate_indices: candidate_indices + .try_into() + .expect("Fails only of array empty, it can't be, qed"), validator: validator_index, - signature: sig, + signature, }, )); - Ok(actions) + gum::debug!( + target: LOG_TARGET, + ?block_hash, + signed_candidates = ?block_entry.num_candidates_pending_signature(), + "Issue approval votes", + ); + block_entry.issued_approval(); + db.write_block_entry(block_entry.into()); + Ok(None) } // Sign an approval vote. Fails if the key isn't present in the store. fn sign_approval( keystore: &LocalKeystore, public: &ValidatorId, - candidate_hash: CandidateHash, + candidate_hashes: Vec, session_index: SessionIndex, ) -> Option { let key = keystore.key_pair::(public).ok().flatten()?; - let payload = ApprovalVote(candidate_hash).signing_payload(session_index); + let payload = ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index); Some(key.sign(&payload[..])) } @@ -2864,3 +3528,27 @@ fn issue_local_invalid_statement( false, )); } + +// Computes what is the latest tick we can send an approval +fn compute_delayed_approval_sending_tick( + state: &State, + block_entry: &BlockEntry, + session_info: &SessionInfo, +) -> Tick { + let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); + + let no_show_duration_ticks = slot_number_to_tick( + state.slot_duration_millis, + Slot::from(u64::from(session_info.no_show_slots)), + ); + let tick_now = state.clock.tick_now(); + + min( + tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, + // We don't want to accidentally cause, no-shows so if we are past + // the 2 thirds of the no show time, force the sending of the + // approval immediately. + // TODO: TBD the right value here, so that we don't accidentally create no-shows. + current_block_tick + (no_show_duration_ticks * 2) / 3, + ) +} diff --git a/node/core/approval-voting/src/ops.rs b/node/core/approval-voting/src/ops.rs index 6f57b2f80e8a..a6f0ecf9d1f0 100644 --- a/node/core/approval-voting/src/ops.rs +++ b/node/core/approval-voting/src/ops.rs @@ -25,7 +25,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, GroupInd use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use super::{ - approval_db::v1::{OurAssignment, StoredBlockRange}, + approval_db::v2::{OurAssignment, StoredBlockRange}, backend::{Backend, OverlayedBackend}, persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}, LOG_TARGET, diff --git a/node/core/approval-voting/src/persisted_entries.rs b/node/core/approval-voting/src/persisted_entries.rs index 9b6592220275..e441c23be122 100644 --- a/node/core/approval-voting/src/persisted_entries.rs +++ b/node/core/approval-voting/src/persisted_entries.rs @@ -20,16 +20,21 @@ //! Within that context, things are plain-old-data. Within this module, //! data and logic are intertwined. -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche, RelayVRFStory}; +use polkadot_node_primitives::approval::{ + v1::{DelayTranche, RelayVRFStory}, + v2::{AssignmentCertV2, CandidateBitfield}, +}; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, - ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; -use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice, vec::BitVec}; +use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; use std::collections::BTreeMap; +use crate::approval_db::v2::Bitfield; + use super::{criteria::OurAssignment, time::Tick}; /// Metadata regarding a specific tranche of assignments for a specific candidate. @@ -53,8 +58,8 @@ impl TrancheEntry { } } -impl From for TrancheEntry { - fn from(entry: crate::approval_db::v1::TrancheEntry) -> Self { +impl From for TrancheEntry { + fn from(entry: crate::approval_db::v2::TrancheEntry) -> Self { TrancheEntry { tranche: entry.tranche, assignments: entry.assignments.into_iter().map(|(v, t)| (v, t.into())).collect(), @@ -62,7 +67,7 @@ impl From for TrancheEntry { } } -impl From for crate::approval_db::v1::TrancheEntry { +impl From for crate::approval_db::v2::TrancheEntry { fn from(entry: TrancheEntry) -> Self { Self { tranche: entry.tranche, @@ -71,6 +76,39 @@ impl From for crate::approval_db::v1::TrancheEntry { } } +impl From for OurApproval { + fn from(approval: crate::approval_db::v2::OurApproval) -> Self { + Self { + signature: approval.signature, + signed_candidates_indices: approval.signed_candidates_indices, + } + } +} +impl From for crate::approval_db::v2::OurApproval { + fn from(approval: OurApproval) -> Self { + Self { + signature: approval.signature, + signed_candidates_indices: approval.signed_candidates_indices, + } + } +} + +impl From for OurApproval { + fn from(value: ValidatorSignature) -> Self { + Self { signature: value, signed_candidates_indices: Default::default() } + } +} + +/// Metadata about our approval signature +#[derive(Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval, an empty value means only + /// the candidate referred by this approval entry was signed. + pub signed_candidates_indices: Option, +} + /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Debug, Clone, PartialEq)] @@ -78,9 +116,9 @@ pub struct ApprovalEntry { tranches: Vec, backing_group: GroupIndex, our_assignment: Option, - our_approval_sig: Option, + our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, } @@ -90,12 +128,19 @@ impl ApprovalEntry { tranches: Vec, backing_group: GroupIndex, our_assignment: Option, - our_approval_sig: Option, + our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, ) -> Self { - Self { tranches, backing_group, our_assignment, our_approval_sig, assignments, approved } + Self { + tranches, + backing_group, + our_assignment, + our_approval_sig, + assigned_validators, + approved, + } } // Access our assignment for this approval entry. @@ -107,7 +152,7 @@ impl ApprovalEntry { pub fn trigger_our_assignment( &mut self, tick_now: Tick, - ) -> Option<(AssignmentCert, ValidatorIndex, DelayTranche)> { + ) -> Option<(AssignmentCertV2, ValidatorIndex, DelayTranche)> { let our = self.our_assignment.as_mut().and_then(|a| { if a.triggered() { return None @@ -125,13 +170,16 @@ impl ApprovalEntry { } /// Import our local approval vote signature for this candidate. - pub fn import_approval_sig(&mut self, approval_sig: ValidatorSignature) { + pub fn import_approval_sig(&mut self, approval_sig: OurApproval) { self.our_approval_sig = Some(approval_sig); } /// Whether a validator is already assigned. pub fn is_assigned(&self, validator_index: ValidatorIndex) -> bool { - self.assignments.get(validator_index.0 as usize).map(|b| *b).unwrap_or(false) + self.assigned_validators + .get(validator_index.0 as usize) + .map(|b| *b) + .unwrap_or(false) } /// Import an assignment. No-op if already assigned on the same tranche. @@ -158,14 +206,14 @@ impl ApprovalEntry { }; self.tranches[idx].assignments.push((validator_index, tick_now)); - self.assignments.set(validator_index.0 as _, true); + self.assigned_validators.set(validator_index.0 as _, true); } // Produce a bitvec indicating the assignments of all validators up to and // including `tranche`. - pub fn assignments_up_to(&self, tranche: DelayTranche) -> BitVec { + pub fn assignments_up_to(&self, tranche: DelayTranche) -> Bitfield { self.tranches.iter().take_while(|e| e.tranche <= tranche).fold( - bitvec::bitvec![u8, BitOrderLsb0; 0; self.assignments.len()], + bitvec::bitvec![u8, BitOrderLsb0; 0; self.assigned_validators.len()], |mut a, e| { for &(v, _) in &e.assignments { a.set(v.0 as _, true); @@ -193,12 +241,12 @@ impl ApprovalEntry { /// Get the number of validators in this approval entry. pub fn n_validators(&self) -> usize { - self.assignments.len() + self.assigned_validators.len() } /// Get the number of assignments by validators, including the local validator. pub fn n_assignments(&self) -> usize { - self.assignments.count_ones() + self.assigned_validators.count_ones() } /// Get the backing group index of the approval entry. @@ -209,7 +257,7 @@ impl ApprovalEntry { /// Get the assignment cert & approval signature. /// /// The approval signature will only be `Some` if the assignment is too. - pub fn local_statements(&self) -> (Option, Option) { + pub fn local_statements(&self) -> (Option, Option) { let approval_sig = self.our_approval_sig.clone(); if let Some(our_assignment) = self.our_assignment.as_ref().filter(|a| a.triggered()) { (Some(our_assignment.clone()), approval_sig) @@ -219,27 +267,27 @@ impl ApprovalEntry { } } -impl From for ApprovalEntry { - fn from(entry: crate::approval_db::v1::ApprovalEntry) -> Self { +impl From for ApprovalEntry { + fn from(entry: crate::approval_db::v2::ApprovalEntry) -> Self { ApprovalEntry { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } } -impl From for crate::approval_db::v1::ApprovalEntry { +impl From for crate::approval_db::v2::ApprovalEntry { fn from(entry: ApprovalEntry) -> Self { Self { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } @@ -253,7 +301,7 @@ pub struct CandidateEntry { // Assignments are based on blocks, so we need to track assignments separately // based on the block we are looking at. pub block_assignments: BTreeMap, - pub approvals: BitVec, + pub approvals: Bitfield, } impl CandidateEntry { @@ -290,8 +338,8 @@ impl CandidateEntry { } } -impl From for CandidateEntry { - fn from(entry: crate::approval_db::v1::CandidateEntry) -> Self { +impl From for CandidateEntry { + fn from(entry: crate::approval_db::v2::CandidateEntry) -> Self { CandidateEntry { candidate: entry.candidate, session: entry.session, @@ -305,7 +353,7 @@ impl From for CandidateEntry { } } -impl From for crate::approval_db::v1::CandidateEntry { +impl From for crate::approval_db::v2::CandidateEntry { fn from(entry: CandidateEntry) -> Self { Self { candidate: entry.candidate, @@ -336,8 +384,21 @@ pub struct BlockEntry { // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. // The i'th bit is `true` iff the candidate has been approved in the context of this // block. The block can be considered approved if the bitfield has all bits set to `true`. - pub approved_bitfield: BitVec, + pub approved_bitfield: Bitfield, pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + candidates_pending_signature: BTreeMap, + // A list of assignments for which wea already distributed the assignment. + // We use this to ensure we don't distribute multiple core assignments twice as we track + // individual wakeups for each core. + distributed_assignments: Bitfield, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateSigningContext { + pub candidate_hash: CandidateHash, + pub send_no_later_than_tick: Tick, } impl BlockEntry { @@ -412,6 +473,92 @@ impl BlockEntry { pub fn parent_hash(&self) -> Hash { self.parent_hash } + + /// Mark distributed assignment for many candidate indices. + /// Returns `true` if an assignment was already distributed for the `candidates`. + pub fn mark_assignment_distributed(&mut self, candidates: CandidateBitfield) -> bool { + let bitfield = candidates.into_inner(); + let total_one_bits = self.distributed_assignments.count_ones(); + + let new_len = std::cmp::max(self.distributed_assignments.len(), bitfield.len()); + self.distributed_assignments.resize(new_len, false); + self.distributed_assignments |= bitfield; + + // If the an operation did not change our current bitfied, we return true. + let distributed = total_one_bits == self.distributed_assignments.count_ones(); + + distributed + } + + /// Defer signing and issuing an approval for a candidate no later than the specified tick + pub fn defer_candidate_signature( + &mut self, + candidate_index: CandidateIndex, + candidate_hash: CandidateHash, + send_no_later_than_tick: Tick, + ) -> Option { + self.candidates_pending_signature.insert( + candidate_index, + CandidateSigningContext { candidate_hash, send_no_later_than_tick }, + ) + } + + /// Returns the number of candidates waiting for an approval to be issued. + pub fn num_candidates_pending_signature(&self) -> usize { + self.candidates_pending_signature.len() + } + + /// Return if we have candidates waiting for signature to be issued + pub fn has_candidates_pending_signature(&self) -> bool { + !self.candidates_pending_signature.is_empty() + } + + /// Candidate hashes for candidates pending signatures + pub fn candidate_hashes_pending_signature(&self) -> Vec { + self.candidates_pending_signature + .values() + .map(|unsigned_approval| unsigned_approval.candidate_hash) + .collect() + } + + /// Candidate indices for candidates pending signature + pub fn candidate_indices_pending_signature(&self) -> Vec { + self.candidates_pending_signature.keys().map(|val| *val).collect() + } + + /// Returns the candidate that has been longest in the queue. + pub fn longest_waiting_candidate_signature(&self) -> Option<&CandidateSigningContext> { + self.candidates_pending_signature + .values() + .min_by(|a, b| a.send_no_later_than_tick.cmp(&b.send_no_later_than_tick)) + } + + /// Signals the approval was issued for the candidates pending signature + pub fn issued_approval(&mut self) { + self.candidates_pending_signature.clear(); + } +} + +impl From for BlockEntry { + fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { + BlockEntry { + block_hash: entry.block_hash, + parent_hash: entry.parent_hash, + block_number: entry.block_number, + session: entry.session, + slot: entry.slot, + relay_vrf_story: RelayVRFStory(entry.relay_vrf_story), + candidates: entry.candidates, + approved_bitfield: entry.approved_bitfield, + children: entry.children, + candidates_pending_signature: entry + .candidates_pending_signature + .into_iter() + .map(|(candidate_index, signing_context)| (candidate_index, signing_context.into())) + .collect(), + distributed_assignments: entry.distributed_assignments, + } + } } impl From for BlockEntry { @@ -426,11 +573,13 @@ impl From for BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: Default::default(), + candidates_pending_signature: Default::default(), } } } -impl From for crate::approval_db::v1::BlockEntry { +impl From for crate::approval_db::v2::BlockEntry { fn from(entry: BlockEntry) -> Self { Self { block_hash: entry.block_hash, @@ -442,6 +591,59 @@ impl From for crate::approval_db::v1::BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + candidates_pending_signature: entry + .candidates_pending_signature + .into_iter() + .map(|(candidate_index, signing_context)| (candidate_index, signing_context.into())) + .collect(), + distributed_assignments: entry.distributed_assignments, + } + } +} + +impl From for CandidateSigningContext { + fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { + Self { + candidate_hash: signing_context.candidate_hash, + send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + } + } +} + +impl From for crate::approval_db::v2::CandidateSigningContext { + fn from(signing_context: CandidateSigningContext) -> Self { + Self { + candidate_hash: signing_context.candidate_hash, + send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + } + } +} + +/// Migration helpers. +impl From for CandidateEntry { + fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ae.into())) + .collect(), + candidate: value.candidate, + session: value.session, + } + } +} + +impl From for ApprovalEntry { + fn from(value: crate::approval_db::v1::ApprovalEntry) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value.our_approval_sig.map(Into::into), + assigned_validators: value.assignments, + approved: value.approved, } } } diff --git a/node/core/approval-voting/src/tests.rs b/node/core/approval-voting/src/tests.rs index f58e60c6a487..3ced911f7109 100644 --- a/node/core/approval-voting/src/tests.rs +++ b/node/core/approval-voting/src/tests.rs @@ -15,10 +15,14 @@ // along with Polkadot. If not, see . use super::*; +use crate::backend::V1ReadBackend; use polkadot_node_primitives::{ approval::{ - AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, - RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, + RELAY_VRF_MODULO_CONTEXT, + }, + v2::{AssignmentCertKindV2, AssignmentCertV2}, }, AvailableData, BlockData, PoV, }; @@ -32,8 +36,8 @@ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_overseer::HeadSupportsParachains; use polkadot_primitives::{ - CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, Id as ParaId, IndexedVec, - ValidationCode, ValidatorSignature, + ApprovalVote, CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, + Id as ParaId, IndexedVec, ValidationCode, ValidatorSignature, }; use std::time::Duration; @@ -51,7 +55,7 @@ use std::{ }; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, backend::BackendWriteOp, import::tests::{ garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, @@ -111,7 +115,7 @@ fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHan #[cfg(test)] pub mod test_constants { - use crate::approval_db::v1::Config as DatabaseConfig; + use crate::approval_db::v2::Config as DatabaseConfig; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; @@ -161,7 +165,7 @@ impl Clock for MockClock { // This mock clock allows us to manipulate the time and // be notified when wakeups have been triggered. -#[derive(Default)] +#[derive(Default, Debug)] struct MockClockInner { tick: Tick, wakeups: Vec<(Tick, oneshot::Sender<()>)>, @@ -231,7 +235,7 @@ where fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -244,13 +248,13 @@ where fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, validator_index: ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result { self.1(validator_index) } } @@ -271,6 +275,18 @@ struct TestStoreInner { candidate_entries: HashMap, } +impl V1ReadBackend for TestStoreInner { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStoreInner { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { Ok(self.block_entries.get(block_hash).cloned()) @@ -342,6 +358,18 @@ pub struct TestStore { store: Arc>, } +impl V1ReadBackend for TestStore { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStore { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { let store = self.store.lock(); @@ -391,6 +419,17 @@ fn garbage_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } } +fn garbage_assignment_cert_v2(kind: AssignmentCertKindV2) -> AssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCertV2 { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + fn sign_approval( key: Sr25519Keyring, candidate_hash: CandidateHash, @@ -399,6 +438,15 @@ fn sign_approval( key.sign(&ApprovalVote(candidate_hash).signing_payload(session_index)).into() } +fn sign_approval_multiple_candidates( + key: Sr25519Keyring, + candidate_hashes: Vec, + session_index: SessionIndex, +) -> ValidatorSignature { + key.sign(&ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index)) + .into() +} + type VirtualOverseer = test_helpers::TestSubsystemContextHandle; #[derive(Default)] @@ -467,6 +515,12 @@ fn test_harness>( config: HarnessConfig, test: impl FnOnce(TestHarness) -> T, ) { + let _ = env_logger::builder() + .is_test(true) + .filter(Some("polkadot_node_core_approval_voting"), log::LevelFilter::Trace) + .filter(Some(LOG_TARGET), log::LevelFilter::Trace) + .try_init(); + let HarnessConfig { sync_oracle, sync_oracle_handle, clock, backend, assignment_criteria } = config; @@ -485,7 +539,7 @@ fn test_harness>( let subsystem = run( context, - ApprovalVotingSubsystem::with_config( + ApprovalVotingSubsystem::with_config_and_cache( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, slot_duration_millis: SLOT_DURATION_MILLIS, @@ -494,6 +548,7 @@ fn test_harness>( Arc::new(keystore), sync_oracle, Metrics::default(), + None, ), clock.clone(), assignment_criteria, @@ -588,7 +643,12 @@ async fn check_and_import_approval( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportApproval( - IndirectSignedApprovalVote { block_hash, candidate_index, validator, signature }, + IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices: candidate_index.into(), + validator, + signature, + }, tx, ), }, @@ -616,12 +676,13 @@ async fn check_and_import_assignment( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash, validator, - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), }, - candidate_index, + candidate_index.into(), tx, ), }, @@ -630,6 +691,38 @@ async fn check_and_import_assignment( rx } +async fn check_and_import_assignment_v2( + overseer: &mut VirtualOverseer, + block_hash: Hash, + core_indices: Vec, + validator: ValidatorIndex, +) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + overseer_send( + overseer, + FromOrchestra::Communication { + msg: ApprovalVotingMessage::CheckAndImportAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: core_indices + .clone() + .into_iter() + .map(|c| CoreIndex(c)) + .collect::>() + .try_into() + .unwrap(), + }), + }, + core_indices.try_into().unwrap(), + tx, + ), + }, + ) + .await; + rx +} struct BlockConfig { slot: Slot, candidates: Option>, @@ -742,7 +835,7 @@ fn session_info(keys: &[Sr25519Keyring]) -> SessionInfo { vec![ValidatorIndex(0)], vec![ValidatorIndex(1)], ]), - n_cores: keys.len() as _, + n_cores: 10, needed_approvals: 2, zeroth_delay_tranche_width: 5, relay_vrf_modulo_samples: 3, @@ -1059,14 +1152,15 @@ fn blank_subsystem_act_on_bad_block() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { - block_hash: bad_block_hash, + IndirectAssignmentCertV2 { + block_hash: bad_block_hash.clone(), validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1322,9 +1416,22 @@ fn subsystem_accepts_duplicate_assignment() { } ); - let block_hash = Hash::repeat_byte(0x01); - let candidate_index = 0; let validator = ValidatorIndex(0); + let candidate_index = 0; + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; // Add block hash 00. ChainBuilder::new() @@ -1332,21 +1439,30 @@ fn subsystem_accepts_duplicate_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: Some(vec![ + (candidate_receipt1, CoreIndex(0), GroupIndex(1)), + (candidate_receipt2, CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, ) .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( + // Initial assignment. + let rx = check_and_import_assignment_v2( &mut virtual_overseer, block_hash, - candidate_index, + vec![candidate_index1, candidate_index2], validator, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); + // Test with single assigned core. let rx = check_and_import_assignment( &mut virtual_overseer, block_hash, @@ -1357,6 +1473,18 @@ fn subsystem_accepts_duplicate_assignment() { assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + // Test with multiple assigned cores. This cannot happen in practice, as tranche0 assignments + // are sent first, but we should still ensure correct behavior. + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![candidate_index1, candidate_index2], + validator, + ) + .await; + + assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + virtual_overseer }); } @@ -1407,6 +1535,63 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { }); } +#[test] +fn subsystem_rejects_oversized_bitfields() { + test_harness(HarnessConfig::default(), |test_harness| async move { + let TestHarness { mut virtual_overseer, sync_oracle_handle: _sync_oracle_handle, .. } = + test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let candidate_index = 10; + let validator = ValidatorIndex(0); + + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + ) + .build(&mut virtual_overseer) + .await; + + let rx = check_and_import_assignment( + &mut virtual_overseer, + block_hash, + candidate_index, + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_index as usize + 1 + ))), + ); + + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![1, 2, 10, 50], + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield(51))), + ); + virtual_overseer + }); +} + #[test] fn subsystem_accepts_and_imports_approval_after_assignment() { test_harness(HarnessConfig::default(), |test_harness| async move { @@ -1727,14 +1912,15 @@ fn linear_import_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1797,14 +1983,15 @@ fn forkful_import_at_same_height_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1817,6 +2004,91 @@ fn forkful_import_at_same_height_act_on_leaf() { }); } +#[test] +fn test_signing_a_single_candidate_is_backwards_compatible() { + let session_index = 1; + let block_hash = Hash::repeat_byte(0x01); + let candidate_descriptors = (1..10) + .into_iter() + .map(|val| make_candidate(ParaId::from(val as u32), &block_hash)) + .collect::>(); + + let candidate_hashes = candidate_descriptors + .iter() + .map(|candidate_descriptor| candidate_descriptor.hash()) + .collect_vec(); + + let first_descriptor = candidate_descriptors.first().expect("TODO"); + + let candidate_hash = first_descriptor.hash(); + + let sig_a = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); + + let sig_b = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_a, + ) + .is_ok()); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_b, + ) + .is_ok()); + + let sig_c = sign_approval_multiple_candidates( + Sr25519Keyring::Alice, + vec![candidate_hash], + session_index, + ); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(vec![candidate_hash]) + ) + .check_signature(&Sr25519Keyring::Alice.public().into(), candidate_hash, session_index, &sig_c,) + .is_ok()); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_c, + ) + .is_ok()); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(vec![candidate_hash]) + ) + .check_signature(&Sr25519Keyring::Alice.public().into(), candidate_hash, session_index, &sig_a,) + .is_ok()); + + let sig_all = sign_approval_multiple_candidates( + Sr25519Keyring::Alice, + candidate_hashes.clone(), + session_index, + ); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()) + ) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + *candidate_hashes.first().expect("test"), + session_index, + &sig_all, + ) + .is_ok()); +} + #[test] fn import_checked_approval_updates_entries_and_schedules() { let config = HarnessConfig::default(); @@ -2248,8 +2520,24 @@ fn subsystem_validate_approvals_cache() { let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), tranche: 0, validator_index: ValidatorIndex(0), triggered: false, @@ -2346,52 +2634,200 @@ fn subsystem_validate_approvals_cache() { }); } -/// Ensure that when two assignments are imported, only one triggers the Approval Checking work -async fn handle_double_assignment_import( - virtual_overseer: &mut VirtualOverseer, - candidate_index: CandidateIndex, -) { - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( - _, - c_index, - )) => { - assert_eq!(candidate_index, c_index); - } - ); +#[test] +fn subsystem_doesnt_distribute_duplicate_compact_assignments() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let cert = garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(), + }); - recover_available_data(virtual_overseer).await; - fetch_validation_code(virtual_overseer).await; + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( - _, - c_index - )) => { - assert_eq!(candidate_index, c_index); - } - ); + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert, + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { - tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) - .unwrap(); - } - ); + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) - ); + test_harness(config, |test_harness| async move { + let TestHarness { + mut virtual_overseer, + sync_oracle_handle: _sync_oracle_handle, + clock, + .. + } = test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); - assert_matches!( + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; + + // Add block hash 00. + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot: Slot::from(0), + candidates: Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(1)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, + ) + .build(&mut virtual_overseer) + .await; + + // Activate the wakeup present above, and sleep to allow process_wakeups to execute.. + assert_eq!(Some(2), clock.inner.lock().next_wakeup()); + gum::trace!("clock \n{:?}\n", clock.inner.lock()); + + clock.inner.lock().wakeup_all(100); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(100)).await; + + // Assignment is distributed only once from `approval-voting` + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(c_indices, vec![candidate_index1, candidate_index2].try_into().unwrap()); + } + ); + + // Candidate 1 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Candidate 2 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Check if assignment was triggered for candidate 1. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt1.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + // Check if assignment was triggered for candidate 2. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt2.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + virtual_overseer + }); +} + +/// Ensure that when two assignments are imported, only one triggers the Approval Checking work +async fn handle_double_assignment_import( + virtual_overseer: &mut VirtualOverseer, + candidate_index: CandidateIndex, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(Into::::into(candidate_index), c_indices); + } + ); + + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_index + )) => { + assert_eq!(Into::::into(candidate_index), c_index); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); // Assert that there are no more messages being sent by the subsystem assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); } @@ -2466,8 +2902,9 @@ where let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), tranche: our_assigned_tranche, validator_index: ValidatorIndex(0), triggered: false, @@ -2601,14 +3038,12 @@ async fn step_until_done(clock: &MockClock) { futures_timer::Delay::new(Duration::from_millis(200)).await; let mut clock = clock.inner.lock(); if let Some(tick) = clock.next_wakeup() { - println!("TICK: {:?}", tick); relevant_ticks.push(tick); clock.set_tick(tick); } else { break } } - println!("relevant_ticks: {:?}", relevant_ticks); } #[test] @@ -3123,3 +3558,454 @@ fn waits_until_approving_assignments_are_old_enough() { virtual_overseer }); } + +#[test] +fn test_approval_is_sent_on_max_approval_coalesce_count() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let assignments_cert = + garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = + test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_commitments = CandidateCommitments::default(); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let candidate_hash1 = candidate_receipt1.hash(); + + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let slot = Slot::from(1); + let candidate_index1 = 0; + let candidate_index2 = 1; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(0)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]); + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + }, + ) + .build(&mut virtual_overseer) + .await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash1).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + handle_approval_on_max_coalesce_count( + &mut virtual_overseer, + vec![candidate_index1, candidate_index2], + ) + .await; + + virtual_overseer + }); +} + +async fn handle_approval_on_max_coalesce_count( + virtual_overseer: &mut VirtualOverseer, + candidate_indicies: Vec, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(TryInto::::try_into(candidate_indicies.clone()).unwrap(), c_indices); + } + ); + + for _ in &candidate_indicies { + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + } + + for _ in &candidate_indicies { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + } + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 2, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 2, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(vote)) => { + assert_eq!(TryInto::::try_into(candidate_indicies).unwrap(), vote.candidate_indices); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); +} + +async fn handle_approval_on_max_wait_time( + virtual_overseer: &mut VirtualOverseer, + candidate_indicies: Vec, + clock: Box, +) { + const TICK_NOW_BEGIN: u64 = 1; + const MAX_COALESCE_COUNT: u32 = 3; + + clock.inner.lock().set_tick(TICK_NOW_BEGIN); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(TryInto::::try_into(candidate_indicies.clone()).unwrap(), c_indices); + } + ); + + for _ in &candidate_indicies { + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + } + + for _ in &candidate_indicies { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + } + + // First time we fetch the configuration when we are ready to approve the first candidate + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: MAX_COALESCE_COUNT, + })); + } + ); + + // Second time we fetch the configuration when we are ready to approve the second candidate + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: MAX_COALESCE_COUNT, + })); + } + ); + + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + // Move the clock just before we should send the approval + clock + .inner + .lock() + .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN - 1); + + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + // Move the clock tick, so we can trigger a force sending of the approvals + clock + .inner + .lock() + .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN); + + // Third time we fetch the configuration when timer expires and we are ready to sent the approval + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 3, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(vote)) => { + assert_eq!(TryInto::::try_into(candidate_indicies).unwrap(), vote.candidate_indices); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); +} + +#[test] +fn test_approval_is_sent_on_max_approval_coalesce_wait() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let assignments_cert = + garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = + test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_commitments = CandidateCommitments::default(); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let candidate_hash1 = candidate_receipt1.hash(); + + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let slot = Slot::from(1); + let candidate_index1 = 0; + let candidate_index2 = 1; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(0)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]); + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + }, + ) + .build(&mut virtual_overseer) + .await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash1).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + handle_approval_on_max_wait_time( + &mut virtual_overseer, + vec![candidate_index1, candidate_index2], + clock, + ) + .await; + + virtual_overseer + }); +} diff --git a/node/core/approval-voting/src/time.rs b/node/core/approval-voting/src/time.rs index 34132dc22b23..61091f3c34cd 100644 --- a/node/core/approval-voting/src/time.rs +++ b/node/core/approval-voting/src/time.rs @@ -16,14 +16,23 @@ //! Time utilities for approval voting. -use futures::prelude::*; -use polkadot_node_primitives::approval::DelayTranche; +use futures::{ + future::BoxFuture, + prelude::*, + stream::{FusedStream, FuturesUnordered}, + Stream, StreamExt, +}; + +use polkadot_node_primitives::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ + collections::HashSet, pin::Pin, + task::Poll, time::{Duration, SystemTime}, }; +use polkadot_primitives::{Hash, ValidatorIndex}; const TICK_DURATION_MILLIS: u64 = 500; /// A base unit of time, starting from the Unix epoch, split into half-second intervals. @@ -88,3 +97,157 @@ pub(crate) fn slot_number_to_tick(slot_duration_millis: u64, slot: Slot) -> Tick let ticks_per_slot = slot_duration_millis / TICK_DURATION_MILLIS; u64::from(slot) * ticks_per_slot } + +/// A list of delayed futures that gets triggered when the waiting time has expired and it is +/// time to sign the candidate. +/// We have a timer per relay-chain block. +#[derive(Default)] +pub struct DelayedApprovalTimer { + timers: FuturesUnordered>, + blocks: HashSet, +} + +impl DelayedApprovalTimer { + /// Starts a single timer per block hash + /// + /// Guarantees that if a timer already exits for the give block hash, + /// no additional timer is started. + pub(crate) fn maybe_arm_timer( + &mut self, + wait_untill: Tick, + clock: &dyn Clock, + block_hash: Hash, + validator_index: ValidatorIndex, + ) { + if self.blocks.insert(block_hash) { + let clock_wait = clock.wait(wait_untill); + self.timers.push(Box::pin(async move { + clock_wait.await; + (block_hash, validator_index) + })); + } + } +} + +impl Stream for DelayedApprovalTimer { + type Item = (Hash, ValidatorIndex); + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let poll_result = self.timers.poll_next_unpin(cx); + match poll_result { + Poll::Ready(Some(result)) => { + self.blocks.remove(&result.0); + Poll::Ready(Some(result)) + }, + _ => poll_result, + } + } +} + +impl FusedStream for DelayedApprovalTimer { + fn is_terminated(&self) -> bool { + self.timers.is_terminated() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::{executor::block_on, FutureExt, StreamExt}; + use futures_timer::Delay; + use polkadot_primitives::{Hash, ValidatorIndex}; + + use crate::time::{Clock, SystemClock}; + + use super::DelayedApprovalTimer; + + #[test] + fn test_select_empty_timer() { + block_on(async move { + let mut timer = DelayedApprovalTimer::default(); + + for _ in 1..10 { + let result = futures::select!( + _ = timer.select_next_some() => { + 0 + } + // Only this arm should fire + _ = Delay::new(Duration::from_millis(100)).fuse() => { + 1 + } + ); + + assert_eq!(result, 1); + } + }); + } + + #[test] + fn test_timer_functionality() { + block_on(async move { + let mut timer = DelayedApprovalTimer::default(); + let test_hashes = + vec![Hash::repeat_byte(0x01), Hash::repeat_byte(0x02), Hash::repeat_byte(0x03)]; + for (index, hash) in test_hashes.iter().enumerate() { + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + } + let timeout_hash = Hash::repeat_byte(0x02); + for i in 0..test_hashes.len() * 2 { + let result = futures::select!( + (hash, _) = timer.select_next_some() => { + hash + } + // Timers should fire only once, so for the rest of the iterations we should timeout through here. + _ = Delay::new(Duration::from_secs(2)).fuse() => { + timeout_hash + } + ); + assert_eq!(test_hashes.get(i).cloned().unwrap_or(timeout_hash), result); + } + + // Now check timer can be restarted if already fired + for (index, hash) in test_hashes.iter().enumerate() { + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + } + + for i in 0..test_hashes.len() * 2 { + let result = futures::select!( + (hash, _) = timer.select_next_some() => { + hash + } + // Timers should fire only once, so for the rest of the iterations we should timeout through here. + _ = Delay::new(Duration::from_secs(2)).fuse() => { + timeout_hash + } + ); + assert_eq!(test_hashes.get(i).cloned().unwrap_or(timeout_hash), result); + } + }); + } +} diff --git a/node/core/dispute-coordinator/src/import.rs b/node/core/dispute-coordinator/src/import.rs index 0da3723ebf22..d2520e6f9a64 100644 --- a/node/core/dispute-coordinator/src/import.rs +++ b/node/core/dispute-coordinator/src/import.rs @@ -34,7 +34,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::overseer; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::{ - CandidateReceipt, DisputeStatement, Hash, IndexedVec, SessionIndex, SessionInfo, + CandidateHash, CandidateReceipt, DisputeStatement, Hash, IndexedVec, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; @@ -109,7 +109,7 @@ pub enum OwnVoteState { } impl OwnVoteState { - fn new(votes: &CandidateVotes, env: &CandidateEnvironment<'_>) -> Self { + fn new(votes: &CandidateVotes, env: &CandidateEnvironment) -> Self { let controlled_indices = env.controlled_indices(); if controlled_indices.is_empty() { return Self::CannotVote @@ -118,7 +118,9 @@ impl OwnVoteState { let our_valid_votes = controlled_indices .iter() .filter_map(|i| votes.valid.raw().get_key_value(i)) - .map(|(index, (kind, sig))| (*index, (DisputeStatement::Valid(*kind), sig.clone()))); + .map(|(index, (kind, sig))| { + (*index, (DisputeStatement::Valid(kind.clone()), sig.clone())) + }); let our_invalid_votes = controlled_indices .iter() .filter_map(|i| votes.invalid.get_key_value(i)) @@ -297,7 +299,7 @@ impl CandidateVoteState { DisputeStatement::Valid(valid_kind) => { let fresh = votes.valid.insert_vote( val_index, - *valid_kind, + valid_kind.clone(), statement.into_validator_signature(), ); if fresh { @@ -503,7 +505,7 @@ impl ImportResult { pub fn import_approval_votes( self, env: &CandidateEnvironment, - approval_votes: HashMap, + approval_votes: HashMap, ValidatorSignature)>, now: Timestamp, ) -> Self { let Self { @@ -517,19 +519,32 @@ impl ImportResult { let (mut votes, _) = new_state.into_old_state(); - for (index, sig) in approval_votes.into_iter() { + for (index, (candidate_hashes, sig)) in approval_votes.into_iter() { debug_assert!( { let pub_key = &env.session_info().validators.get(index).expect("indices are validated by approval-voting subsystem; qed"); - let candidate_hash = votes.candidate_receipt.hash(); let session_index = env.session_index(); - DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) - .check_signature(pub_key, candidate_hash, session_index, &sig) + candidate_hashes.contains(&votes.candidate_receipt.hash()) && DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone())) + .check_signature(pub_key, *candidate_hashes.first().expect("Valid votes have at least one candidate; qed"), session_index, &sig) .is_ok() }, "Signature check for imported approval votes failed! This is a serious bug. Session: {:?}, candidate hash: {:?}, validator index: {:?}", env.session_index(), votes.candidate_receipt.hash(), index ); - if votes.valid.insert_vote(index, ValidDisputeStatementKind::ApprovalChecking, sig) { + if votes.valid.insert_vote( + index, + // There is a hidden dependency here between approval-voting and this subsystem. + // We should be able to start emitting ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates only after: + // 1. Runtime have been upgraded to know about the new format. + // 2. All nodes have been upgraded to know about the new format. + // Once those two requirements have been met we should be able to increase max_approval_coalesce_count to values + // greater than 1. + if candidate_hashes.len() > 1 { + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes) + } else { + ValidDisputeStatementKind::ApprovalChecking + }, + sig, + ) { imported_valid_votes += 1; imported_approval_votes += 1; } diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index c1d02ef976cb..7afc120d4161 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -633,7 +633,7 @@ impl Initialized { }; debug_assert!( SignedDisputeStatement::new_checked( - DisputeStatement::Valid(valid_statement_kind), + DisputeStatement::Valid(valid_statement_kind.clone()), candidate_hash, session, validator_public.clone(), @@ -647,7 +647,7 @@ impl Initialized { ); let signed_dispute_statement = SignedDisputeStatement::new_unchecked_from_trusted_source( - DisputeStatement::Valid(valid_statement_kind), + DisputeStatement::Valid(valid_statement_kind.clone()), candidate_hash, session, validator_public, diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index a2c500e08e28..b18cf8aa4aa9 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -575,7 +575,7 @@ pub fn make_dispute_message( .next() .ok_or(DisputeMessageCreationError::NoOppositeVote)?; let other_vote = SignedDisputeStatement::new_checked( - DisputeStatement::Valid(*statement_kind), + DisputeStatement::Valid(statement_kind.clone()), *our_vote.candidate_hash(), our_vote.session_index(), validators diff --git a/node/core/dispute-coordinator/src/tests.rs b/node/core/dispute-coordinator/src/tests.rs index 75eae8200dc6..552613c566b9 100644 --- a/node/core/dispute-coordinator/src/tests.rs +++ b/node/core/dispute-coordinator/src/tests.rs @@ -651,7 +651,7 @@ fn make_candidate_included_event(candidate_receipt: CandidateReceipt) -> Candida pub async fn handle_approval_vote_request( ctx_handle: &mut VirtualOverseer, expected_hash: &CandidateHash, - votes_to_send: HashMap, + votes_to_send: HashMap, ValidatorSignature)>, ) { assert_matches!( ctx_handle.recv().await, @@ -858,9 +858,12 @@ fn approval_vote_import_works() { .await; gum::trace!("After sending `ImportStatements`"); - let approval_votes = [(ValidatorIndex(4), approval_vote.into_validator_signature())] - .into_iter() - .collect(); + let approval_votes = [( + ValidatorIndex(4), + (vec![candidate_receipt1.hash()], approval_vote.into_validator_signature()), + )] + .into_iter() + .collect(); handle_approval_vote_request(&mut virtual_overseer, &candidate_hash1, approval_votes) .await; diff --git a/node/core/provisioner/src/disputes/prioritized_selection/mod.rs b/node/core/provisioner/src/disputes/prioritized_selection/mod.rs index 096b73d271a8..cb55ce39bc89 100644 --- a/node/core/provisioner/src/disputes/prioritized_selection/mod.rs +++ b/node/core/provisioner/src/disputes/prioritized_selection/mod.rs @@ -221,7 +221,7 @@ where votes.valid.retain(|validator_idx, (statement_kind, _)| { is_vote_worth_to_keep( validator_idx, - DisputeStatement::Valid(*statement_kind), + DisputeStatement::Valid(statement_kind.clone()), &onchain_state, ) }); diff --git a/node/core/runtime-api/src/cache.rs b/node/core/runtime-api/src/cache.rs index 4c23ce2fa3c7..d74ae8062804 100644 --- a/node/core/runtime-api/src/cache.rs +++ b/node/core/runtime-api/src/cache.rs @@ -20,12 +20,12 @@ use lru::LruCache; use sp_consensus_babe::Epoch; use polkadot_primitives::{ - vstaging, AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; /// For consistency we have the same capacity for all caches. We use 128 as we'll only need that @@ -64,6 +64,8 @@ pub(crate) struct RequestResultCache { LruCache<(Hash, ParaId, OccupiedCoreAssumption), Option>, version: LruCache, disputes: LruCache)>>, + approval_voting_params: LruCache, + unapplied_slashes: LruCache>, key_ownership_proof: @@ -97,6 +99,7 @@ impl Default for RequestResultCache { disputes: LruCache::new(DEFAULT_CACHE_CAP), unapplied_slashes: LruCache::new(DEFAULT_CACHE_CAP), key_ownership_proof: LruCache::new(DEFAULT_CACHE_CAP), + approval_voting_params: LruCache::new(DEFAULT_CACHE_CAP), } } } @@ -393,6 +396,21 @@ impl RequestResultCache { self.disputes.put(relay_parent, value); } + pub(crate) fn approval_voting_params( + &mut self, + relay_parent: &Hash, + ) -> Option<&ApprovalVotingParams> { + self.approval_voting_params.get(relay_parent) + } + + pub(crate) fn cache_approval_voting_params( + &mut self, + relay_parent: Hash, + value: ApprovalVotingParams, + ) { + self.approval_voting_params.put(relay_parent, value); + } + pub(crate) fn unapplied_slashes( &mut self, relay_parent: &Hash, @@ -476,4 +494,5 @@ pub(crate) enum RequestResult { vstaging::slashing::OpaqueKeyOwnershipProof, Option<()>, ), + ApprovalVotingParams(Hash, ApprovalVotingParams), } diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 0ee5ca24ceee..71c7d614a349 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -162,6 +162,8 @@ where KeyOwnershipProof(relay_parent, validator_id, key_ownership_proof) => self .requests_cache .cache_key_ownership_proof((relay_parent, validator_id), key_ownership_proof), + RequestResult::ApprovalVotingParams(relay_parent, params) => + self.requests_cache.cache_approval_voting_params(relay_parent, params), SubmitReportDisputeLost(_, _, _, _) => {}, } } @@ -288,6 +290,8 @@ where Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) }, ), + Request::ApprovalVotingParams(sender) => query!(approval_voting_params(), sender) + .map(|sender| Request::ApprovalVotingParams(sender)), } } @@ -532,6 +536,9 @@ where ver = Request::KEY_OWNERSHIP_PROOF_RUNTIME_REQUIREMENT, sender ), + Request::ApprovalVotingParams(sender) => { + query!(ApprovalVotingParams, approval_voting_params(), ver = 6, sender) + }, Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) => query!( SubmitReportDisputeLost, submit_report_dispute_lost(dispute_proof, key_ownership_proof), diff --git a/node/core/runtime-api/src/tests.rs b/node/core/runtime-api/src/tests.rs index 53b3fd56bf3e..d0b41df66124 100644 --- a/node/core/runtime-api/src/tests.rs +++ b/node/core/runtime-api/src/tests.rs @@ -20,12 +20,12 @@ use polkadot_node_primitives::{BabeAllowedSlots, BabeEpoch, BabeEpochConfigurati use polkadot_node_subsystem::SpawnGlue; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_primitives::{ - vstaging, AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, Slot, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + Slot, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_api::ApiError; use sp_core::testing::TaskExecutor; @@ -242,6 +242,11 @@ impl RuntimeApiSubsystemClient for MockSubsystemClient { todo!("Not required for tests") } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, _: Hash) -> Result { + todo!("Not required for tests") + } + async fn current_epoch(&self, _: Hash) -> Result { Ok(self.babe_epoch.as_ref().unwrap().clone()) } diff --git a/node/network/approval-distribution/Cargo.toml b/node/network/approval-distribution/Cargo.toml index bd683b320285..113b9429c911 100644 --- a/node/network/approval-distribution/Cargo.toml +++ b/node/network/approval-distribution/Cargo.toml @@ -14,10 +14,12 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-primitives = { path = "../../../primitives" } polkadot-node-jaeger = { path = "../../jaeger" } rand = "0.8" +itertools = "0.10.5" futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [dev-dependencies] sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index b94ebb282219..617e578c60a4 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -20,16 +20,26 @@ #![warn(missing_docs)] +use self::metrics::Metrics; use futures::{channel::oneshot, select, FutureExt as _}; +use itertools::Itertools; +use net_protocol::peer_set::{ProtocolVersion, ValidationVersion}; use polkadot_node_jaeger as jaeger; use polkadot_node_network_protocol::{ - self as net_protocol, + self as net_protocol, filter_by_peer_version, grid_topology::{RandomRouting, RequiredRouting, SessionGridTopologies, SessionGridTopology}, peer_set::MAX_NOTIFICATION_SIZE, - v1 as protocol_v1, PeerId, UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::approval::{ - AssignmentCert, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{ + AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + }, + v2::{ + AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, + IndirectSignedApprovalVoteV2, + }, }; use polkadot_node_subsystem::{ messages::{ @@ -48,8 +58,6 @@ use std::{ time::Duration, }; -use self::metrics::Metrics; - mod metrics; #[cfg(test)] @@ -63,11 +71,15 @@ const COST_DUPLICATE_MESSAGE: Rep = Rep::CostMinorRepeated("Peer sent identical const COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE: Rep = Rep::CostMinor("The vote was valid but too far in the future"); const COST_INVALID_MESSAGE: Rep = Rep::CostMajor("The vote was bad"); +const COST_OVERSIZED_BITFIELD: Rep = Rep::CostMajor("Oversized certificate or candidate bitfield"); const BENEFIT_VALID_MESSAGE: Rep = Rep::BenefitMinor("Peer sent a valid message"); const BENEFIT_VALID_MESSAGE_FIRST: Rep = Rep::BenefitMinorFirst("Valid message with new information"); +// Maximum valid size for the `CandidateBitfield` in the assignment messages. +const MAX_BITFIELD_SIZE: usize = 500; + /// The Approval Distribution subsystem. pub struct ApprovalDistribution { metrics: Metrics, @@ -96,7 +108,147 @@ impl RecentlyOutdated { } } -// In case the original gtid topology mechanisms don't work on their own, we need to trade bandwidth +// Contains topology routing information for assignments and approvals. +struct ApprovalRouting { + required_routing: RequiredRouting, + local: bool, + random_routing: RandomRouting, +} + +// This struct is responsible for tracking the full state of an assignment and grid routing +// information. +struct ApprovalEntry { + // The assignment certificate. + assignment: IndirectAssignmentCertV2, + // The candidates claimed by the certificate. A mapping between bit index and candidate index. + candidates: CandidateBitfield, + // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. + approvals: HashMap, + // The validator index of the assignment signer. + validator_index: ValidatorIndex, + // Information required for gossiping to other peers using the grid topology. + routing_info: ApprovalRouting, +} + +#[derive(Debug)] +enum ApprovalEntryError { + InvalidValidatorIndex, + CandidateIndexOutOfBounds, + InvalidCandidateIndex, + DuplicateApproval, +} + +impl ApprovalEntry { + pub fn new( + assignment: IndirectAssignmentCertV2, + candidates: CandidateBitfield, + routing_info: ApprovalRouting, + ) -> ApprovalEntry { + Self { + validator_index: assignment.validator, + assignment, + approvals: HashMap::with_capacity(candidates.len()), + candidates, + routing_info, + } + } + + // Create a `MessageSubject` to reference the assignment. + pub fn create_assignment_knowledge(&self, block_hash: Hash) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, self.candidates.clone(), self.validator_index), + MessageKind::Assignment, + ) + } + + // Create a `MessageSubject` to reference the approval. + pub fn create_approval_knowledge( + &self, + block_hash: Hash, + candidate_index: CandidateIndex, + ) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, candidate_index.into(), self.validator_index), + MessageKind::Approval, + ) + } + + // Updates routing information and returns the previous information if any. + pub fn routing_info_mut(&mut self) -> &mut ApprovalRouting { + &mut self.routing_info + } + + // Get the routing information. + pub fn routing_info(&self) -> &ApprovalRouting { + &self.routing_info + } + + // Update routing information. + pub fn update_required_routing(&mut self, required_routing: RequiredRouting) { + self.routing_info.required_routing = required_routing; + } + + // Records a new approval. Returns false if the claimed candidate is not found or we already + // have received the approval. + pub fn note_approval( + &mut self, + approval: IndirectSignedApprovalVoteV2, + candidate_index: CandidateIndex, + ) -> Result<(), ApprovalEntryError> { + // First do some sanity checks: + // - check validator index matches + // - check claimed candidate + // - check for duplicate approval + if self.validator_index != approval.validator { + return Err(ApprovalEntryError::InvalidValidatorIndex) + } + + if self.candidates.len() <= candidate_index as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + if !self.candidates.bit_at((candidate_index).as_bit_index()) { + return Err(ApprovalEntryError::InvalidCandidateIndex) + } + + if self.approvals.contains_key(&candidate_index) { + return Err(ApprovalEntryError::DuplicateApproval) + } + + self.approvals.insert(candidate_index, approval.clone()); + Ok(()) + } + + // Get the assignment certiticate and claimed candidates. + pub fn get_assignment(&self) -> (IndirectAssignmentCertV2, CandidateBitfield) { + (self.assignment.clone(), self.candidates.clone()) + } + + // Get all approvals for all candidates claimed by the assignment. + pub fn get_approvals(&self) -> Vec { + self.approvals.values().cloned().collect::>() + } + + // Get the approval for a specific candidate index. + pub fn get_approval( + &self, + candidate_index: CandidateIndex, + ) -> Option { + self.approvals.get(&candidate_index).cloned() + } + + // Get validator index. + pub fn get_validator_index(&self) -> ValidatorIndex { + self.validator_index + } +} + +// We keep track of each peer view and protocol version using this struct. +struct PeerEntry { + pub view: View, + pub version: ProtocolVersion, +} + +// In case the original grid topology mechanisms don't work on their own, we need to trade bandwidth // for protocol liveliness by introducing aggression. // // Aggression has 3 levels: @@ -116,7 +268,6 @@ impl RecentlyOutdated { // not to all blocks older than the threshold. Most likely, a few assignments struggle to // be propagated in a single block and this holds up all of its descendants blocks. // Accordingly, we only step on the gas for the block which is most obviously holding up finality. - /// Aggression configuration representation #[derive(Clone)] struct AggressionConfig { @@ -179,7 +330,7 @@ struct State { pending_known: HashMap>, /// Peer data is partially stored here, and partially inline within the [`BlockEntry`]s - peer_views: HashMap, + peer_views: HashMap, /// Keeps a topology for various different sessions. topologies: SessionGridTopologies, @@ -187,12 +338,12 @@ struct State { /// Tracks recently finalized blocks. recent_outdated_blocks: RecentlyOutdated, - /// Config for aggression. - aggression_config: AggressionConfig, - - /// `HashMap` from active leaves to spans + /// HashMap from active leaves to spans spans: HashMap, + /// Aggression configuration. + aggression_config: AggressionConfig, + /// Current approval checking finality lag. approval_checking_lag: BlockNumber, @@ -206,8 +357,11 @@ enum MessageKind { Approval, } +// Utility structure to identify assignments and approvals for specific candidates. +// Assignments can span multiple candidates, while approvals refer to only one candidate. +// #[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct MessageSubject(Hash, CandidateIndex, ValidatorIndex); +struct MessageSubject(Hash, pub CandidateBitfield, ValidatorIndex); #[derive(Debug, Clone, Default)] struct Knowledge { @@ -228,9 +382,11 @@ impl Knowledge { } fn insert(&mut self, message: MessageSubject, kind: MessageKind) -> bool { - match self.known_messages.entry(message) { + let mut success = match self.known_messages.entry(message.clone()) { hash_map::Entry::Vacant(vacant) => { vacant.insert(kind); + // If there are multiple candidates assigned in the message, create + // separate entries for each one. true }, hash_map::Entry::Occupied(mut occupied) => match (*occupied.get(), kind) { @@ -242,7 +398,25 @@ impl Knowledge { true }, }, + }; + + // In case of succesful insertion of multiple candidate assignments create additional + // entries for each assigned candidate. This fakes knowledge of individual assignments, but + // we need to share the same `MessageSubject` with the followup approval candidate index. + if kind == MessageKind::Assignment && success && message.1.count_ones() > 1 { + for candidate_index in message.1.iter_ones() { + success = success && + self.insert( + MessageSubject( + message.0, + vec![candidate_index as u32].try_into().expect("Non-empty vec; qed"), + message.2, + ), + kind, + ); + } } + success } } @@ -276,47 +450,113 @@ struct BlockEntry { candidates: Vec, /// The session index of this block. session: SessionIndex, + /// Approval entries for whole block. These also contain all approvals in the case of multiple + /// candidates being claimed by assignments. + approval_entries: HashMap<(ValidatorIndex, CandidateBitfield), ApprovalEntry>, } -#[derive(Debug)] -enum ApprovalState { - Assigned(AssignmentCert), - Approved(AssignmentCert, ValidatorSignature), -} +impl BlockEntry { + // Returns the peer which currently know this block. + pub fn known_by(&self) -> Vec { + self.known_by.keys().cloned().collect::>() + } -impl ApprovalState { - fn assignment_cert(&self) -> &AssignmentCert { - match *self { - ApprovalState::Assigned(ref cert) => cert, - ApprovalState::Approved(ref cert, _) => cert, + pub fn insert_approval_entry(&mut self, entry: ApprovalEntry) -> &mut ApprovalEntry { + // First map one entry per candidate to the same key we will use in `approval_entries`. + // Key is (Validator_index, CandidateBitfield) that links the `ApprovalEntry` to the (K,V) + // entry in `candidate_entry.messages`. + for claimed_candidate_index in entry.candidates.iter_ones() { + match self.candidates.get_mut(claimed_candidate_index) { + Some(candidate_entry) => { + candidate_entry + .messages + .entry(entry.get_validator_index()) + .or_insert(entry.candidates.clone()); + }, + None => { + // This should never happen, but if it happens, it means the subsystem is + // broken. + gum::warn!( + target: LOG_TARGET, + hash = ?entry.assignment.block_hash, + ?claimed_candidate_index, + "Missing candidate entry on `import_and_circulate_assignment`", + ); + }, + }; } + + self.approval_entries + .entry((entry.validator_index, entry.candidates.clone())) + .or_insert(entry) } - fn approval_signature(&self) -> Option { - match *self { - ApprovalState::Assigned(_) => None, - ApprovalState::Approved(_, ref sig) => Some(sig.clone()), - } + // Returns `true` if we have an approval for `candidate_index` from validator + // `validator_index`. + pub fn contains_approval_entry( + &self, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + ) -> bool { + self.candidates + .get(candidate_index as usize) + .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) + .map_or(false, |candidate_indices| { + self.approval_entries + .contains_key(&(validator_index, candidate_indices.clone())) + }) } -} -// routing state bundled with messages for the candidate. Corresponding assignments -// and approvals are stored together and should be routed in the same way, with -// assignments preceding approvals in all cases. -#[derive(Debug)] -struct MessageState { - required_routing: RequiredRouting, - local: bool, - random_routing: RandomRouting, - approval_state: ApprovalState, + // Returns a mutable reference of `ApprovalEntry` for `candidate_index` from validator + // `validator_index`. + pub fn get_approval_entry( + &mut self, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + ) -> Option<&mut ApprovalEntry> { + self.candidates + .get(candidate_index as usize) + .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) + .map_or(None, |candidate_indices| { + self.approval_entries.get_mut(&(validator_index, candidate_indices.clone())) + }) + } + + // Get all approval entries for a given candidate. + pub fn get_approval_entries(&self, candidate_index: CandidateIndex) -> Vec<&ApprovalEntry> { + // Get the keys for fetching `ApprovalEntry` from `self.approval_entries`, + let approval_entry_keys = self + .candidates + .get(candidate_index as usize) + .map(|candidate_entry| &candidate_entry.messages); + + if let Some(approval_entry_keys) = approval_entry_keys { + // Ensure no duplicates. + let approval_entry_keys = approval_entry_keys.iter().unique().collect::>(); + + let mut entries = Vec::new(); + for (validator_index, candidate_indices) in approval_entry_keys { + if let Some(entry) = + self.approval_entries.get(&(*validator_index, candidate_indices.clone())) + { + entries.push(entry); + } + } + entries + } else { + vec![] + } + } } -/// Information about candidates in the context of a particular block they are included in. -/// In other words, multiple `CandidateEntry`s may exist for the same candidate, -/// if it is included by multiple blocks - this is likely the case when there are forks. +// Information about candidates in the context of a particular block they are included in. +// In other words, multiple `CandidateEntry`s may exist for the same candidate, +// if it is included by multiple blocks - this is likely the case when there are forks. #[derive(Debug, Default)] struct CandidateEntry { - messages: HashMap, + // The value represents part of the lookup key in `approval_entries` to fetch the assignment + // and existing votes. + messages: HashMap, } #[derive(Debug, Clone, PartialEq)] @@ -335,8 +575,8 @@ impl MessageSource { } enum PendingMessage { - Assignment(IndirectAssignmentCert, CandidateIndex), - Approval(IndirectSignedApprovalVote), + Assignment(IndirectAssignmentCertV2, CandidateBitfield), + Approval(IndirectSignedApprovalVoteV2), } #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] @@ -349,10 +589,12 @@ impl State { rng: &mut (impl CryptoRng + Rng), ) { match event { - NetworkBridgeEvent::PeerConnected(peer_id, role, _, _) => { + NetworkBridgeEvent::PeerConnected(peer_id, role, version, _) => { // insert a blank view if none already present gum::trace!(target: LOG_TARGET, ?peer_id, ?role, "Peer connected"); - self.peer_views.entry(peer_id).or_default(); + self.peer_views + .entry(peer_id) + .or_insert(PeerEntry { view: Default::default(), version }); }, NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); @@ -393,12 +635,12 @@ impl State { live }); }, + NetworkBridgeEvent::PeerMessage(peer_id, message) => { + self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; + }, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { // The approval-distribution subsystem doesn't deal with `AuthorityDiscoveryId`s. }, - NetworkBridgeEvent::PeerMessage(peer_id, Versioned::V1(msg)) => { - self.process_incoming_peer_message(ctx, metrics, peer_id, msg, rng).await; - }, } } @@ -433,6 +675,7 @@ impl State { knowledge: Knowledge::default(), candidates, session: meta.session, + approval_entries: HashMap::new(), }); self.topologies.inc_session_refs(meta.session); @@ -455,7 +698,7 @@ impl State { { let sender = ctx.sender(); - for (peer_id, view) in self.peer_views.iter() { + for (peer_id, PeerEntry { view, version }) in self.peer_views.iter() { let intersection = view.iter().filter(|h| new_hashes.contains(h)); let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( @@ -465,6 +708,7 @@ impl State { &self.topologies, self.peer_views.len(), *peer_id, + *version, view_intersection, rng, ) @@ -502,13 +746,13 @@ impl State { for (peer_id, message) in to_import { match message { - PendingMessage::Assignment(assignment, claimed_index) => { + PendingMessage::Assignment(assignment, claimed_indices) => { self.import_and_circulate_assignment( ctx, metrics, MessageSource::Peer(peer_id), assignment, - claimed_index, + claimed_indices, rng, ) .await; @@ -551,101 +795,170 @@ impl State { &self.topologies, |block_entry| block_entry.session == session, |required_routing, local, validator_index| { - if *required_routing == RequiredRouting::PendingTopology { - *required_routing = topology + if required_routing == &RequiredRouting::PendingTopology { + topology .local_grid_neighbors() - .required_routing_by_index(*validator_index, local); + .required_routing_by_index(*validator_index, local) + } else { + *required_routing } }, + &self.peer_views, ) .await; } + async fn process_incoming_assignments( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + rng: &mut R, + ) where + R: CryptoRng + Rng, + { + for (assignment, claimed_indices) in assignments { + if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { + let block_hash = &assignment.block_hash; + let validator_index = assignment.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?claimed_indices, + ?validator_index, + "Pending assignment", + ); + + pending.push((peer_id, PendingMessage::Assignment(assignment, claimed_indices))); + + continue + } + + self.import_and_circulate_assignment( + ctx, + metrics, + MessageSource::Peer(peer_id), + assignment, + claimed_indices, + rng, + ) + .await; + } + } + + async fn process_incoming_approvals( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + approvals: Vec, + ) { + gum::trace!( + target: LOG_TARGET, + peer_id = %peer_id, + num = approvals.len(), + "Processing approvals from a peer", + ); + for approval_vote in approvals.into_iter() { + if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { + let block_hash = approval_vote.block_hash; + let validator_index = approval_vote.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?validator_index, + "Pending assignment candidates {:?}", + approval_vote.candidate_indices, + ); + + pending.push((peer_id, PendingMessage::Approval(approval_vote))); + + continue + } + + self.import_and_circulate_approval( + ctx, + metrics, + MessageSource::Peer(peer_id), + approval_vote, + ) + .await; + } + } + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, - msg: protocol_v1::ApprovalDistributionMessage, + msg: Versioned< + protocol_v1::ApprovalDistributionMessage, + protocol_vstaging::ApprovalDistributionMessage, + >, rng: &mut R, ) where R: CryptoRng + Rng, { match msg { - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) => { + Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( + assignments, + )) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, num = assignments.len(), "Processing assignments from a peer", ); - for (assignment, claimed_index) in assignments.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { - let message_subject = MessageSubject( - assignment.block_hash, - claimed_index, - assignment.validator, - ); - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?message_subject, - "Pending assignment", - ); + let sanitized_assignments = + self.sanitize_v2_assignments(peer_id, ctx.sender(), assignments).await; - pending - .push((peer_id, PendingMessage::Assignment(assignment, claimed_index))); - - continue - } - - self.import_and_circulate_assignment( - ctx, - metrics, - MessageSource::Peer(peer_id), - assignment, - claimed_index, - rng, - ) - .await; - } + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; }, - protocol_v1::ApprovalDistributionMessage::Approvals(approvals) => { + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, - num = approvals.len(), - "Processing approvals from a peer", + num = assignments.len(), + "Processing assignments from a peer", ); - for approval_vote in approvals.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { - let message_subject = MessageSubject( - approval_vote.block_hash, - approval_vote.candidate_index, - approval_vote.validator, - ); - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?message_subject, - "Pending approval", - ); - - pending.push((peer_id, PendingMessage::Approval(approval_vote))); - continue - } + let sanitized_assignments = + self.sanitize_v1_assignments(peer_id, ctx.sender(), assignments).await; - self.import_and_circulate_approval( - ctx, - metrics, - MessageSource::Peer(peer_id), - approval_vote, - ) - .await; - } + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; + }, + Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( + approvals, + )) => { + self.process_incoming_approvals(ctx, metrics, peer_id, approvals).await; + }, + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) => { + self.process_incoming_approvals( + ctx, + metrics, + peer_id, + approvals.into_iter().map(|approval| approval.into()).collect::>(), + ) + .await; }, } } @@ -664,8 +977,22 @@ impl State { { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); let finalized_number = view.finalized_number; - let old_view = - self.peer_views.get_mut(&peer_id).map(|d| std::mem::replace(d, view.clone())); + + let (old_view, protocol_version) = + if let Some(peer_entry) = self.peer_views.get_mut(&peer_id) { + (Some(std::mem::replace(&mut peer_entry.view, view.clone())), peer_entry.version) + } else { + // This shouldn't happen, but if it does we assume protocol version 1. + gum::warn!( + target: LOG_TARGET, + ?peer_id, + ?view, + "Peer view change for missing `peer_entry`" + ); + + (None, ValidationVersion::V1.into()) + }; + let old_finalized_number = old_view.map(|v| v.finalized_number).unwrap_or(0); // we want to prune every block known_by peer up to (including) view.finalized_number @@ -693,6 +1020,7 @@ impl State { &self.topologies, self.peer_views.len(), peer_id, + protocol_version, view, rng, ) @@ -733,8 +1061,8 @@ impl State { ctx: &mut Context, metrics: &Metrics, source: MessageSource, - assignment: IndirectAssignmentCert, - claimed_candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + claimed_candidate_indices: CandidateBitfield, rng: &mut R, ) where R: CryptoRng + Rng, @@ -776,15 +1104,20 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + gum::debug!(target: LOG_TARGET, "Received assignment for invalid block"); + metrics.on_assignment_recent_outdated(); } } + metrics.on_assignment_invalid_block(); return }, }; - // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, claimed_candidate_index, validator_index); - let message_kind = MessageKind::Assignment; + // Compute metadata on the assignment. + let (message_subject, message_kind) = ( + MessageSubject(block_hash, claimed_candidate_indices.clone(), validator_index), + MessageKind::Assignment, + ); if let Some(peer_id) = source.peer_id() { // check if our knowledge of the peer already contains this assignment @@ -800,6 +1133,7 @@ impl State { ?message_subject, "Duplicate assignment", ); + modify_reputation( &mut self.reputation, ctx.sender(), @@ -807,6 +1141,16 @@ impl State { COST_DUPLICATE_MESSAGE, ) .await; + metrics.on_assignment_duplicate(); + } else { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + hash = ?block_hash, + ?validator_index, + ?message_subject, + "We sent the message to the peer while peer was sending it to us. Known race condition.", + ); } return } @@ -825,6 +1169,7 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + metrics.on_assignment_out_of_view(); }, } @@ -841,6 +1186,7 @@ impl State { gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known assignment"); peer_knowledge.received.insert(message_subject, message_kind); } + metrics.on_assignment_good_known(); return } @@ -848,7 +1194,7 @@ impl State { ctx.send_message(ApprovalVotingMessage::CheckAndImportAssignment( assignment.clone(), - claimed_candidate_index, + claimed_candidate_indices.clone(), tx, )) .await; @@ -879,7 +1225,7 @@ impl State { BENEFIT_VALID_MESSAGE_FIRST, ) .await; - entry.knowledge.known_messages.insert(message_subject.clone(), message_kind); + entry.knowledge.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); } @@ -897,6 +1243,8 @@ impl State { ?peer_id, "Got an `AcceptedDuplicate` assignment", ); + metrics.on_assignment_duplicatevoting(); + return }, AssignmentCheckResult::TooFarInFuture => { @@ -913,6 +1261,8 @@ impl State { COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE, ) .await; + metrics.on_assignment_far(); + return }, AssignmentCheckResult::Bad(error) => { @@ -930,6 +1280,7 @@ impl State { COST_INVALID_MESSAGE, ) .await; + metrics.on_assignment_bad(); return }, } @@ -953,7 +1304,7 @@ impl State { // Invariant: to our knowledge, none of the peers except for the `source` know about the // assignment. - metrics.on_assignment_imported(); + metrics.on_assignment_imported(&assignment.cert.kind); let topology = self.topologies.get_topology(entry.session); let local = source == MessageSource::Local; @@ -962,28 +1313,14 @@ impl State { t.local_grid_neighbors().required_routing_by_index(validator_index, local) }); - let message_state = match entry.candidates.get_mut(claimed_candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Assigned - // unless the approval state is set already - candidate_entry.messages.entry(validator_index).or_insert_with(|| MessageState { - required_routing, - local, - random_routing: Default::default(), - approval_state: ApprovalState::Assigned(assignment.cert.clone()), - }) - }, - None => { - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?claimed_candidate_index, - "Expected a candidate entry on import_and_circulate_assignment", - ); + // All the peers that know the relay chain block. + let peers_to_filter = entry.known_by(); - return - }, - }; + let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( + assignment.clone(), + claimed_candidate_indices.clone(), + ApprovalRouting { required_routing, local, random_routing: Default::default() }, + )); // Dispatch the message to all peers in the routing set which // know the block. @@ -991,35 +1328,41 @@ impl State { // If the topology isn't known yet (race with networking subsystems) // then messages will be sent when we get it. - let assignments = vec![(assignment, claimed_candidate_index)]; + let assignments = vec![(assignment, claimed_candidate_indices.clone())]; let n_peers_total = self.peer_views.len(); let source_peer = source.peer_id(); - let mut peer_filter = move |peer| { - if Some(peer) == source_peer.as_ref() { - return false + // Peers that we will send the assignment to. + let mut peers = Vec::new(); + + // Filter destination peers + for peer in peers_to_filter.into_iter() { + if Some(peer) == source_peer { + continue } if let Some(true) = topology .as_ref() - .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, peer)) + .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, &peer)) { - return true + peers.push(peer); + continue } + if !topology.map(|topology| topology.is_validator(&peer)).unwrap_or(false) { + continue + } // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. - let route_random = message_state.random_routing.sample(n_peers_total, rng); + let route_random = + approval_entry.routing_info().random_routing.sample(n_peers_total, rng); if route_random { - message_state.random_routing.inc_sent(); + approval_entry.routing_info_mut().random_routing.inc_sent(); + peers.push(peer); } - - route_random - }; - - let peers = entry.known_by.keys().filter(|p| peer_filter(p)).cloned().collect::>(); + } // Add the metadata of the assignment to the knowledge of each peer. for peer in peers.iter() { @@ -1033,86 +1376,43 @@ impl State { gum::trace!( target: LOG_TARGET, ?block_hash, - ?claimed_candidate_index, + ?claimed_candidate_indices, local = source.peer_id().is_none(), num_peers = peers.len(), "Sending an assignment to peers", ); - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments), - )), - )) - .await; + let peers = peers + .iter() + .filter_map(|peer_id| { + self.peer_views.get(peer_id).map(|peer_entry| (*peer_id, peer_entry.version)) + }) + .collect::>(); + + send_assignments_batched(ctx.sender(), assignments, &peers).await; } } - async fn import_and_circulate_approval( - &mut self, + async fn check_approval_can_be_processed( ctx: &mut Context, + message_subjects: &Vec, + message_kind: MessageKind, + entry: &mut BlockEntry, + reputation: &mut ReputationAggregator, + peer_id: PeerId, metrics: &Metrics, - source: MessageSource, - vote: IndirectSignedApprovalVote, - ) { - let _span = self - .spans - .get(&vote.block_hash) - .map(|span| { - span.child(if source.peer_id().is_some() { - "peer-import-and-distribute-approval" - } else { - "local-import-and-distribute-approval" - }) - }) - .unwrap_or_else(|| jaeger::Span::new(&vote.block_hash, "distribute-approval")) - .with_string_tag("block-hash", format!("{:?}", vote.block_hash)) - .with_optional_peer_id(source.peer_id().as_ref()) - .with_stage(jaeger::Stage::ApprovalDistribution); - - let block_hash = vote.block_hash; - let validator_index = vote.validator; - let candidate_index = vote.candidate_index; - - let entry = match self.blocks.get_mut(&block_hash) { - Some(entry) if entry.candidates.get(candidate_index as usize).is_some() => entry, - _ => { - if let Some(peer_id) = source.peer_id() { - if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; - } - } - return - }, - }; - - // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, candidate_index, validator_index); - let message_kind = MessageKind::Approval; - - if let Some(peer_id) = source.peer_id() { + ) -> bool { + for message_subject in message_subjects { if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { - gum::debug!( + gum::trace!( target: LOG_TARGET, ?peer_id, ?message_subject, "Unknown approval assignment", ); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; - return + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + metrics.on_approval_unknown_assignment(); + return false } // check if our knowledge of the peer already contains this approval @@ -1121,7 +1421,7 @@ impl State { let peer_knowledge = knowledge.get_mut(); if peer_knowledge.contains(&message_subject, message_kind) { if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { - gum::debug!( + gum::trace!( target: LOG_TARGET, ?peer_id, ?message_subject, @@ -1129,14 +1429,15 @@ impl State { ); modify_reputation( - &mut self.reputation, + reputation, ctx.sender(), peer_id, COST_DUPLICATE_MESSAGE, ) .await; + metrics.on_approval_duplicate(); } - return + return false } }, hash_map::Entry::Vacant(_) => { @@ -1146,30 +1447,121 @@ impl State { ?message_subject, "Approval from a peer is out of view", ); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE) + .await; + metrics.on_approval_out_of_view(); }, } + } - // if the approval is known to be valid, reward the peer - if entry.knowledge.contains(&message_subject, message_kind) { - gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known approval"); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - BENEFIT_VALID_MESSAGE, - ) - .await; - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); + let good_known_approval = + message_subjects.iter().fold(false, |accumulator, message_subject| { + // if the approval is known to be valid, reward the peer + if entry.knowledge.contains(&message_subject, message_kind) { + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge.received.insert(message_subject.clone(), message_kind); + } + // We already processed this approval no need to continue. + true + } else { + accumulator + } + }); + if good_known_approval { + gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subjects, "Known approval"); + metrics.on_approval_good_known(); + modify_reputation(reputation, ctx.sender(), peer_id, BENEFIT_VALID_MESSAGE).await; + } + + !good_known_approval + } + + async fn import_and_circulate_approval( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + source: MessageSource, + vote: IndirectSignedApprovalVoteV2, + ) { + let _span = self + .spans + .get(&vote.block_hash) + .map(|span| { + span.child(if source.peer_id().is_some() { + "peer-import-and-distribute-approval" + } else { + "local-import-and-distribute-approval" + }) + }) + .unwrap_or_else(|| jaeger::Span::new(&vote.block_hash, "distribute-approval")) + .with_string_tag("block-hash", format!("{:?}", vote.block_hash)) + .with_optional_peer_id(source.peer_id().as_ref()) + .with_stage(jaeger::Stage::ApprovalDistribution); + + let block_hash = vote.block_hash; + let validator_index = vote.validator; + let candidate_indices = &vote.candidate_indices; + let entry = match self.blocks.get_mut(&block_hash) { + Some(entry) + if vote.candidate_indices.iter_ones().fold(true, |result, candidate_index| { + let approval_entry_exists = + entry.contains_approval_entry(candidate_index as _, validator_index); + if !approval_entry_exists { + gum::debug!( + target: LOG_TARGET, ?block_hash, ?candidate_index, validator_index = ?vote.validator, candidate_indices = ?vote.candidate_indices, + peer_id = ?source.peer_id(), "Received approval before assignment" + ); + metrics.on_approval_entry_not_found(); + } + approval_entry_exists && result + }) => + entry, + _ => { + if let Some(peer_id) = source.peer_id() { + if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { + modify_reputation( + &mut self.reputation, + ctx.sender(), + peer_id, + COST_UNEXPECTED_MESSAGE, + ) + .await; + gum::debug!(target: LOG_TARGET, "Received approval for invalid block"); + metrics.on_approval_invalid_block(); + } else { + metrics.on_approval_recent_outdated(); + } } return + }, + }; + + // compute metadata on the assignment. + let message_subjects = candidate_indices + .iter_ones() + .map(|candidate_index| { + MessageSubject( + block_hash, + (candidate_index as CandidateIndex).into(), + validator_index, + ) + }) + .collect_vec(); + let message_kind = MessageKind::Approval; + + if let Some(peer_id) = source.peer_id() { + if !Self::check_approval_can_be_processed( + ctx, + &message_subjects, + message_kind, + entry, + &mut self.reputation, + peer_id, + metrics, + ) + .await + { + return } let (tx, rx) = oneshot::channel(); @@ -1190,7 +1582,6 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, - ?message_subject, ?result, "Checked approval", ); @@ -1204,9 +1595,11 @@ impl State { ) .await; - entry.knowledge.insert(message_subject.clone(), message_kind); - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); + for message_subject in &message_subjects { + entry.knowledge.insert(message_subject.clone(), message_kind); + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge.received.insert(message_subject.clone(), message_kind); + } } }, ApprovalCheckResult::Bad(error) => { @@ -1223,96 +1616,80 @@ impl State { %error, "Got a bad approval from peer", ); + metrics.on_approval_bad(); return }, } } else { - if !entry.knowledge.insert(message_subject.clone(), message_kind) { - // if we already imported an approval, there is no need to distribute it again + let all_approvals_imported_already = + message_subjects.iter().fold(true, |result, message_subject| { + !entry.knowledge.insert(message_subject.clone(), message_kind) && result + }); + if all_approvals_imported_already { + // if we already imported all approvals, there is no need to distribute it again gum::warn!( target: LOG_TARGET, - ?message_subject, "Importing locally an already known approval", ); return } else { gum::debug!( target: LOG_TARGET, - ?message_subject, "Importing locally a new approval", ); } } - // Invariant: to our knowledge, none of the peers except for the `source` know about the - // approval. - metrics.on_approval_imported(); + let mut required_routing = RequiredRouting::None; - let required_routing = match entry.candidates.get_mut(candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Approved - // it should be in assigned state already - match candidate_entry.messages.remove(&validator_index) { - Some(MessageState { - approval_state: ApprovalState::Assigned(cert), - required_routing, - local, - random_routing, - }) => { - candidate_entry.messages.insert( - validator_index, - MessageState { - approval_state: ApprovalState::Approved( - cert, - vote.signature.clone(), - ), - required_routing, - local, - random_routing, - }, - ); + for candidate_index in candidate_indices.iter_ones() { + // The entry is created when assignment is imported, so we assume this exists. + let approval_entry = entry.get_approval_entry(candidate_index as _, validator_index); + if approval_entry.is_none() { + let peer_id = source.peer_id(); + // This indicates a bug in approval-distribution, since we check the knowledge at + // the begining of the function. + gum::warn!( + target: LOG_TARGET, + ?peer_id, + "Unknown approval assignment", + ); + // No rep change as this is caused by an issue + metrics.on_approval_unexpected(); + return + } - required_routing - }, - Some(_) => { - unreachable!( - "we only insert it after the metadata, checked the metadata above; qed" - ); - }, - None => { - // this would indicate a bug in approval-voting - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Importing an approval we don't have an assignment for", - ); + let approval_entry = approval_entry.expect("Just checked above; qed"); - return - }, - } - }, - None => { + if let Err(err) = approval_entry.note_approval(vote.clone(), candidate_index as _) { + // this would indicate a bug in approval-voting: + // - validator index mismatch + // - candidate index mismatch + // - duplicate approval gum::warn!( target: LOG_TARGET, hash = ?block_hash, ?candidate_index, ?validator_index, - "Expected a candidate entry on import_and_circulate_approval", + ?err, + "Possible bug: Vote import failed", ); - + metrics.on_approval_bug(); return - }, - }; + } + required_routing = approval_entry.routing_info().required_routing; + } + + // Invariant: to our knowledge, none of the peers except for the `source` know about + // the approval. + metrics.on_approval_imported(); // Dispatch a ApprovalDistributionV1Message::Approval(vote) // to all peers required by the topology, with the exception of the source peer. - let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let message_subject = &message_subject; + let message_subjects_clone = message_subjects.clone(); let peer_filter = move |peer, knowledge: &PeerKnowledge| { if Some(peer) == source_peer.as_ref() { return false @@ -1329,22 +1706,26 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains(message_subject, MessageKind::Assignment) + in_topology || + message_subjects_clone.iter().fold(true, |result, message_subject| { + result && knowledge.sent.contains(message_subject, MessageKind::Assignment) + }) }; let peers = entry .known_by .iter() .filter(|(p, k)| peer_filter(p, k)) - .map(|(p, _)| p) - .cloned() + .filter_map(|(p, _)| self.peer_views.get(p).map(|entry| (*p, entry.version))) .collect::>(); // Add the metadata of the assignment to the knowledge of each peer. for peer in peers.iter() { // we already filtered peers above, so this should always be Some - if let Some(entry) = entry.known_by.get_mut(peer) { - entry.sent.insert(message_subject.clone(), message_kind); + if let Some(entry) = entry.known_by.get_mut(&peer.0) { + for message_subject in &message_subjects { + entry.sent.insert(message_subject.clone(), message_kind); + } } } @@ -1353,19 +1734,48 @@ impl State { gum::trace!( target: LOG_TARGET, ?block_hash, - ?candidate_index, local = source.peer_id().is_none(), num_peers = peers.len(), "Sending an approval to peers", ); - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals), - )), - )) - .await; + let v1_peers = filter_by_peer_version(&peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(&peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals( + approvals + .clone() + .into_iter() + .filter(|approval| approval.candidate_indices.count_ones() == 1) + .map(|approval| { + approval + .try_into() + .expect("We checked len() == 1 so it should not fail; qed") + }) + .collect::>(), + ), + )), + )) + .await; + metrics.on_approval_sent_v1(); + } + + if !v2_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers, + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), + ), + ), + )) + .await; + metrics.on_approval_sent_v2(); + } } } @@ -1373,7 +1783,7 @@ impl State { fn get_approval_signatures( &mut self, indices: HashSet<(Hash, CandidateIndex)>, - ) -> HashMap { + ) -> HashMap, ValidatorSignature)> { let mut all_sigs = HashMap::new(); for (hash, index) in indices { let _span = self @@ -1396,25 +1806,26 @@ impl State { Some(e) => e, }; - let candidate_entry = match block_entry.candidates.get(index as usize) { - None => { - gum::debug!( - target: LOG_TARGET, - ?hash, - ?index, - "`get_approval_signatures`: could not find candidate entry for given hash and index!" - ); - continue - }, - Some(e) => e, - }; - let sigs = - candidate_entry.messages.iter().filter_map(|(validator_index, message_state)| { - match &message_state.approval_state { - ApprovalState::Approved(_, sig) => Some((*validator_index, sig.clone())), - ApprovalState::Assigned(_) => None, - } - }); + let sigs = block_entry + .get_approval_entries(index) + .into_iter() + .filter_map(|approval_entry| approval_entry.get_approval(index)) + .map(|approval| { + ( + approval.validator, + ( + hash, + approval + .candidate_indices + .iter_ones() + .map(|val| val as CandidateIndex) + .collect_vec(), + approval.signature, + ), + ) + }) + .collect::, ValidatorSignature)>>( + ); all_sigs.extend(sigs); } all_sigs @@ -1427,6 +1838,7 @@ impl State { topologies: &SessionGridTopologies, total_peers: usize, peer_id: PeerId, + protocol_version: ProtocolVersion, view: View, rng: &mut (impl CryptoRng + Rng), ) { @@ -1439,6 +1851,8 @@ impl State { let view_finalized_number = view.finalized_number; for head in view.into_iter() { let mut block = head; + + // Walk the chain back to last finalized block of the peer view. loop { let entry = match entries.get_mut(&block) { Some(entry) if entry.number > view_finalized_number => entry, @@ -1453,25 +1867,29 @@ impl State { } let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - // Iterate all messages in all candidates. - for (candidate_index, validator, message_state) in - entry.candidates.iter_mut().enumerate().flat_map(|(c_i, c)| { - c.messages.iter_mut().map(move |(k, v)| (c_i as _, k, v)) - }) { + // We want to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { // Propagate the message to all peers in the required routing set OR // randomly sample peers. { - let random_routing = &mut message_state.random_routing; - let required_routing = message_state.required_routing; + let required_routing = approval_entry.routing_info().required_routing; + let random_routing = &mut approval_entry.routing_info_mut().random_routing; let rng = &mut *rng; let mut peer_filter = move |peer_id| { let in_topology = topology.as_ref().map_or(false, |t| { t.local_grid_neighbors().route_to_peer(required_routing, peer_id) }); in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) + { + return false + } + let route_random = random_routing.sample(total_peers, rng); if route_random { random_routing.inc_sent(); @@ -1486,40 +1904,49 @@ impl State { } } - let message_subject = MessageSubject(block, candidate_index, *validator); - - let assignment_message = ( - IndirectAssignmentCert { - block_hash: block, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: block, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.get_assignment(); + let approval_messages = approval_entry.get_approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); assignments_to_send.push(assignment_message); } - if let Some(approval_message) = approval_message { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in approval_messages { + let (should_forward_approval, candidates_covered_by_approvals) = + approval_message.candidate_indices.iter_ones().fold( + (true, Vec::new()), + |(should_forward_approval, mut new_covered_approvals), + approval_candidate_index| { + let (message_subject, message_kind) = approval_entry + .create_approval_knowledge( + block, + approval_candidate_index as _, + ); + // The assignments for all candidates signed in the approval + // should already have been sent to the peer, otherwise we can't + // send our approval and risk breaking our reputation. + let should_forward_approval = should_forward_approval && + peer_knowledge + .contains(&message_subject, MessageKind::Assignment); + if !peer_knowledge.contains(&message_subject, message_kind) { + new_covered_approvals.push((message_subject, message_kind)) + } + + (should_forward_approval, new_covered_approvals) + }, + ); + if should_forward_approval { approvals_to_send.push(approval_message); + candidates_covered_by_approvals.into_iter().for_each( + |(approval_knowledge, message_kind)| { + peer_knowledge.sent.insert(approval_knowledge, message_kind); + }, + ); } } } @@ -1532,22 +1959,30 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = assignments_to_send.len(), "Sending assignments to unified peer", ); - send_assignments_batched(sender, assignments_to_send, peer_id).await; + send_assignments_batched( + sender, + assignments_to_send, + &vec![(peer_id, protocol_version)], + ) + .await; } if !approvals_to_send.is_empty() { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = approvals_to_send.len(), "Sending approvals to unified peer", ); - send_approvals_batched(sender, approvals_to_send, peer_id).await; + send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) + .await; } } @@ -1604,7 +2039,8 @@ impl State { false } }, - |_, _, _| {}, + |required_routing, _, _| *required_routing, + &self.peer_views, ) .await; @@ -1628,30 +2064,114 @@ impl State { lag = ?self.approval_checking_lag, "Encountered old block pending gossip topology", ); - return + return *required_routing } + let mut new_required_routing = *required_routing; + if config.l1_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. - if local && *required_routing != RequiredRouting::All { + if local && new_required_routing != RequiredRouting::All { metrics.on_aggression_l1(); - *required_routing = RequiredRouting::All; + new_required_routing = RequiredRouting::All; } } if config.l2_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. Everyone else sends to XY. - if !local && *required_routing != RequiredRouting::GridXY { + if !local && new_required_routing != RequiredRouting::GridXY { metrics.on_aggression_l2(); - *required_routing = RequiredRouting::GridXY; + new_required_routing = RequiredRouting::GridXY; } } + new_required_routing }, + &self.peer_views, ) .await; } + + // Filter out invalid candidate index and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v1_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_index) in assignments.into_iter() { + let cert_bitfield_bits = match cert.cert.kind { + AssignmentCertKind::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKind::RelayVRFModulo { .. } => candidate_index as usize + 1, + }; + + let candidate_bitfield_bits = candidate_index as usize + 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE || candidate_bitfield_bits > MAX_BITFIELD_SIZE + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + gum::error!(target: LOG_TARGET, block_hash = ?cert.block_hash, ?candidate_index, validator_index = ?cert.validator, kind = ?cert.cert.kind, "Bad assignment v1"); + } else { + sanitized_assignments.push((cert.into(), candidate_index.into())) + } + } + + sanitized_assignments + } + + // Filter out oversized candidate and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v2_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_bitfield) in assignments.into_iter() { + let cert_bitfield_bits = match &cert.cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKindV2::RelayVRFModulo { .. } => candidate_bitfield.len(), + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.len(), + }; + + let candidate_bitfield_bits = candidate_bitfield.len(); + + // Our bitfield has `Lsb0`. + let msb = candidate_bitfield_bits - 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE + || candidate_bitfield_bits > MAX_BITFIELD_SIZE + // Ensure minimum bitfield size - MSB needs to be one. + || !candidate_bitfield.bit_at(msb.as_bit_index()) + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + for candidate_index in candidate_bitfield.iter_ones() { + gum::error!(target: LOG_TARGET, block_hash = ?cert.block_hash, ?candidate_index, validator_index = ?cert.validator, "Bad assignment v2"); + } + } else { + sanitized_assignments.push((cert, candidate_bitfield)) + } + } + + sanitized_assignments + } } // This adjusts the required routing of messages in blocks that pass the block filter @@ -1673,9 +2193,10 @@ async fn adjust_required_routing_and_propagate, ) where BlockFilter: Fn(&mut BlockEntry) -> bool, - RoutingModifier: Fn(&mut RequiredRouting, bool, &ValidatorIndex), + RoutingModifier: Fn(&RequiredRouting, bool, &ValidatorIndex) -> RequiredRouting, { let mut peer_assignments = HashMap::new(); let mut peer_approvals = HashMap::new(); @@ -1687,68 +2208,65 @@ async fn adjust_required_routing_and_propagate t, + None => continue, + }; - if message_state.required_routing.is_empty() { - continue - } + // We just need to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in block_entry.approval_entries.values_mut() { + let new_required_routing = routing_modifier( + &approval_entry.routing_info().required_routing, + approval_entry.routing_info().local, + &approval_entry.get_validator_index(), + ); - let topology = match topologies.get_topology(block_entry.session) { - Some(t) => t, - None => continue, - }; + approval_entry.update_required_routing(new_required_routing); - // Propagate the message to all peers in the required routing set. - let message_subject = MessageSubject(*block_hash, candidate_index, *validator); + if approval_entry.routing_info().required_routing.is_empty() { + continue + } - let assignment_message = ( - IndirectAssignmentCert { - block_hash: *block_hash, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: *block_hash, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.get_assignment(); + let approval_messages = approval_entry.get_approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(*block_hash); for (peer, peer_knowledge) in &mut block_entry.known_by { if !topology .local_grid_neighbors() - .route_to_peer(message_state.required_routing, peer) + .route_to_peer(approval_entry.routing_info().required_routing, peer) { continue } - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge.clone(), message_kind); peer_assignments .entry(*peer) .or_insert_with(Vec::new) .push(assignment_message.clone()); } - if let Some(approval_message) = approval_message.as_ref() { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Approval); - peer_approvals - .entry(*peer) - .or_insert_with(Vec::new) - .push(approval_message.clone()); + // Filter approval votes. + for approval_message in &approval_messages { + let mut queued_to_be_sent: bool = false; + for approval_candidate_index in approval_message.candidate_indices.iter_ones() { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(*block_hash, approval_candidate_index as _); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); + if !queued_to_be_sent { + peer_approvals + .entry(*peer) + .or_insert_with(Vec::new) + .push(approval_message.clone()); + queued_to_be_sent = false; + } + } } } } @@ -1756,13 +2274,32 @@ async fn adjust_required_routing_and_propagate { state.handle_new_blocks(ctx, metrics, metas, rng).await; }, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index) => { + ApprovalDistributionMessage::DistributeAssignment(cert, candidate_indices) => { + // TODO: Fix warning: `Importing locally an already known assignment` for multiple + // candidate assignments. This is due to the fact that we call this on wakeup, and + // we do have a wakeup for each candidate index, but a single assignment claiming + // the candidates. + let _span = state + .spans + .get(&cert.block_hash) + .map(|span| span.child("import-and-distribute-assignment")) + .unwrap_or_else(|| jaeger::Span::new(&cert.block_hash, "distribute-assignment")) + .with_string_tag("block-hash", format!("{:?}", cert.block_hash)) + .with_stage(jaeger::Stage::ApprovalDistribution); + gum::debug!( target: LOG_TARGET, - "Distributing our assignment on candidate (block={}, index={})", - cert.block_hash, - candidate_index, + ?candidate_indices, + block_hash = ?cert.block_hash, + "Distributing our assignment on candidates", ); state @@ -1878,7 +2427,7 @@ impl ApprovalDistribution { &metrics, MessageSource::Local, cert, - candidate_index, + candidate_indices, rng, ) .await; @@ -1886,9 +2435,9 @@ impl ApprovalDistribution { ApprovalDistributionMessage::DistributeApproval(vote) => { gum::debug!( target: LOG_TARGET, - "Distributing our approval vote on candidate (block={}, index={})", + "Distributing our approval vote on candidate (block={}, index={:?})", vote.block_hash, - vote.candidate_index, + vote.candidate_indices, ); state @@ -1936,15 +2485,63 @@ const fn ensure_size_not_zero(size: usize) -> usize { /// the protocol configuration. pub const MAX_ASSIGNMENT_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / - std::mem::size_of::<(IndirectAssignmentCert, CandidateIndex)>() / + std::mem::size_of::<(IndirectAssignmentCertV2, CandidateIndex)>() / 3, ); /// The maximum amount of approvals per batch is 33% of maximum allowed by protocol. pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( - MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, + MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, ); +// Low level helper for sending assignments. +async fn send_assignments_batched_inner( + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + batch: impl IntoIterator, + peers: &[PeerId], + peer_version: ValidationVersion, +) { + let peers = peers.into_iter().cloned().collect::>(); + if peer_version == ValidationVersion::VStaging { + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments( + batch.into_iter().collect(), + ), + )), + )) + .await; + } else { + // Create a batch of v1 assignments from v2 assignments that are compatible with v1. + // `IndirectAssignmentCertV2` -> `IndirectAssignmentCert` + let batch = batch + .into_iter() + .filter_map(|(cert, candidates)| { + cert.try_into().ok().map(|cert| { + ( + cert, + // First 1 bit index is the candidate index. + candidates + .first_one() + .map(|index| index as CandidateIndex) + .expect("Assignment was checked for not being empty; qed"), + ) + }) + }) + .collect(); + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(batch), + )), + )) + .await; + } +} + /// Send assignments while honoring the `max_notification_size` of the protocol. /// /// Splitting the messages into multiple notifications allows more granular processing at the @@ -1952,43 +2549,86 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( /// of assignments and can `select!` other tasks. pub(crate) async fn send_assignments_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, - peer: PeerId, + v2_assignments: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = assignments.into_iter().peekable(); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + // Older peers(v1) do not understand `AssignmentsV2` messages, so we have to filter these + // out. + let v1_assignments = v2_assignments + .clone() + .into_iter() + .filter(|(_, candidates)| candidates.count_ones() == 1); + + let mut v1_batches = v1_assignments.peekable(); + + while v1_batches.peek().is_some() { + let batch: Vec<_> = v1_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); + send_assignments_batched_inner(sender, batch, &v1_peers, ValidationVersion::V1).await; + } + } - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); + if !v2_peers.is_empty() { + let mut v2_batches = v2_assignments.into_iter().peekable(); - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage( - vec![peer], - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(batch), - )), - )) - .await; + while v2_batches.peek().is_some() { + let batch = v2_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); + send_assignments_batched_inner(sender, batch, &v2_peers, ValidationVersion::VStaging) + .await; + } } } -/// Send approvals while honoring the `max_notification_size` of the protocol. +/// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - approvals: Vec, - peer: PeerId, + approvals: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = approvals.into_iter().peekable(); - - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + let mut batches = approvals + .clone() + .into_iter() + .filter(|approval| approval.candidate_indices.count_ones() == 1) + .map(|val| val.try_into().expect("We checked conversion should succeed; qed")) + .peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers.clone(), + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(batch), + )), + )) + .await; + } + } - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage( - vec![peer], - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(batch), - )), - )) - .await; + if !v2_peers.is_empty() { + let mut batches = approvals.into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers.clone(), + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(batch), + ), + ), + )) + .await; + } } } diff --git a/node/network/approval-distribution/src/metrics.rs b/node/network/approval-distribution/src/metrics.rs index 896866ce099a..094874820f26 100644 --- a/node/network/approval-distribution/src/metrics.rs +++ b/node/network/approval-distribution/src/metrics.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait}; +use polkadot_node_primitives::approval::v2::AssignmentCertKindV2; /// Approval Distribution metrics. #[derive(Default, Clone)] @@ -22,21 +23,36 @@ pub struct Metrics(Option); #[derive(Clone)] struct MetricsInner { - assignments_imported_total: prometheus::Counter, + assignments_imported_total: prometheus::CounterVec, approvals_imported_total: prometheus::Counter, unified_with_peer_total: prometheus::Counter, aggression_l1_messages_total: prometheus::Counter, aggression_l2_messages_total: prometheus::Counter, - time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, time_awaiting_approval_voting: prometheus::Histogram, + assignments_received_result: prometheus::CounterVec, + approvals_received_result: prometheus::CounterVec, +} + +trait AsLabel { + fn as_label(&self) -> &str; +} + +impl AsLabel for &AssignmentCertKindV2 { + fn as_label(&self) -> &str { + match self { + AssignmentCertKindV2::RelayVRFDelay { .. } => "VRF Delay", + AssignmentCertKindV2::RelayVRFModulo { .. } => "VRF Modulo", + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => "VRF Modulo Compact", + } + } } impl Metrics { - pub(crate) fn on_assignment_imported(&self) { + pub(crate) fn on_assignment_imported(&self, kind: &AssignmentCertKindV2) { if let Some(metrics) = &self.0 { - metrics.assignments_imported_total.inc(); + metrics.assignments_imported_total.with_label_values(&[kind.as_label()]).inc(); } } @@ -64,6 +80,144 @@ impl Metrics { .map(|metrics| metrics.time_import_pending_now_known.start_timer()) } + pub fn on_approval_already_known(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["known"]).inc() + } + } + + pub fn on_approval_entry_not_found(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["noapprovalentry"]).inc() + } + } + + pub fn on_approval_recent_outdated(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["outdated"]).inc() + } + } + + pub fn on_approval_invalid_block(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["invalidblock"]).inc() + } + } + + pub fn on_approval_unknown_assignment(&self) { + if let Some(metrics) = &self.0 { + metrics + .approvals_received_result + .with_label_values(&["unknownassignment"]) + .inc() + } + } + + pub fn on_approval_duplicate(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["duplicate"]).inc() + } + } + + pub fn on_approval_out_of_view(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["outofview"]).inc() + } + } + + pub fn on_approval_good_known(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["goodknown"]).inc() + } + } + + pub fn on_approval_bad(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["bad"]).inc() + } + } + + pub fn on_approval_unexpected(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["unexpected"]).inc() + } + } + + pub fn on_approval_bug(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["bug"]).inc() + } + } + + pub fn on_approval_sent_v1(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["v1"]).inc() + } + } + + pub fn on_approval_sent_v2(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["v2"]).inc() + } + } + + pub fn on_assignment_already_known(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["known"]).inc() + } + } + + pub fn on_assignment_recent_outdated(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["outdated"]).inc() + } + } + + pub fn on_assignment_invalid_block(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["invalidblock"]).inc() + } + } + + pub fn on_assignment_duplicate(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["duplicate"]).inc() + } + } + + pub fn on_assignment_out_of_view(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["outofview"]).inc() + } + } + + pub fn on_assignment_good_known(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["goodknown"]).inc() + } + } + + pub fn on_assignment_bad(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["bad"]).inc() + } + } + + pub fn on_assignment_duplicatevoting(&self) { + if let Some(metrics) = &self.0 { + metrics + .assignments_received_result + .with_label_values(&["duplicatevoting"]) + .inc() + } + } + + pub fn on_assignment_far(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["far"]).inc() + } + } + pub(crate) fn time_awaiting_approval_voting( &self, ) -> Option { @@ -89,9 +243,12 @@ impl MetricsTrait for Metrics { fn try_register(registry: &prometheus::Registry) -> Result { let metrics = MetricsInner { assignments_imported_total: prometheus::register( - prometheus::Counter::new( - "polkadot_parachain_assignments_imported_total", - "Number of valid assignments imported locally or from other peers.", + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_imported_total", + "Number of valid assignments imported locally or from other peers.", + ), + &["kind"], )?, registry, )?, @@ -124,10 +281,16 @@ impl MetricsTrait for Metrics { registry, )?, time_unify_with_peer: prometheus::register( - prometheus::Histogram::with_opts(prometheus::HistogramOpts::new( - "polkadot_parachain_time_unify_with_peer", - "Time spent within fn `unify_with_peer`.", - ).buckets(vec![0.000625, 0.00125,0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,]))?, + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_time_unify_with_peer", + "Time spent within fn `unify_with_peer`.", + ) + .buckets(vec![ + 0.000625, 0.00125, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, + 0.5, 1.0, 2.5, 5.0, 10.0, + ]), + )?, registry, )?, time_import_pending_now_known: prometheus::register( @@ -144,6 +307,26 @@ impl MetricsTrait for Metrics { ).buckets(vec![0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, 4.9152, 6.5536,]))?, registry, )?, + assignments_received_result: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_received_result", + "Result of a processed assignement", + ), + &["status"] + )?, + registry, + )?, + approvals_received_result: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_approvals_received_result", + "Result of a processed approval", + ), + &["status"] + )?, + registry, + )?, }; Ok(Metrics(Some(metrics))) } diff --git a/node/network/approval-distribution/src/tests.rs b/node/network/approval-distribution/src/tests.rs index bfd7c945069c..8ae561b449c7 100644 --- a/node/network/approval-distribution/src/tests.rs +++ b/node/network/approval-distribution/src/tests.rs @@ -24,20 +24,26 @@ use polkadot_node_network_protocol::{ view, ObservedRole, }; use polkadot_node_primitives::approval::{ - AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, VrfOutput, VrfProof, + VrfSignature, + }, + v2::{ + AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, IndirectAssignmentCertV2, + RELAY_VRF_MODULO_CONTEXT, + }, }; use polkadot_node_subsystem::messages::{ network_bridge_event, AllMessages, ApprovalCheckError, ReportPeerMessage, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt as _}; -use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, HashT}; +use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, CoreIndex, HashT}; use polkadot_primitives_test_helpers::dummy_signature; use rand::SeedableRng; use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; use sp_core::crypto::Pair as PairT; use std::time::Duration; - type VirtualOverseer = test_helpers::TestSubsystemContextHandle; fn test_harness>( @@ -127,14 +133,13 @@ fn make_gossip_topology( all_peers: &[(PeerId, AuthorityDiscoveryId)], neighbors_x: &[usize], neighbors_y: &[usize], + local_index: usize, ) -> network_bridge_event::NewGossipTopology { // This builds a grid topology which is a square matrix. // The local validator occupies the top left-hand corner. // The X peers occupy the same row and the Y peers occupy // the same column. - let local_index = 1; - assert_eq!( neighbors_x.len(), neighbors_y.len(), @@ -220,13 +225,14 @@ async fn setup_peer_with_view( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, view: View, + version: ValidationVersion, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( *peer_id, ObservedRole::Full, - ValidationVersion::V1.into(), + version.into(), None, )), ) @@ -255,6 +261,21 @@ async fn send_message_from_peer( .await; } +async fn send_message_from_peer_v2( + virtual_overseer: &mut VirtualOverseer, + peer_id: &PeerId, + msg: protocol_vstaging::ApprovalDistributionMessage, +) { + overseer_send( + virtual_overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + peer_id.clone(), + Versioned::VStaging(msg), + )), + ) + .await; +} + fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> IndirectAssignmentCert { let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); let msg = b"WhenParachains?"; @@ -273,6 +294,28 @@ fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> Indirect } } +fn fake_assignment_cert_v2( + block_hash: Hash, + validator: ValidatorIndex, + core_bitfield: CoreBitfield, +) -> IndirectAssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield }, + vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) }, + }, + } +} + async fn expect_reputation_change( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, @@ -321,19 +364,23 @@ fn state_with_reputation_delay() -> State { /// the new peer sends us the same assignment #[test] fn try_import_the_same_assignment() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let peer_d = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, view![]).await; - setup_peer_with_view(overseer, &peer_b, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; + + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -362,10 +409,11 @@ fn try_import_the_same_assignment() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_indices, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(claimed_indices, 0u32.into()); + assert_eq!(assignment, cert.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -385,8 +433,8 @@ fn try_import_the_same_assignment() { } ); - // setup new peer - setup_peer_with_view(overseer, &peer_d, view![]).await; + // setup new peer with V2 + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; // send the same assignment from peer_d let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); @@ -400,6 +448,102 @@ fn try_import_the_same_assignment() { }); } +/// Just like `try_import_the_same_assignment` but use `VRFModuloCompact` assignments for multiple +/// cores. +#[test] +fn try_import_the_same_assignment_v2() { + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cores = vec![1, 2, 3, 4]; + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; + + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v2(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; + + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_indices, + tx, + )) => { + assert_eq!(claimed_indices, cores.try_into().unwrap()); + assert_eq!(assignment, cert.into()); + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); + + // setup new peer + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + + // send the same assignment from peer_d + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + /// import an assignment /// connect a new peer /// state sends aggregated reputation change @@ -413,7 +557,7 @@ fn delay_reputation_change() { let overseer = &mut virtual_overseer; // Setup peers - setup_peer_with_view(overseer, &peer, view![]).await; + setup_peer_with_view(overseer, &peer, view![], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -440,10 +584,11 @@ fn delay_reputation_change() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_candidates, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(assignment.cert, cert.cert.into()); + assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -474,7 +619,7 @@ fn spam_attack_results_in_negative_reputation_change() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash_b` with 20 candidates let candidates_count = 20; @@ -513,8 +658,8 @@ fn spam_attack_results_in_negative_reputation_change() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -552,13 +697,18 @@ fn spam_attack_results_in_negative_reputation_change() { #[test] fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let parent_hash = Hash::repeat_byte(0xFF); - let peer_a = PeerId::random(); let hash = Hash::repeat_byte(0xAA); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; + + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; // new block `hash` with 1 candidates let meta = BlockApprovalMeta { @@ -578,7 +728,10 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -625,18 +778,20 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { #[test] fn import_approval_happy_path() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, view![]).await; - setup_peer_with_view(overseer, &peer_b, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, view![hash]).await; + // setup peers with V1 and V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -650,16 +805,23 @@ fn import_approval_happy_path() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Set up a gossip topology, where a, b, and c are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // import an assignment related to `hash` locally let validator_index = ValidatorIndex(0); let candidate_index = 0u32; let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; + // 1 peer is v1 assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( @@ -668,20 +830,34 @@ fn import_approval_happy_path() { protocol_v1::ApprovalDistributionMessage::Assignments(assignments) )) )) => { - assert_eq!(peers.len(), 2); + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + // 1 peer is v2 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); assert_eq!(assignments.len(), 1); } ); // send the an approval from peer_b - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -722,8 +898,8 @@ fn import_approval_bad() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, view![]).await; - setup_peer_with_view(overseer, &peer_b, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -742,14 +918,14 @@ fn import_approval_bad() { let cert = fake_assignment_cert(hash, validator_index); // send the an approval from peer_b, we don't have an assignment yet - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -765,8 +941,8 @@ fn import_approval_bad() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -774,8 +950,8 @@ fn import_approval_bad() { expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; // and try again - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -874,7 +1050,8 @@ fn update_peer_view() { let hash_b = Hash::repeat_byte(0xBB); let hash_c = Hash::repeat_byte(0xCC); let hash_d = Hash::repeat_byte(0xDD); - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let peer = &peer_a; let state = test_harness(State::default(), |mut virtual_overseer| async move { @@ -908,15 +1085,26 @@ fn update_peer_view() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_a, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), + ) + .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_b, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), + ) + .await; // connect a peer - setup_peer_with_view(overseer, peer, view![hash_a]).await; + setup_peer_with_view(overseer, peer, view![hash_a], ValidationVersion::V1).await; // we should send relevant assignments to the peer assert_matches!( @@ -934,7 +1122,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_views.get(peer).map(|v| v.finalized_number), Some(0)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(0)); assert_eq!( state .blocks @@ -965,7 +1153,7 @@ fn update_peer_view() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_c.clone(), 0), + ApprovalDistributionMessage::DistributeAssignment(cert_c.clone().into(), 0.into()), ) .await; @@ -986,7 +1174,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_views.get(peer).map(|v| v.finalized_number), Some(2)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(2)); assert_eq!( state .blocks @@ -1016,7 +1204,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_views.get(peer).map(|v| v.finalized_number), Some(finalized_number)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(finalized_number)); assert!(state.blocks.get(&hash_c).unwrap().known_by.get(peer).is_none()); } @@ -1031,7 +1219,7 @@ fn import_remotely_then_locally() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup the peer - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1061,8 +1249,8 @@ fn import_remotely_then_locally() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.clone().into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1072,21 +1260,24 @@ fn import_remotely_then_locally() { // import the same assignment locally overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); // send the approval remotely - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, peer, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1110,7 +1301,8 @@ fn import_remotely_then_locally() { #[test] fn sends_assignments_even_when_state_is_approved() { - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; @@ -1130,6 +1322,9 @@ fn sends_assignments_even_when_state_is_approved() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let validator_index = ValidatorIndex(0); let candidate_index = 0u32; @@ -1144,15 +1339,21 @@ fn sends_assignments_even_when_state_is_approved() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; // connect the peer. - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1188,6 +1389,115 @@ fn sends_assignments_even_when_state_is_approved() { }); } +/// Same as `sends_assignments_even_when_state_is_approved_v2` but with `VRFModuloCompact` assignemnts. +#[test] +fn sends_assignments_even_when_state_is_approved_v2() { + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + let peer = &peer_a; + + let _ = test_harness(State::default(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 4], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + + let validator_index = ValidatorIndex(0); + let cores = vec![0, 1, 2, 3]; + let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); + + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + + // Assumes candidate index == core index. + let approvals = cores + .iter() + .map(|core| IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices: (*core).into(), + validator: validator_index, + signature: dummy_signature(), + }) + .collect::>(); + + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_bitfield.clone(), + ), + ) + .await; + + for approval in &approvals { + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone()), + ) + .await; + } + + // connect the peer. + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::VStaging).await; + + let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(peers, vec![peer.clone()]); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a + // hashmap as well. + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); + + assert_eq!(peers, vec![peer.clone()]); + assert_eq!(sent_approvals, approvals); + } + ); + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + /// /// /// 1. Receive remote peer view update with an unknown head @@ -1216,7 +1526,7 @@ fn race_condition_in_local_vs_remote_view_update() { }; // This will send a peer view that is ahead of our view - setup_peer_with_view(overseer, peer, view![hash_b]).await; + setup_peer_with_view(overseer, peer, view![hash_b], ValidationVersion::V1).await; // Send our view update to include a new head overseer_send( @@ -1254,8 +1564,8 @@ fn race_condition_in_local_vs_remote_view_update() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1280,19 +1590,25 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology( + 1, + &peers, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), ) .await; let expected_indices = [ // Both dimensions in the gossip topology - 0, 10, 20, 30, 50, 51, 52, 53, + 0, 10, 20, 30, 40, 60, 70, 80, 50, 51, 52, 53, 54, 55, 56, 57, ]; // new block `hash_a` with 1 candidates @@ -1322,12 +1638,18 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1385,13 +1707,13 @@ fn propagates_assignments_along_unshared_dimension() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -1527,20 +1849,26 @@ fn propagates_to_required_after_connect() { // Connect all peers except omitted. for (i, (peer, _)) in peers.iter().enumerate() { if !omitted.contains(&i) { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } } // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology( + 1, + &peers, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), ) .await; let expected_indices = [ // Both dimensions in the gossip topology, minus omitted. - 20, 30, 52, 53, + 20, 30, 40, 60, 70, 80, 52, 53, 54, 55, 56, 57, ]; // new block `hash_a` with 1 candidates @@ -1570,12 +1898,18 @@ fn propagates_to_required_after_connect() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1616,7 +1950,7 @@ fn propagates_to_required_after_connect() { ); for i in omitted.iter().copied() { - setup_peer_with_view(overseer, &peers[i].0, view![hash]).await; + setup_peer_with_view(overseer, &peers[i].0, view![hash], ValidationVersion::V1).await; assert_matches!( overseer_recv(overseer).await, @@ -1665,7 +1999,7 @@ fn sends_to_more_peers_after_getting_topology() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1695,57 +2029,28 @@ fn sends_to_more_peers_after_getting_topology() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; - let mut expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; - let assignment_sent_peers = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - // Only sends to random peers. - assert_eq!(sent_peers.len(), 4); - for peer in &sent_peers { - let i = peers.iter().position(|p| peer == &p.0).unwrap(); - // Random gossip before topology can send to topology-targeted peers. - // Remove them from the expected indices so we don't expect - // them to get the messages again after the assignment. - expected_indices.retain(|&i2| i2 != i); - } - assert_eq!(sent_assignments, assignments); - sent_peers - } - ); - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Random sampling is reused from the assignment. - assert_eq!(sent_peers, assignment_sent_peers); - assert_eq!(sent_approvals, approvals); - } - ); + let expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -1817,7 +2122,7 @@ fn originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1848,18 +2153,24 @@ fn originator_aggression_l1() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1976,7 +2287,7 @@ fn non_originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2001,11 +2312,11 @@ fn non_originator_aggression_l1() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; - let assignments = vec![(cert.clone(), candidate_index)]; + let assignments = vec![(cert.clone().into(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); // Issuer of the message is important, not the peer we receive from. @@ -2081,7 +2392,7 @@ fn non_originator_aggression_l2() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2106,7 +2417,7 @@ fn non_originator_aggression_l2() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2246,13 +2557,13 @@ fn resends_messages_periodically() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2389,21 +2700,33 @@ fn batch_test_round(message_count: usize) { let validators = 0..message_count; let assignments: Vec<_> = validators .clone() - .map(|index| (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)), 0)) + .map(|index| { + (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)).into(), 0.into()) + }) .collect(); let approvals: Vec<_> = validators - .map(|index| IndirectSignedApprovalVote { + .map(|index| IndirectSignedApprovalVoteV2 { block_hash: Hash::zero(), - candidate_index: 0, + candidate_indices: 0u32.into(), validator: ValidatorIndex(index as u32), signature: dummy_signature(), }) .collect(); let peer = PeerId::random(); - send_assignments_batched(&mut sender, assignments.clone(), peer).await; - send_approvals_batched(&mut sender, approvals.clone(), peer).await; + send_assignments_batched( + &mut sender, + assignments.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; + send_approvals_batched( + &mut sender, + approvals.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; // Check expected assignments batches. for assignment_index in (0..assignments.len()).step_by(super::MAX_ASSIGNMENT_BATCH_SIZE) { @@ -2425,7 +2748,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, assignment) in sent_assignments.iter().enumerate() { - assert_eq!(assignment.0, assignments[assignment_index + message_index].0); + assert_eq!(assignment.0, assignments[assignment_index + message_index].0.clone().try_into().unwrap()); assert_eq!(assignment.1, 0); } } @@ -2452,7 +2775,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, approval) in sent_approvals.iter().enumerate() { - assert_eq!(approval, &approvals[approval_index + message_index]); + assert_eq!(approval, &approvals[approval_index + message_index].clone().try_into().unwrap()); } } ); diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index fb0cdb720571..8c8adf97cbef 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -104,7 +104,7 @@ const TIMEOUT_START_NEW_REQUESTS: Duration = CHUNK_REQUEST_TIMEOUT; const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); /// PoV size limit in bytes for which prefer fetching from backers. -const SMALL_POV_LIMIT: usize = 128 * 1024; +const SMALL_POV_LIMIT: usize = 3 * 1024 * 1024; #[derive(Clone, PartialEq)] /// The strategy we use to recover the PoV. diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index ddb1d83b8a05..08db04f9fd93 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -25,10 +25,11 @@ use futures::{channel::oneshot, FutureExt}; use polkadot_node_network_protocol::{ - self as net_protocol, + self as net_protocol, filter_by_peer_version, grid_topology::{ GridNeighbors, RandomRouting, RequiredRouting, SessionBoundGridTopologyStorage, }, + peer_set::{ProtocolVersion, ValidationVersion}, v1 as protocol_v1, OurView, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_subsystem::{ @@ -76,25 +77,45 @@ struct BitfieldGossipMessage { } impl BitfieldGossipMessage { - fn into_validation_protocol(self) -> net_protocol::VersionedValidationProtocol { - self.into_network_message().into() + fn into_validation_protocol( + self, + protocol_version: ProtocolVersion, + ) -> net_protocol::VersionedValidationProtocol { + self.into_network_message(protocol_version).into() } - fn into_network_message(self) -> net_protocol::BitfieldDistributionMessage { - Versioned::V1(protocol_v1::BitfieldDistributionMessage::Bitfield( + fn into_network_message( + self, + protocol_version: ProtocolVersion, + ) -> net_protocol::BitfieldDistributionMessage { + // VStaging re-exports v1 message type. + let message = protocol_v1::BitfieldDistributionMessage::Bitfield( self.relay_parent, self.signed_availability.into(), - )) + ); + + match ValidationVersion::try_from(protocol_version) { + Ok(ValidationVersion::V1) => Versioned::V1(message), + Ok(ValidationVersion::VStaging) => Versioned::VStaging(message), + Err(_) => unreachable!("Invalid peer protocol"), + } } } +// We keep track of each peer view and protocol version using this struct. +#[derive(Debug, PartialEq)] +struct PeerEntry { + pub view: View, + pub version: net_protocol::peer_set::ProtocolVersion, +} + /// Data used to track information of peers and relay parents the /// overseer ordered us to work on. -#[derive(Default, Debug)] +#[derive(Default)] struct ProtocolState { - /// Track all active peers and their views + /// Track all active peer views and protocol versions /// to determine what is relevant to them. - peer_views: HashMap, + peer_entries: HashMap, /// The current and previous gossip topologies topologies: SessionBoundGridTopologyStorage, @@ -357,7 +378,7 @@ async fn handle_bitfield_distribution( ctx, job_data, topology, - &mut state.peer_views, + &mut state.peer_entries, validator, msg, required_routing, @@ -376,7 +397,7 @@ async fn relay_message( ctx: &mut Context, job_data: &mut PerRelayParentData, topology_neighbors: &GridNeighbors, - peer_views: &mut HashMap, + peer_entries: &mut HashMap, validator: ValidatorId, message: BitfieldGossipMessage, required_routing: RequiredRouting, @@ -394,16 +415,16 @@ async fn relay_message( .await; drop(_span); - let total_peers = peer_views.len(); + let total_peers = peer_entries.len(); let mut random_routing: RandomRouting = Default::default(); let _span = span.child("interested-peers"); // pass on the bitfield distribution to all interested peers - let interested_peers = peer_views + let interested_peers = peer_entries .iter() - .filter_map(|(peer, view)| { + .filter_map(|(peer, entry)| { // check interest in the peer in this message's relay parent - if view.contains(&message.relay_parent) { + if entry.view.contains(&message.relay_parent) { let message_needed = job_data.message_from_validator_needed_by_peer(&peer, &validator); if message_needed { @@ -418,7 +439,7 @@ async fn relay_message( }; if need_routing { - Some(*peer) + Some((*peer, entry.version)) } else { None } @@ -429,13 +450,13 @@ async fn relay_message( None } }) - .collect::>(); + .collect::>(); interested_peers.iter().for_each(|peer| { // track the message as sent for this peer job_data .message_sent_to_peer - .entry(*peer) + .entry(peer.0) .or_default() .insert(validator.clone()); }); @@ -450,11 +471,26 @@ async fn relay_message( ); } else { let _span = span.child("gossip"); - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - interested_peers, - message.into_validation_protocol(), - )) - .await; + + let v1_peers = filter_by_peer_version(&interested_peers, ValidationVersion::V1.into()); + let v2_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers, + message.clone().into_validation_protocol(ValidationVersion::V1.into()), + )) + .await; + } + + if !v2_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers, + message.into_validation_protocol(ValidationVersion::VStaging.into()), + )) + .await; + } } } @@ -616,7 +652,7 @@ async fn process_incoming_peer_message( ctx, job_data, topology, - &mut state.peer_views, + &mut state.peer_entries, validator, message, required_routing, @@ -647,15 +683,18 @@ async fn handle_network_msg( let _timer = metrics.time_handle_network_msg(); match bridge_message { - NetworkBridgeEvent::PeerConnected(peer, role, _, _) => { + NetworkBridgeEvent::PeerConnected(peer, role, version, _) => { gum::trace!(target: LOG_TARGET, ?peer, ?role, "Peer connected"); // insert if none already present - state.peer_views.entry(peer).or_default(); + state + .peer_entries + .entry(peer) + .or_insert(PeerEntry { view: Default::default(), version }); }, NetworkBridgeEvent::PeerDisconnected(peer) => { gum::trace!(target: LOG_TARGET, ?peer, "Peer disconnected"); // get rid of superfluous data - state.peer_views.remove(&peer); + state.peer_entries.remove(&peer); }, NetworkBridgeEvent::NewGossipTopology(gossip_topology) => { let session_index = gossip_topology.session; @@ -683,19 +722,23 @@ async fn handle_network_msg( // in case we already knew that peer in the past // it might have had an existing view, we use to initialize // and minimize the delta on `PeerViewChange` to be sent - if let Some(old_view) = state.peer_views.remove(&new_peer) { - handle_peer_view_change(ctx, state, new_peer, old_view, rng).await; + if let Some(entry) = state.peer_entries.remove(&new_peer) { + handle_peer_view_change(ctx, state, new_peer, entry.version, entry.view, rng) + .await; } } }, - NetworkBridgeEvent::PeerViewChange(peerid, new_view) => { - gum::trace!(target: LOG_TARGET, ?peerid, ?new_view, "Peer view change"); - handle_peer_view_change(ctx, state, peerid, new_view, rng).await; + NetworkBridgeEvent::PeerViewChange(peer_id, new_view) => { + gum::trace!(target: LOG_TARGET, ?peer_id, ?new_view, "Peer view change"); + if let Some(entry) = state.peer_entries.get(&peer_id) { + handle_peer_view_change(ctx, state, peer_id, entry.version, new_view, rng).await; + } }, NetworkBridgeEvent::OurViewChange(new_view) => { gum::trace!(target: LOG_TARGET, ?new_view, "Our view change"); handle_our_view_change(state, new_view); }, + NetworkBridgeEvent::PeerMessage(remote, Versioned::VStaging(message)) | NetworkBridgeEvent::PeerMessage(remote, Versioned::V1(message)) => process_incoming_peer_message(ctx, state, metrics, remote, message, rng).await, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { @@ -733,13 +776,15 @@ async fn handle_peer_view_change( ctx: &mut Context, state: &mut ProtocolState, origin: PeerId, + version: ProtocolVersion, view: View, rng: &mut (impl CryptoRng + Rng), ) { let added = state - .peer_views + .peer_entries .entry(origin) - .or_default() + .or_insert(PeerEntry { version, view: Default::default() }) + .view .replace_difference(view) .cloned() .collect::>(); @@ -780,7 +825,7 @@ async fn handle_peer_view_change( .collect(); for (validator, message) in delta_set.into_iter() { - send_tracked_gossip_message(ctx, state, origin, validator, message).await; + send_tracked_gossip_message(ctx, state, origin, version, validator, message).await; } } @@ -790,6 +835,7 @@ async fn send_tracked_gossip_message( ctx: &mut Context, state: &mut ProtocolState, dest: PeerId, + protocol_version: ProtocolVersion, validator: ValidatorId, message: BitfieldGossipMessage, ) { @@ -812,7 +858,7 @@ async fn send_tracked_gossip_message( ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( vec![dest], - message.into_validation_protocol(), + message.into_validation_protocol(protocol_version), )) .await; } diff --git a/node/network/bitfield-distribution/src/tests.rs b/node/network/bitfield-distribution/src/tests.rs index 9aea5a7178b7..7ff8f997e429 100644 --- a/node/network/bitfield-distribution/src/tests.rs +++ b/node/network/bitfield-distribution/src/tests.rs @@ -85,7 +85,16 @@ fn prewarmed_state( span: PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"), }, }, - peer_views: peers.iter().cloned().map(|peer| (peer, view!(relay_parent))).collect(), + peer_entries: peers + .iter() + .cloned() + .map(|peer| { + ( + peer, + PeerEntry { view: view!(relay_parent), version: ValidationVersion::V1.into() }, + ) + }) + .collect(), topologies, view: our_view!(relay_parent), reputation: ReputationAggregator::new(|_| true), @@ -211,7 +220,10 @@ fn receive_invalid_signature() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, invalid_msg.into_network_message()), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + invalid_msg.into_network_message(ValidationVersion::V1.into()) + ), &mut rng, )); @@ -222,7 +234,10 @@ fn receive_invalid_signature() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, invalid_msg_2.into_network_message()), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + invalid_msg_2.into_network_message(ValidationVersion::V1.into()) + ), &mut rng, )); // reputation change due to invalid signature @@ -256,7 +271,10 @@ fn receive_invalid_validator_index() { let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a, ReputationAggregator::new(|_| true)); - state.peer_views.insert(peer_b, view![hash_a]); + state.peer_entries.insert( + peer_b.clone(), + PeerEntry { view: view![hash_a], version: ValidationVersion::V1.into() }, + ); let payload = AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); let signed = Signed::::sign( @@ -281,7 +299,10 @@ fn receive_invalid_validator_index() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, msg.into_network_message()), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.into_network_message(ValidationVersion::V1.into()) + ), &mut rng, )); @@ -344,7 +365,10 @@ fn receive_duplicate_messages() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -377,7 +401,10 @@ fn receive_duplicate_messages() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_a, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_a.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -396,7 +423,10 @@ fn receive_duplicate_messages() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -463,7 +493,10 @@ fn delay_reputation_change() { handle .send(FromOrchestra::Communication { msg: BitfieldDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(peer, msg.clone().into_network_message()), + NetworkBridgeEvent::PeerMessage( + peer.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), ), }) .await; @@ -486,7 +519,10 @@ fn delay_reputation_change() { handle .send(FromOrchestra::Communication { msg: BitfieldDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(peer, msg.clone().into_network_message()), + NetworkBridgeEvent::PeerMessage( + peer.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), ), }) .await; @@ -546,8 +582,14 @@ fn do_not_relay_message_twice() { .flatten() .expect("should be signed"); - state.peer_views.insert(peer_b, view![hash]); - state.peer_views.insert(peer_a, view![hash]); + state.peer_entries.insert( + peer_b.clone(), + PeerEntry { view: view![hash], version: ValidationVersion::V1.into() }, + ); + state.peer_entries.insert( + peer_a.clone(), + PeerEntry { view: view![hash], version: ValidationVersion::V1.into() }, + ); let msg = BitfieldGossipMessage { relay_parent: hash, signed_availability: signed_bitfield.clone() }; @@ -564,7 +606,7 @@ fn do_not_relay_message_twice() { &mut ctx, state.per_relay_parent.get_mut(&hash).unwrap(), &gossip_peers, - &mut state.peer_views, + &mut state.peer_entries, validator.clone(), msg.clone(), RequiredRouting::GridXY, @@ -591,7 +633,7 @@ fn do_not_relay_message_twice() { assert_eq!(2, peers.len()); assert!(peers.contains(&peer_a)); assert!(peers.contains(&peer_b)); - assert_eq!(send_msg, msg.clone().into_validation_protocol()); + assert_eq!(send_msg, msg.clone().into_validation_protocol(ValidationVersion::V1.into())); } ); @@ -600,7 +642,7 @@ fn do_not_relay_message_twice() { &mut ctx, state.per_relay_parent.get_mut(&hash).unwrap(), &gossip_peers, - &mut state.peer_views, + &mut state.peer_entries, validator.clone(), msg.clone(), RequiredRouting::GridXY, @@ -687,14 +729,17 @@ fn changing_view() { &mut rng, )); - assert!(state.peer_views.contains_key(&peer_b)); + assert!(state.peer_entries.contains_key(&peer_b)); // recv a first message from the network launch!(handle_network_msg( &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -729,8 +774,11 @@ fn changing_view() { &mut rng, )); - assert!(state.peer_views.contains_key(&peer_b)); - assert_eq!(state.peer_views.get(&peer_b).expect("Must contain value for peer B"), &view![]); + assert!(state.peer_entries.contains_key(&peer_b)); + assert_eq!( + state.peer_entries.get(&peer_b).expect("Must contain value for peer B"), + &PeerEntry { view: view![], version: ValidationVersion::V1.into() } + ); // on rx of the same message, since we are not interested, // should give penalty @@ -738,7 +786,10 @@ fn changing_view() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -770,7 +821,10 @@ fn changing_view() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_a, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_a.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -817,8 +871,14 @@ fn do_not_send_message_back_to_origin() { .flatten() .expect("should be signed"); - state.peer_views.insert(peer_b, view![hash]); - state.peer_views.insert(peer_a, view![hash]); + state.peer_entries.insert( + peer_b.clone(), + PeerEntry { view: view![hash], version: ValidationVersion::V1.into() }, + ); + state.peer_entries.insert( + peer_a.clone(), + PeerEntry { view: view![hash], version: ValidationVersion::V1.into() }, + ); let msg = BitfieldGossipMessage { relay_parent: hash, signed_availability: signed_bitfield.clone() }; @@ -833,7 +893,10 @@ fn do_not_send_message_back_to_origin() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b, msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -855,7 +918,7 @@ fn do_not_send_message_back_to_origin() { ) => { assert_eq!(1, peers.len()); assert!(peers.contains(&peer_a)); - assert_eq!(send_msg, msg.clone().into_validation_protocol()); + assert_eq!(send_msg, msg.clone().into_validation_protocol(ValidationVersion::V1.into())); } ); @@ -932,7 +995,10 @@ fn topology_test() { .expect("should be signed"); peers_x.iter().chain(peers_y.iter()).for_each(|peer| { - state.peer_views.insert(*peer, view![hash]); + state.peer_entries.insert( + peer.clone(), + PeerEntry { view: view![hash], version: ValidationVersion::V1.into() }, + ); }); let msg = @@ -948,7 +1014,10 @@ fn topology_test() { &mut ctx, &mut state, &Default::default(), - NetworkBridgeEvent::PeerMessage(peers_x[0], msg.clone().into_network_message(),), + NetworkBridgeEvent::PeerMessage( + peers_x[0].clone(), + msg.clone().into_network_message(ValidationVersion::V1.into()), + ), &mut rng, )); @@ -975,7 +1044,7 @@ fn topology_test() { assert!(topology.peers_x.iter().filter(|peer| peers.contains(&peer)).count() == 4); // Must never include originator assert!(!peers.contains(&peers_x[0])); - assert_eq!(send_msg, msg.clone().into_validation_protocol()); + assert_eq!(send_msg, msg.clone().into_validation_protocol(ValidationVersion::V1.into())); } ); diff --git a/node/network/bridge/src/network.rs b/node/network/bridge/src/network.rs index 04026197f6eb..a12bd29fd3b8 100644 --- a/node/network/bridge/src/network.rs +++ b/node/network/bridge/src/network.rs @@ -28,45 +28,95 @@ use sc_network::{ }; use polkadot_node_network_protocol::{ - peer_set::{PeerSet, PeerSetProtocolNames, ProtocolVersion}, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, - PeerId, + v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, }; use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash}; -use crate::validator_discovery::AuthorityDiscovery; +use crate::{metrics::Metrics, validator_discovery::AuthorityDiscovery, WireMessage}; // network bridge network abstraction log target const LOG_TARGET: &'static str = "parachain::network-bridge-net"; -/// Send a message to the network. +// Helper function to send a validation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v1 message to peers",); + + send_message(net, peers, PeerSet::Validation, peerset_protocol_names, message, metrics); +} + +// Helper function to send a validation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v2 message to peers",); + + send_message(net, peers, PeerSet::Validation, peerset_protocol_names, message, metrics); +} + +// Helper function to send a collation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message(net, peers, PeerSet::Collation, peerset_protocol_names, message, metrics); +} + +/// Lower level function that sends a message to the network using the main protocol version. /// /// This function is only used internally by the network-bridge, which is responsible to only send /// messages that are compatible with the passed peer set, as that is currently not enforced by /// this function. These are messages of type `WireMessage` parameterized on the matching type. -pub(crate) fn send_message( +fn send_message( net: &mut impl Network, mut peers: Vec, peer_set: PeerSet, - version: ProtocolVersion, protocol_names: &PeerSetProtocolNames, message: M, metrics: &super::Metrics, ) where M: Encode + Clone, { + // Always use main version for sending messages. + let version = peer_set.get_main_version(); let message = { let encoded = message.encode(); metrics.on_notification_sent(peer_set, version, encoded.len(), peers.len()); encoded }; + // optimization: generate the protocol name once. + let protocol_name = protocol_names.get_name(peer_set, version); + gum::trace!( + target: LOG_TARGET, + ?peers, + ?version, + ?protocol_name, + ?message, + "Sending message to peers", + ); + // optimization: avoid cloning the message for the last peer in the // list. The message payload can be quite large. If the underlying // network used `Bytes` this would not be necessary. let last_peer = peers.pop(); - // optimization: generate the protocol name once. - let protocol_name = protocol_names.get_name(peer_set, version); peers.into_iter().for_each(|peer| { net.write_notification(peer, protocol_name.clone(), message.clone()); }); diff --git a/node/network/bridge/src/rx/mod.rs b/node/network/bridge/src/rx/mod.rs index 950bb3d6e6da..1b20fc9a3300 100644 --- a/node/network/bridge/src/rx/mod.rs +++ b/node/network/bridge/src/rx/mod.rs @@ -33,7 +33,8 @@ use polkadot_node_network_protocol::{ CollationVersion, PeerSet, PeerSetProtocolNames, PerPeerSet, ProtocolVersion, ValidationVersion, }, - v1 as protocol_v1, ObservedRole, OurView, PeerId, UnifiedReputationChange as Rep, View, + v1 as protocol_v1, vstaging as protocol_vstaging, ObservedRole, OurView, PeerId, + UnifiedReputationChange as Rep, View, }; use polkadot_node_subsystem::{ @@ -63,9 +64,10 @@ use super::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; - -use crate::network::get_peer_id_by_authority_id; +use crate::network::{ + send_collation_message_v1, send_validation_message_v1, send_validation_message_v2, Network, +}; +use crate::{network::get_peer_id_by_authority_id, WireMessage}; use super::metrics::Metrics; @@ -246,15 +248,28 @@ where ) .await; - send_message( - &mut network_service, - vec![peer], - PeerSet::Validation, - version, - &peerset_protocol_names, - WireMessage::::ViewUpdate(local_view), - &metrics, - ); + match ValidationVersion::try_from(version) + .expect("try_get_protocol has already checked version is known; qed") + { + ValidationVersion::V1 => send_validation_message_v1( + &mut network_service, + vec![peer], + &peerset_protocol_names, + WireMessage::::ViewUpdate( + local_view, + ), + &metrics, + ), + ValidationVersion::VStaging => send_validation_message_v2( + &mut network_service, + vec![peer], + &peerset_protocol_names, + WireMessage::::ViewUpdate( + local_view, + ), + &metrics, + ), + } }, PeerSet::Collation => { dispatch_collation_events_to_all( @@ -271,11 +286,9 @@ where ) .await; - send_message( + send_collation_message_v1( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate(local_view), &metrics, @@ -349,8 +362,16 @@ where .filter_map(|(protocol, msg_bytes)| { // version doesn't matter because we always receive on the 'correct' // protocol name, not the negotiated fallback. - let (peer_set, _version) = + let (peer_set, version) = peerset_protocol_names.try_get_protocol(protocol)?; + gum::trace!( + target: LOG_TARGET, + ?peer_set, + ?protocol, + ?version, + "Received notification" + ); + if peer_set == PeerSet::Validation { if expected_versions[PeerSet::Validation].is_none() { return Some(Err(UNCONNECTED_PEERSET_COST)) @@ -421,7 +442,17 @@ where if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V1.into()) { - handle_v1_peer_messages::( + handle_peer_messages::( + remote, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + v_messages, + &metrics, + ) + } else if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::VStaging.into()) + { + handle_peer_messages::( remote, PeerSet::Validation, &mut shared.0.lock().validation_peers, @@ -435,7 +466,7 @@ where "Major logic bug. Peer somehow has unsupported validation protocol version." ); - never!("Only version 1 is supported; peer set connection checked above; qed"); + never!("Only version 1/2 is supported; peer set connection checked above; qed"); // If a peer somehow triggers this, we'll disconnect them // eventually. @@ -454,7 +485,7 @@ where if expected_versions[PeerSet::Collation] == Some(CollationVersion::V1.into()) { - handle_v1_peer_messages::( + handle_peer_messages::( remote, PeerSet::Collation, &mut shared.0.lock().collation_peers, @@ -738,18 +769,43 @@ fn update_our_view( } ( - shared.validation_peers.keys().cloned().collect::>(), + shared + .validation_peers + .iter() + .map(|(peer_id, peer_data)| (*peer_id, peer_data.version)) + .collect::>(), shared.collation_peers.keys().cloned().collect::>(), ) }; - send_validation_message_v1( - net, - validation_peers, - peerset_protocol_names, - WireMessage::ViewUpdate(new_view.clone()), - metrics, - ); + let filter_by_version = |peers: &[(PeerId, ProtocolVersion)], version| { + peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() + }; + + let v1_validation_peers = + filter_by_version(validation_peers.as_slice(), ValidationVersion::V1.into()); + let vstaging_validation_peers = + filter_by_version(&validation_peers, ValidationVersion::VStaging.into()); + + if v1_validation_peers.len() > 0 { + send_validation_message_v1( + net, + v1_validation_peers, + peerset_protocol_names, + WireMessage::ViewUpdate(new_view.clone()), + metrics, + ); + } + + if vstaging_validation_peers.len() > 0 { + send_validation_message_v2( + net, + vstaging_validation_peers, + peerset_protocol_names, + WireMessage::ViewUpdate(new_view.clone()), + metrics, + ); + } send_collation_message_v1( net, @@ -777,7 +833,7 @@ fn update_our_view( // Handle messages on a specific v1 peer-set. The peer is expected to be connected on that // peer-set. -fn handle_v1_peer_messages>( +fn handle_peer_messages>( peer: PeerId, peer_set: PeerSet, peers: &mut HashMap, @@ -828,42 +884,6 @@ fn handle_v1_peer_messages>( (outgoing_events, reports) } -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, diff --git a/node/network/bridge/src/rx/tests.rs b/node/network/bridge/src/rx/tests.rs index b04884edefaa..f658c2cc3efe 100644 --- a/node/network/bridge/src/rx/tests.rs +++ b/node/network/bridge/src/rx/tests.rs @@ -25,6 +25,7 @@ use parking_lot::Mutex; use std::{ collections::HashSet, sync::atomic::{AtomicBool, Ordering}, + task::Poll, }; use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; @@ -46,7 +47,7 @@ use polkadot_node_subsystem_test_helpers::{ SingleItemSink, SingleItemStream, TestSubsystemContextHandle, }; use polkadot_node_subsystem_util::metered; -use polkadot_primitives::{AuthorityDiscoveryId, Hash}; +use polkadot_primitives::{AuthorityDiscoveryId, CandidateHash, Hash}; use sc_network::Multiaddr; use sp_keyring::Sr25519Keyring; @@ -142,8 +143,7 @@ impl Network for TestNetwork { } fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName) { - let (peer_set, version) = self.protocol_names.try_get_protocol(&protocol).unwrap(); - assert_eq!(version, peer_set.get_main_version()); + let (peer_set, _) = self.protocol_names.try_get_protocol(&protocol).unwrap(); self.action_tx .lock() @@ -152,8 +152,7 @@ impl Network for TestNetwork { } fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - let (peer_set, version) = self.protocol_names.try_get_protocol(&protocol).unwrap(); - assert_eq!(version, peer_set.get_main_version()); + let (peer_set, _) = self.protocol_names.try_get_protocol(&protocol).unwrap(); self.action_tx .lock() @@ -195,10 +194,17 @@ impl TestNetworkHandle { v } - async fn connect_peer(&mut self, peer: PeerId, peer_set: PeerSet, role: ObservedRole) { + async fn connect_peer( + &mut self, + peer: PeerId, + protocol_version: ValidationVersion, + peer_set: PeerSet, + role: ObservedRole, + ) { + let protocol_version = ProtocolVersion::from(protocol_version); self.send_network_event(NetworkEvent::NotificationStreamOpened { remote: peer, - protocol: self.protocol_names.get_main_name(peer_set), + protocol: self.protocol_names.get_name(peer_set, protocol_version), negotiated_fallback: None, role: role.into(), received_handshake: vec![], @@ -432,8 +438,22 @@ fn send_our_view_upon_connection() { handle.await_mode_switch().await; - network_handle.connect_peer(peer, PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer, PeerSet::Collation, ObservedRole::Full).await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) + .await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) + .await; await_peer_connections(&shared, 1, 1).await; @@ -478,10 +498,20 @@ fn sends_view_updates_to_peers() { handle.await_mode_switch().await; network_handle - .connect_peer(peer_a, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_a.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; network_handle - .connect_peer(peer_b, PeerSet::Collation, ObservedRole::Full) + .connect_peer( + peer_b.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) .await; await_peer_connections(&shared, 1, 1).await; @@ -541,10 +571,20 @@ fn do_not_send_view_update_until_synced() { assert_ne!(peer_a, peer_b); network_handle - .connect_peer(peer_a, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_a.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; network_handle - .connect_peer(peer_b, PeerSet::Collation, ObservedRole::Full) + .connect_peer( + peer_b.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) .await; await_peer_connections(&shared, 1, 1).await; @@ -636,10 +676,20 @@ fn do_not_send_view_update_when_only_finalized_block_changed() { let peer_b = PeerId::random(); network_handle - .connect_peer(peer_a, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_a.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; network_handle - .connect_peer(peer_b, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_b.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; await_peer_connections(&shared, 2, 0).await; @@ -696,7 +746,14 @@ fn peer_view_updates_sent_via_overseer() { let peer = PeerId::random(); - network_handle.connect_peer(peer, PeerSet::Validation, ObservedRole::Full).await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) + .await; await_peer_connections(&shared, 1, 0).await; @@ -746,7 +803,14 @@ fn peer_messages_sent_via_overseer() { let peer = PeerId::random(); - network_handle.connect_peer(peer, PeerSet::Validation, ObservedRole::Full).await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) + .await; await_peer_connections(&shared, 1, 0).await; @@ -819,8 +883,22 @@ fn peer_disconnect_from_just_one_peerset() { let peer = PeerId::random(); - network_handle.connect_peer(peer, PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer, PeerSet::Collation, ObservedRole::Full).await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) + .await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) + .await; await_peer_connections(&shared, 1, 1).await; @@ -907,10 +985,20 @@ fn relays_collation_protocol_messages() { let peer_b = PeerId::random(); network_handle - .connect_peer(peer_a, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_a.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; network_handle - .connect_peer(peer_b, PeerSet::Collation, ObservedRole::Full) + .connect_peer( + peer_b.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) .await; await_peer_connections(&shared, 1, 1).await; @@ -1011,8 +1099,22 @@ fn different_views_on_different_peer_sets() { let peer = PeerId::random(); - network_handle.connect_peer(peer, PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer, PeerSet::Collation, ObservedRole::Full).await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) + .await; + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) + .await; await_peer_connections(&shared, 1, 1).await; @@ -1097,7 +1199,12 @@ fn sent_views_include_finalized_number_update() { let peer_a = PeerId::random(); network_handle - .connect_peer(peer_a, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_a.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; await_peer_connections(&shared, 1, 0).await; @@ -1140,7 +1247,12 @@ fn view_finalized_number_can_not_go_down() { let peer_a = PeerId::random(); network_handle - .connect_peer(peer_a, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer_a.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .await; await_peer_connections(&shared, 1, 0).await; @@ -1225,3 +1337,161 @@ fn our_view_updates_decreasing_order_and_limited_to_max() { virtual_overseer }); } + +#[test] +fn network_protocol_versioning_view_update() { + let (oracle, handle) = make_sync_oracle(false); + test_harness(Box::new(oracle), |test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer, .. } = test_harness; + + let peer_ids: Vec<_> = (0..2).map(|_| PeerId::random()).collect(); + let peers = [ + (peer_ids[0], PeerSet::Validation, ValidationVersion::VStaging), + (peer_ids[1], PeerSet::Validation, ValidationVersion::V1), + ]; + + let head = Hash::repeat_byte(1); + virtual_overseer + .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: head, + number: 1, + status: LeafStatus::Fresh, + span: Arc::new(jaeger::Span::Disabled), + }), + ))) + .await; + + handle.await_mode_switch().await; + + for &(peer_id, peer_set, version) in &peers { + network_handle + .connect_peer(peer_id, version, peer_set, ObservedRole::Full) + .await; + } + + let view = view![head]; + let actions = network_handle.next_network_actions(2).await; + + for &(peer_id, peer_set, version) in &peers { + let wire_msg = match version { + ValidationVersion::V1 => + WireMessage::::ViewUpdate(view.clone()) + .encode(), + ValidationVersion::VStaging => + WireMessage::::ViewUpdate(view.clone()) + .encode(), + }; + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification(peer_id, peer_set, wire_msg), + ); + } + + virtual_overseer + }); +} + +#[test] +fn network_protocol_versioning_subsystem_msg() { + let (oracle, _handle) = make_sync_oracle(false); + test_harness(Box::new(oracle), |test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer, .. } = test_harness; + + let peer = PeerId::random(); + + network_handle + .connect_peer( + peer.clone(), + ValidationVersion::VStaging, + PeerSet::Validation, + ObservedRole::Full, + ) + .await; + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected( + peer.clone(), + ObservedRole::Full, + ValidationVersion::VStaging.into(), + None, + ), + &mut virtual_overseer, + ) + .await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ) + .await; + } + + let approval_distribution_message = + protocol_vstaging::ApprovalDistributionMessage::Approvals(Vec::new()); + + let msg = protocol_vstaging::ValidationProtocol::ApprovalDistribution( + approval_distribution_message.clone(), + ); + + network_handle + .peer_message( + peer.clone(), + PeerSet::Validation, + WireMessage::ProtocolMessage(msg.clone()).encode(), + ) + .await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ApprovalDistribution( + ApprovalDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage(p, Versioned::VStaging(m)) + ) + ) => { + assert_eq!(p, peer); + assert_eq!(m, approval_distribution_message); + } + ); + + let metadata = protocol_v1::StatementMetadata { + relay_parent: Hash::zero(), + candidate_hash: CandidateHash::default(), + signed_by: ValidatorIndex(0), + signature: sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64]), + }; + let statement_distribution_message = + protocol_vstaging::StatementDistributionMessage::LargeStatement(metadata); + + let msg = protocol_vstaging::ValidationProtocol::StatementDistribution( + statement_distribution_message.clone(), + ); + + network_handle + .peer_message( + peer.clone(), + PeerSet::Validation, + WireMessage::ProtocolMessage(msg.clone()).encode(), + ) + .await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage(p, Versioned::VStaging(m)) + ) + ) => { + assert_eq!(p, peer); + assert_eq!(m, statement_distribution_message); + } + ); + + // No more messages. + assert_matches!(futures::poll!(virtual_overseer.recv().boxed()), Poll::Pending); + + virtual_overseer + }); +} diff --git a/node/network/bridge/src/tx/mod.rs b/node/network/bridge/src/tx/mod.rs index 93916dd70fec..4011932f1f52 100644 --- a/node/network/bridge/src/tx/mod.rs +++ b/node/network/bridge/src/tx/mod.rs @@ -18,9 +18,7 @@ use super::*; use polkadot_node_network_protocol::{ - peer_set::{CollationVersion, PeerSet, PeerSetProtocolNames, ValidationVersion}, - request_response::ReqProtocolNames, - v1 as protocol_v1, PeerId, Versioned, + peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames, Versioned, }; use polkadot_node_subsystem::{ @@ -40,7 +38,9 @@ use crate::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; +use crate::network::{ + send_collation_message_v1, send_validation_message_v1, send_validation_message_v2, Network, +}; use crate::metrics::Metrics; @@ -186,6 +186,7 @@ where gum::trace!( target: LOG_TARGET, action = "SendValidationMessages", + ?msg, num_messages = 1usize, ); @@ -197,6 +198,13 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), + Versioned::VStaging(msg) => send_validation_message_v2( + &mut network_service, + peers, + peerset_protocol_names, + WireMessage::ProtocolMessage(msg), + &metrics, + ), } }, NetworkBridgeTxMessage::SendValidationMessages(msgs) => { @@ -204,6 +212,7 @@ where target: LOG_TARGET, action = "SendValidationMessages", num_messages = %msgs.len(), + ?msgs, ); for (peers, msg) in msgs { @@ -215,6 +224,13 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), + Versioned::VStaging(msg) => send_validation_message_v2( + &mut network_service, + peers, + peerset_protocol_names, + WireMessage::ProtocolMessage(msg), + &metrics, + ), } } }, @@ -233,6 +249,14 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), + _ => { + gum::warn!( + target: LOG_TARGET, + action = "SendCollationMessages", + num_messages = 1usize, + "Attempted to send collation message on invalid protocol version. Only v1 supported." + ); + }, } }, NetworkBridgeTxMessage::SendCollationMessages(msgs) => { @@ -251,6 +275,12 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), + _ => gum::warn!( + target: LOG_TARGET, + action = "SendCollationMessages", + num_messages = 1usize, + "Attempted to send collation message on invalid protocol version" + ), } } }, @@ -345,39 +375,3 @@ where Ok(()) } - -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} diff --git a/node/network/bridge/src/tx/tests.rs b/node/network/bridge/src/tx/tests.rs index e851cb0dee6f..79f5a94ef77f 100644 --- a/node/network/bridge/src/tx/tests.rs +++ b/node/network/bridge/src/tx/tests.rs @@ -25,9 +25,9 @@ use std::collections::HashSet; use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSetProtocolNames, ValidationVersion}, request_response::{outgoing::Requests, ReqProtocolNames}, - ObservedRole, Versioned, + v1 as protocol_v1, vstaging as protocol_vstaging, ObservedRole, Versioned, }; use polkadot_node_subsystem::{FromOrchestra, OverseerSignal}; use polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; @@ -130,8 +130,7 @@ impl Network for TestNetwork { } fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName) { - let (peer_set, version) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); - assert_eq!(version, peer_set.get_main_version()); + let (peer_set, _) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); self.action_tx .lock() @@ -140,8 +139,7 @@ impl Network for TestNetwork { } fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - let (peer_set, version) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); - assert_eq!(version, peer_set.get_main_version()); + let (peer_set, _) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); self.action_tx .lock() @@ -173,10 +171,17 @@ impl TestNetworkHandle { self.action_rx.next().await.expect("subsystem concluded early") } - async fn connect_peer(&mut self, peer: PeerId, peer_set: PeerSet, role: ObservedRole) { + async fn connect_peer( + &mut self, + peer: PeerId, + protocol_version: ValidationVersion, + peer_set: PeerSet, + role: ObservedRole, + ) { + let protocol_version = ProtocolVersion::from(protocol_version); self.send_network_event(NetworkEvent::NotificationStreamOpened { remote: peer, - protocol: self.peerset_protocol_names.get_main_name(peer_set), + protocol: self.peerset_protocol_names.get_name(peer_set, protocol_version), negotiated_fallback: None, role: role.into(), received_handshake: vec![], @@ -242,7 +247,12 @@ fn send_messages_to_peers() { let peer = PeerId::random(); network_handle - .connect_peer(peer, PeerSet::Validation, ObservedRole::Full) + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Validation, + ObservedRole::Full, + ) .timeout(TIMEOUT) .await .expect("Timeout does not occur"); @@ -251,7 +261,12 @@ fn send_messages_to_peers() { // so the single item sink has to be free explicitly network_handle - .connect_peer(peer, PeerSet::Collation, ObservedRole::Full) + .connect_peer( + peer.clone(), + ValidationVersion::V1, + PeerSet::Collation, + ObservedRole::Full, + ) .timeout(TIMEOUT) .await .expect("Timeout does not occur"); @@ -328,3 +343,64 @@ fn send_messages_to_peers() { virtual_overseer }); } + +#[test] +fn network_protocol_versioning_send() { + test_harness(|test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; + + let peer_ids: Vec<_> = (0..2).map(|_| PeerId::random()).collect(); + let peers = [ + (peer_ids[0], PeerSet::Validation, ValidationVersion::VStaging), + (peer_ids[1], PeerSet::Validation, ValidationVersion::V1), + ]; + + for &(peer_id, peer_set, version) in &peers { + network_handle + .connect_peer(peer_id, version, peer_set, ObservedRole::Full) + .timeout(TIMEOUT) + .await + .expect("Timeout does not occur"); + } + + // send a validation protocol message. + { + let approval_distribution_message = + protocol_vstaging::ApprovalDistributionMessage::Approvals(Vec::new()); + + let msg = protocol_vstaging::ValidationProtocol::ApprovalDistribution( + approval_distribution_message.clone(), + ); + + // Note that bridge doesn't ensure neither peer's protocol version + // or peer set match the message. + let receivers = vec![peer_ids[0], peer_ids[1]]; + virtual_overseer + .send(FromOrchestra::Communication { + msg: NetworkBridgeTxMessage::SendValidationMessage( + receivers.clone(), + Versioned::VStaging(msg.clone()), + ), + }) + .timeout(TIMEOUT) + .await + .expect("Timeout does not occur"); + + for peer in &receivers { + assert_eq!( + network_handle + .next_network_action() + .timeout(TIMEOUT) + .await + .expect("Timeout does not occur"), + NetworkAction::WriteNotification( + *peer, + PeerSet::Validation, + WireMessage::ProtocolMessage(msg.clone()).encode(), + ) + ); + } + } + virtual_overseer + }); +} diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index e4adfdc9d941..5bb4ddf7ebb4 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -915,6 +915,10 @@ async fn handle_network_msg( NewGossipTopology { .. } => { // impossible! }, + PeerMessage(_, Versioned::VStaging(_)) => gum::warn!( + target: LOG_TARGET, + "Received message on invalid collator protocol version. Only v1 supported", + ), } Ok(()) diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index f87a14971e8a..67a7560020ee 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -1132,6 +1132,10 @@ async fn handle_network_msg( PeerMessage(remote, Versioned::V1(msg)) => { process_incoming_peer_message(ctx, state, remote, msg).await; }, + PeerMessage(_, Versioned::VStaging(_)) => gum::warn!( + target: LOG_TARGET, + "Received message on invalid collator protocol version. Only v1 supported", + ), UpdatedAuthorityIds { .. } => { // The validator side doesn't deal with `AuthorityDiscoveryId`s. }, diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index b92aa4e9fe39..d00fb4bc292c 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -40,8 +40,7 @@ use sp_application_crypto::{AppCrypto, ByteArray}; use sp_keystore::{Keystore, KeystorePtr}; use polkadot_node_network_protocol::{ - authority_discovery::AuthorityDiscovery, peer_set::PeerSet, GossipSupportNetworkMessage, - PeerId, Versioned, + authority_discovery::AuthorityDiscovery, peer_set::PeerSet, GossipSupportNetworkMessage, PeerId, }; use polkadot_node_subsystem::{ messages::{ @@ -448,9 +447,7 @@ where NetworkBridgeEvent::OurViewChange(_) => {}, NetworkBridgeEvent::PeerViewChange(_, _) => {}, NetworkBridgeEvent::NewGossipTopology { .. } => {}, - NetworkBridgeEvent::PeerMessage(_, Versioned::V1(v)) => { - match v {}; - }, + NetworkBridgeEvent::PeerMessage(_, _) => {}, } } diff --git a/node/network/protocol/Cargo.toml b/node/network/protocol/Cargo.toml index cfb7a5c2d0f0..30a67de841bc 100644 --- a/node/network/protocol/Cargo.toml +++ b/node/network/protocol/Cargo.toml @@ -26,3 +26,6 @@ gum = { package = "tracing-gum", path = "../../gum" } [dev-dependencies] rand_chacha = "0.3.1" + +[features] +network-protocol-staging = [] diff --git a/node/network/protocol/src/grid_topology.rs b/node/network/protocol/src/grid_topology.rs index 99dd513c4d79..8bd9adbc17c1 100644 --- a/node/network/protocol/src/grid_topology.rs +++ b/node/network/protocol/src/grid_topology.rs @@ -73,12 +73,20 @@ pub struct SessionGridTopology { shuffled_indices: Vec, /// The canonical shuffling of validators for the session. canonical_shuffling: Vec, + /// The list of peer-ids in an efficient way to search. + peer_ids: HashSet, } impl SessionGridTopology { /// Create a new session grid topology. pub fn new(shuffled_indices: Vec, canonical_shuffling: Vec) -> Self { - SessionGridTopology { shuffled_indices, canonical_shuffling } + let mut peer_ids = HashSet::new(); + for peer_info in canonical_shuffling.iter() { + for peer_id in peer_info.peer_ids.iter() { + peer_ids.insert(*peer_id); + } + } + SessionGridTopology { shuffled_indices, canonical_shuffling, peer_ids } } /// Produces the outgoing routing logic for a particular peer. @@ -111,6 +119,11 @@ impl SessionGridTopology { Some(grid_subset) } + + /// Tells if a given peer id is validator in a session + pub fn is_validator(&self, peer: &PeerId) -> bool { + self.peer_ids.contains(peer) + } } struct MatrixNeighbors { @@ -273,6 +286,11 @@ impl SessionGridTopologyEntry { pub fn get(&self) -> &SessionGridTopology { &self.topology } + + /// Tells if a given peer id is validator in a session + pub fn is_validator(&self, peer: &PeerId) -> bool { + self.topology.is_validator(peer) + } } /// A set of topologies indexed by session @@ -347,6 +365,7 @@ impl Default for SessionBoundGridTopologyStorage { topology: SessionGridTopology { shuffled_indices: Vec::new(), canonical_shuffling: Vec::new(), + peer_ids: Default::default(), }, local_neighbors: GridNeighbors::empty(), }, diff --git a/node/network/protocol/src/lib.rs b/node/network/protocol/src/lib.rs index 2df926ac55d8..3c9cd321d981 100644 --- a/node/network/protocol/src/lib.rs +++ b/node/network/protocol/src/lib.rs @@ -253,22 +253,26 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), + /// VStaging type. + VStaging(VStaging), } -impl Versioned<&'_ V1> { +impl Versioned<&'_ V1, &'_ VStaging> { /// Convert to a fully-owned version of the message. - pub fn clone_inner(&self) -> Versioned { + pub fn clone_inner(&self) -> Versioned { match *self { Versioned::V1(inner) => Versioned::V1(inner.clone()), + Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()), } } } /// All supported versions of the validation protocol message. -pub type VersionedValidationProtocol = Versioned; +pub type VersionedValidationProtocol = + Versioned; impl From for VersionedValidationProtocol { fn from(v1: v1::ValidationProtocol) -> Self { @@ -276,8 +280,14 @@ impl From for VersionedValidationProtocol { } } +impl From for VersionedValidationProtocol { + fn from(vstaging: vstaging::ValidationProtocol) -> Self { + VersionedValidationProtocol::VStaging(vstaging) + } +} + /// All supported versions of the collation protocol message. -pub type VersionedCollationProtocol = Versioned; +pub type VersionedCollationProtocol = Versioned; impl From for VersionedCollationProtocol { fn from(v1: v1::CollationProtocol) -> Self { @@ -291,16 +301,21 @@ macro_rules! impl_versioned_full_protocol_from { fn from(versioned_from: $from) -> $out { match versioned_from { Versioned::V1(x) => Versioned::V1(x.into()), + Versioned::VStaging(x) => Versioned::VStaging(x.into()), } } } }; } - /// Implement `TryFrom` for one versioned enum variant into the inner type. /// `$m_ty::$variant(inner) -> Ok(inner)` macro_rules! impl_versioned_try_from { - ($from:ty, $out:ty, $v1_pat:pat => $v1_out:expr) => { + ( + $from:ty, + $out:ty, + $v1_pat:pat => $v1_out:expr, + $vstaging_pat:pat => $vstaging_out:expr + ) => { impl TryFrom<$from> for $out { type Error = crate::WrongVariant; @@ -308,6 +323,7 @@ macro_rules! impl_versioned_try_from { #[allow(unreachable_patterns)] // when there is only one variant match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)), + Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)), _ => Err(crate::WrongVariant), } } @@ -320,6 +336,8 @@ macro_rules! impl_versioned_try_from { #[allow(unreachable_patterns)] // when there is only one variant match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), + Versioned::VStaging($vstaging_pat) => + Ok(Versioned::VStaging($vstaging_out.clone())), _ => Err(crate::WrongVariant), } } @@ -328,7 +346,8 @@ macro_rules! impl_versioned_try_from { } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type BitfieldDistributionMessage = Versioned; +pub type BitfieldDistributionMessage = + Versioned; impl_versioned_full_protocol_from!( BitfieldDistributionMessage, VersionedValidationProtocol, @@ -337,11 +356,13 @@ impl_versioned_full_protocol_from!( impl_versioned_try_from!( VersionedValidationProtocol, BitfieldDistributionMessage, - v1::ValidationProtocol::BitfieldDistribution(x) => x + v1::ValidationProtocol::BitfieldDistribution(x) => x, + vstaging::ValidationProtocol::BitfieldDistribution(x) => x ); /// Version-annotated messages used by the statement distribution subsystem. -pub type StatementDistributionMessage = Versioned; +pub type StatementDistributionMessage = + Versioned; impl_versioned_full_protocol_from!( StatementDistributionMessage, VersionedValidationProtocol, @@ -350,11 +371,13 @@ impl_versioned_full_protocol_from!( impl_versioned_try_from!( VersionedValidationProtocol, StatementDistributionMessage, - v1::ValidationProtocol::StatementDistribution(x) => x + v1::ValidationProtocol::StatementDistribution(x) => x, + vstaging::ValidationProtocol::StatementDistribution(x) => x ); /// Version-annotated messages used by the approval distribution subsystem. -pub type ApprovalDistributionMessage = Versioned; +pub type ApprovalDistributionMessage = + Versioned; impl_versioned_full_protocol_from!( ApprovalDistributionMessage, VersionedValidationProtocol, @@ -363,11 +386,14 @@ impl_versioned_full_protocol_from!( impl_versioned_try_from!( VersionedValidationProtocol, ApprovalDistributionMessage, - v1::ValidationProtocol::ApprovalDistribution(x) => x + v1::ValidationProtocol::ApprovalDistribution(x) => x, + vstaging::ValidationProtocol::ApprovalDistribution(x) => x + ); /// Version-annotated messages used by the gossip-support subsystem (this is void). -pub type GossipSupportNetworkMessage = Versioned; +pub type GossipSupportNetworkMessage = + Versioned; // This is a void enum placeholder, so never gets sent over the wire. impl TryFrom for GossipSupportNetworkMessage { type Error = WrongVariant; @@ -384,7 +410,8 @@ impl<'a> TryFrom<&'a VersionedValidationProtocol> for GossipSupportNetworkMessag } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type CollatorProtocolMessage = Versioned; +pub type CollatorProtocolMessage = + Versioned; impl_versioned_full_protocol_from!( CollatorProtocolMessage, VersionedCollationProtocol, @@ -393,9 +420,59 @@ impl_versioned_full_protocol_from!( impl_versioned_try_from!( VersionedCollationProtocol, CollatorProtocolMessage, - v1::CollationProtocol::CollatorProtocol(x) => x + v1::CollationProtocol::CollatorProtocol(x) => x, + vstaging::CollationProtocol::CollatorProtocol(x) => x ); +/// A staging version of the validation/collator protocol. +/// Changes: +/// - assignment cert type changed, see `IndirectAssignmentCertV2`. +pub mod vstaging { + use parity_scale_codec::{Decode, Encode}; + use polkadot_node_primitives::approval::v2::{ + CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, + }; + + // Re-export stuff that has not changed since v1. + pub use crate::v1::{ + declare_signature_payload, BitfieldDistributionMessage, CollationProtocol, + CollatorProtocolMessage, GossipSupportNetworkMessage, StatementDistributionMessage, + StatementMetadata, + }; + + /// All network messages on the validation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum ValidationProtocol { + /// Bitfield distribution messages + #[codec(index = 0)] + #[from] + BitfieldDistribution(BitfieldDistributionMessage), + /// Statement distribution messages + #[codec(index = 1)] + #[from] + StatementDistribution(StatementDistributionMessage), + /// Approval distribution messages + #[codec(index = 2)] + #[from] + ApprovalDistribution(ApprovalDistributionMessage), + } + + /// Network messages used by the approval distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum ApprovalDistributionMessage { + /// Assignments for candidates in recent, unfinalized blocks. + /// We use a bitfield to reference claimed candidates, where the bit index is equal to candidate index. + /// + /// Actually checking the assignment may yield a different result. + /// TODO: Look at getting rid of bitfield in the future. + #[codec(index = 0)] + Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), + /// Approvals for candidates in some recent, unfinalized block. + #[codec(index = 1)] + Approvals(Vec), + } +} + /// v1 notification protocol types. pub mod v1 { use parity_scale_codec::{Decode, Encode}; @@ -406,7 +483,7 @@ pub mod v1 { }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::v1::{IndirectAssignmentCert, IndirectSignedApprovalVote}, UncheckedSignedFullStatement, }; @@ -553,3 +630,11 @@ pub mod v1 { payload } } + +/// Returns the subset of `peers` with the specified `version`. +pub fn filter_by_peer_version( + peers: &[(PeerId, peer_set::ProtocolVersion)], + version: peer_set::ProtocolVersion, +) -> Vec { + peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() +} diff --git a/node/network/protocol/src/peer_set.rs b/node/network/protocol/src/peer_set.rs index b9fa80d5c4a2..2e99f0d6a38c 100644 --- a/node/network/protocol/src/peer_set.rs +++ b/node/network/protocol/src/peer_set.rs @@ -118,8 +118,15 @@ impl PeerSet { /// Networking layer relies on `get_main_version()` being the version /// of the main protocol name reported by [`PeerSetProtocolNames::get_main_name()`]. pub fn get_main_version(self) -> ProtocolVersion { + #[cfg(not(feature = "network-protocol-staging"))] match self { - PeerSet::Validation => ValidationVersion::V1.into(), + PeerSet::Validation => ValidationVersion::VStaging.into(), + PeerSet::Collation => CollationVersion::V1.into(), + } + + #[cfg(feature = "network-protocol-staging")] + match self { + PeerSet::Validation => ValidationVersion::VStaging.into(), PeerSet::Collation => CollationVersion::V1.into(), } } @@ -145,6 +152,8 @@ impl PeerSet { PeerSet::Validation => if version == ValidationVersion::V1.into() { Some("validation/1") + } else if version == ValidationVersion::VStaging.into() { + Some("validation/2") } else { None }, @@ -212,6 +221,8 @@ impl From for u32 { pub enum ValidationVersion { /// The first version. V1 = 1, + /// The staging version adds `AssignmentsV2` message to approval distribution. + VStaging = 2, } /// Supported collation protocol versions. Only versions defined here must be used in the codebase. @@ -221,6 +232,38 @@ pub enum CollationVersion { V1 = 1, } +/// Marker indicating the version is unknown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnknownVersion; + +impl TryFrom for ValidationVersion { + type Error = UnknownVersion; + + fn try_from(p: ProtocolVersion) -> Result { + for v in Self::iter() { + if v as u32 == p.0 { + return Ok(v) + } + } + + Err(UnknownVersion) + } +} + +impl TryFrom for CollationVersion { + type Error = UnknownVersion; + + fn try_from(p: ProtocolVersion) -> Result { + for v in Self::iter() { + if v as u32 == p.0 { + return Ok(v) + } + } + + Err(UnknownVersion) + } +} + impl From for ProtocolVersion { fn from(version: ValidationVersion) -> ProtocolVersion { ProtocolVersion(version as u32) diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 4cdf0d8af467..a514c0b41997 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -26,12 +26,13 @@ use error::{log_error, FatalResult, JfyiErrorResult}; use parity_scale_codec::Encode; use polkadot_node_network_protocol::{ - self as net_protocol, + self as net_protocol, filter_by_peer_version, grid_topology::{GridNeighbors, RequiredRouting, SessionBoundGridTopologyStorage}, - peer_set::{IsAuthority, PeerSet}, + peer_set::{IsAuthority, PeerSet, ProtocolVersion, ValidationVersion}, request_response::{v1 as request_v1, IncomingRequestReceiver}, v1::{self as protocol_v1, StatementMetadata}, - IfDisconnected, PeerId, UnifiedReputationChange as Rep, Versioned, View, + vstaging as protocol_vstaging, IfDisconnected, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{SignedFullStatement, Statement, UncheckedSignedFullStatement}; use polkadot_node_subsystem_util::{self as util, rand, MIN_GOSSIP_PEERS}; @@ -453,6 +454,8 @@ struct PeerData { view_knowledge: HashMap, /// Peer might be known as authority with the given ids. maybe_authority: Option>, + /// Protocol version + version: ProtocolVersion, } impl PeerData { @@ -974,24 +977,47 @@ fn statement_message( relay_parent: Hash, statement: SignedFullStatement, metrics: &Metrics, + protocol_version: ProtocolVersion, ) -> net_protocol::VersionedValidationProtocol { let (is_large, size) = is_statement_large(&statement); if let Some(size) = size { metrics.on_created_message(size); } - let msg = if is_large { - protocol_v1::StatementDistributionMessage::LargeStatement(StatementMetadata { - relay_parent, - candidate_hash: statement.payload().candidate_hash(), - signed_by: statement.validator_index(), - signature: statement.signature().clone(), - }) - } else { - protocol_v1::StatementDistributionMessage::Statement(relay_parent, statement.into()) - }; + match ValidationVersion::try_from(protocol_version) { + Ok(ValidationVersion::V1) => { + let msg = if is_large { + protocol_v1::StatementDistributionMessage::LargeStatement(StatementMetadata { + relay_parent, + candidate_hash: statement.payload().candidate_hash(), + signed_by: statement.validator_index(), + signature: statement.signature().clone(), + }) + } else { + protocol_v1::StatementDistributionMessage::Statement(relay_parent, statement.into()) + }; + + protocol_v1::ValidationProtocol::StatementDistribution(msg).into() + }, + Ok(ValidationVersion::VStaging) => { + let msg = if is_large { + protocol_vstaging::StatementDistributionMessage::LargeStatement(StatementMetadata { + relay_parent, + candidate_hash: statement.payload().candidate_hash(), + signed_by: statement.validator_index(), + signature: statement.signature().clone(), + }) + } else { + protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.into(), + ) + }; - protocol_v1::ValidationProtocol::StatementDistribution(msg).into() + protocol_vstaging::ValidationProtocol::StatementDistribution(msg).into() + }, + Err(_) => unreachable!("Invalid peer protocol"), + } } /// Check whether a statement should be treated as large statement. @@ -1076,20 +1102,19 @@ async fn circulate_statement<'a, Context>( peers_to_send.len() == peers_to_send.clone().into_iter().collect::>().len(), "We filter out duplicates above. qed.", ); - let peers_to_send: Vec<(PeerId, bool)> = peers_to_send + let peers_to_send: Vec<(PeerId, bool, ProtocolVersion)> = peers_to_send .into_iter() .map(|peer_id| { - let new = peers - .get_mut(&peer_id) - .expect("a subset is taken above, so it exists; qed") - .send(&relay_parent, &fingerprint); - (peer_id, new) + let peer_data = + peers.get_mut(&peer_id).expect("a subset is taken above, so it exists; qed"); + + let new = peer_data.send(&relay_parent, &fingerprint); + (peer_id, new, peer_data.version) }) .collect(); // Send all these peers the initial statement. if !peers_to_send.is_empty() { - let payload = statement_message(relay_parent, stored.statement.clone(), metrics); gum::trace!( target: LOG_TARGET, ?peers_to_send, @@ -1097,16 +1122,38 @@ async fn circulate_statement<'a, Context>( statement = ?stored.statement, "Sending statement", ); - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - peers_to_send.iter().map(|(p, _)| *p).collect(), - payload, - )) - .await; + + let peers_to_send = + peers_to_send.iter().map(|(p, _, version)| (*p, *version)).collect::>(); + let v1_peers = filter_by_peer_version(&peers_to_send, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(&peers_to_send, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + let payload = statement_message( + relay_parent, + stored.statement.clone(), + metrics, + ValidationVersion::V1.into(), + ); + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage(v1_peers, payload)) + .await; + } + + if !v2_peers.is_empty() { + let payload = statement_message( + relay_parent, + stored.statement.clone(), + metrics, + ValidationVersion::VStaging.into(), + ); + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage(v2_peers, payload)) + .await; + } } peers_to_send .into_iter() - .filter_map(|(peer, needs_dependent)| if needs_dependent { Some(peer) } else { None }) + .filter_map(|(peer, needs_dependent, _)| if needs_dependent { Some(peer) } else { None }) .collect() } @@ -1127,7 +1174,12 @@ async fn send_statements_about( continue } peer_data.send(&relay_parent, &fingerprint); - let payload = statement_message(relay_parent, statement.statement.clone(), metrics); + let payload = statement_message( + relay_parent, + statement.statement.clone(), + metrics, + peer_data.version, + ); gum::trace!( target: LOG_TARGET, @@ -1160,7 +1212,12 @@ async fn send_statements( continue } peer_data.send(&relay_parent, &fingerprint); - let payload = statement_message(relay_parent, statement.statement.clone(), metrics); + let payload = statement_message( + relay_parent, + statement.statement.clone(), + metrics, + peer_data.version, + ); gum::trace!( target: LOG_TARGET, @@ -1661,7 +1718,7 @@ async fn handle_network_update( R: rand::Rng, { match update { - NetworkBridgeEvent::PeerConnected(peer, role, _, maybe_authority) => { + NetworkBridgeEvent::PeerConnected(peer, role, version, maybe_authority) => { gum::trace!(target: LOG_TARGET, ?peer, ?role, "Peer connected"); peers.insert( peer, @@ -1669,6 +1726,7 @@ async fn handle_network_update( view: Default::default(), view_knowledge: Default::default(), maybe_authority: maybe_authority.clone(), + version, }, ); if let Some(authority_ids) = maybe_authority { @@ -1716,6 +1774,23 @@ async fn handle_network_update( } } }, + NetworkBridgeEvent::PeerMessage(peer, Versioned::VStaging(message)) => { + handle_incoming_message_and_circulate( + peer, + topology_storage, + peers, + active_heads, + recent_outdated_heads, + ctx, + message, + req_sender, + metrics, + runtime, + rng, + reputation, + ) + .await; + }, NetworkBridgeEvent::PeerMessage(peer, Versioned::V1(message)) => { handle_incoming_message_and_circulate( peer, diff --git a/node/network/statement-distribution/src/tests.rs b/node/network/statement-distribution/src/tests.rs index affed80fce30..02c1b698ee57 100644 --- a/node/network/statement-distribution/src/tests.rs +++ b/node/network/statement-distribution/src/tests.rs @@ -505,6 +505,7 @@ fn peer_view_update_sends_messages() { k }, maybe_authority: None, + version: ValidationVersion::V1.into(), }; let pool = sp_core::testing::TaskExecutor::new(); @@ -553,9 +554,13 @@ fn peer_view_update_sends_messages() { // it will not change between runs of the program. for statement in active_head.statements_about(candidate_hash) { let message = handle.recv().await; - let expected_to = vec![peer]; - let expected_payload = - statement_message(hash_c, statement.statement.clone(), &Metrics::default()); + let expected_to = vec![peer.clone()]; + let expected_payload = statement_message( + hash_c, + statement.statement.clone(), + &Metrics::default(), + ValidationVersion::V1.into(), + ); assert_matches!( message, @@ -598,6 +603,7 @@ fn circulated_statement_goes_to_all_peers_with_view() { view: view.clone(), view_knowledge: view.iter().map(|v| (*v, Default::default())).collect(), maybe_authority: None, + version: ValidationVersion::V1.into(), }; let mut peer_data: HashMap<_, _> = vec![ @@ -697,7 +703,7 @@ fn circulated_statement_goes_to_all_peers_with_view() { assert_eq!( payload, - statement_message(hash_b, statement.statement.clone(), &Metrics::default()), + statement_message(hash_b, statement.statement.clone(), &Metrics::default(), ValidationVersion::V1.into()), ); } ) diff --git a/node/primitives/Cargo.toml b/node/primitives/Cargo.toml index fbd85c1a0551..a8381916ffd6 100644 --- a/node/primitives/Cargo.toml +++ b/node/primitives/Cargo.toml @@ -21,6 +21,7 @@ polkadot-parachain = { path = "../../parachain", default-features = false } schnorrkel = "0.9.1" thiserror = "1.0.31" serde = { version = "1.0.163", features = ["derive"] } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] zstd = { version = "0.11.2", default-features = false } diff --git a/node/primitives/src/approval.rs b/node/primitives/src/approval.rs index 01f45a207874..08a515580738 100644 --- a/node/primitives/src/approval.rs +++ b/node/primitives/src/approval.rs @@ -16,190 +16,570 @@ //! Types relevant for approval. -pub use sp_consensus_babe::{Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript}; - -use parity_scale_codec::{Decode, Encode}; -use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, - ValidatorIndex, ValidatorSignature, -}; -use sp_application_crypto::ByteArray; -use sp_consensus_babe as babe_primitives; - -/// Validators assigning to check a particular candidate are split up into tranches. -/// Earlier tranches of validators check first, with later tranches serving as backup. -pub type DelayTranche = u32; - -/// A static context used to compute the Relay VRF story based on the -/// VRF output included in the header-chain. -pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; - -/// A static context used for transcripts indicating assigned availability core. -pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; - -/// A static context associated with producing randomness for a core. -pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; - -/// A static context associated with producing randomness for a tranche. -pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; - -/// random bytes derived from the VRF submitted within the block by the -/// block author as a credential and used as input to approval assignment criteria. -#[derive(Debug, Clone, Encode, Decode, PartialEq)] -pub struct RelayVRFStory(pub [u8; 32]); - -/// Different kinds of input data or criteria that can prove a validator's assignment -/// to check a particular parachain. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub enum AssignmentCertKind { - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with a sample number. +/// A list of primitives introduced in v1. +pub mod v1 { + use sp_consensus_babe as babe_primitives; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + + use parity_scale_codec::{Decode, Encode}; + use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, + ValidatorIndex, ValidatorSignature, + }; + use sp_application_crypto::ByteArray; + + /// Validators assigning to check a particular candidate are split up into tranches. + /// Earlier tranches of validators check first, with later tranches serving as backup. + pub type DelayTranche = u32; + + /// A static context used to compute the Relay VRF story based on the + /// VRF output included in the header-chain. + pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; + + /// A static context used for transcripts indicating assigned availability core. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; + + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; + + /// A static context associated with producing randomness for a tranche. + pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; + + /// random bytes derived from the VRF submitted within the block by the + /// block author as a credential and used as input to approval assignment criteria. + #[derive(Debug, Clone, Encode, Decode, PartialEq)] + pub struct RelayVRFStory(pub [u8; 32]); + + /// Different kinds of input data or criteria that can prove a validator's assignment + /// to check a particular parachain. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKind { + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCert { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKind, + /// The VRF signature showing the criterion is met. + pub vrf: VrfSignature, + } + + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCert { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCert, + } + + /// A signed approval vote which references the candidate indirectly via the block. /// - /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] - RelayVRFModulo { - /// The sample number used in this cert. - sample: u32, - }, - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with the index of a particular core. + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVote { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_index: CandidateIndex, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } + + /// Metadata about a block which is now live in the approval protocol. + #[derive(Debug)] + pub struct BlockApprovalMeta { + /// The hash of the block. + pub hash: Hash, + /// The number of the block. + pub number: BlockNumber, + /// The hash of the parent block. + pub parent_hash: Hash, + /// The candidates included by the block. + /// Note that these are not the same as the candidates that appear within the block body. + pub candidates: Vec, + /// The consensus slot of the block. + pub slot: Slot, + /// The session of the block. + pub session: SessionIndex, + } + + /// Errors that can occur during the approvals protocol. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum ApprovalError { + #[error("Schnorrkel signature error")] + SchnorrkelSignature(schnorrkel::errors::SignatureError), + #[error("Authority index {0} out of bounds")] + AuthorityOutOfBounds(usize), + } + + /// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. + pub struct UnsafeVRFOutput { + vrf_output: VrfOutput, + slot: Slot, + authority_index: u32, + } + + impl UnsafeVRFOutput { + /// Get the slot. + pub fn slot(&self) -> Slot { + self.slot + } + + /// Compute the randomness associated with this VRF output. + pub fn compute_randomness( + self, + authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], + randomness: &babe_primitives::Randomness, + epoch_index: u64, + ) -> Result { + let author = match authorities.get(self.authority_index as usize) { + None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), + Some(x) => &x.0, + }; + + let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) + .map_err(ApprovalError::SchnorrkelSignature)?; + + let transcript = + sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + + let inout = self + .vrf_output + .0 + .attach_input_hash(&pubkey, transcript.0) + .map_err(ApprovalError::SchnorrkelSignature)?; + Ok(RelayVRFStory(inout.make_bytes(super::v1::RELAY_VRF_STORY_CONTEXT))) + } + } + + /// Extract the slot number and relay VRF from a header. /// - /// The context is [`RELAY_VRF_DELAY_CONTEXT`] - RelayVRFDelay { - /// The core index chosen in this cert. - core_index: CoreIndex, - }, -} + /// This fails if either there is no BABE `PreRuntime` digest or + /// the digest has type `SecondaryPlain`, which Substrate nodes do + /// not produce or accept anymore. + pub fn babe_unsafe_vrf_info(header: &Header) -> Option { + use babe_primitives::digests::CompatibleDigestItem; -/// A certification of assignment. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct AssignmentCert { - /// The criterion which is claimed to be met by this cert. - pub kind: AssignmentCertKind, - /// The VRF signature showing the criterion is met. - pub vrf: VrfSignature, -} + for digest in &header.digest.logs { + if let Some(pre) = digest.as_babe_pre_digest() { + let slot = pre.slot(); + let authority_index = pre.authority_index(); -/// An assignment criterion which refers to the candidate under which the assignment is -/// relevant by block hash. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectAssignmentCert { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The validator index. - pub validator: ValidatorIndex, - /// The cert itself. - pub cert: AssignmentCert, -} + return pre.vrf_signature().map(|sig| UnsafeVRFOutput { + vrf_output: sig.output.clone(), + slot, + authority_index, + }) + } + } -/// A signed approval vote which references the candidate indirectly via the block. -/// -/// In practice, we have a look-up from block hash and candidate index to candidate hash, -/// so this can be transformed into a `SignedApprovalVote`. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectSignedApprovalVote { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The index of the candidate in the list of candidates fully included as-of the block. - pub candidate_index: CandidateIndex, - /// The validator index. - pub validator: ValidatorIndex, - /// The signature by the validator. - pub signature: ValidatorSignature, + None + } } -/// Metadata about a block which is now live in the approval protocol. -#[derive(Debug)] -pub struct BlockApprovalMeta { - /// The hash of the block. - pub hash: Hash, - /// The number of the block. - pub number: BlockNumber, - /// The hash of the parent block. - pub parent_hash: Hash, - /// The candidates included by the block. - /// Note that these are not the same as the candidates that appear within the block body. - pub candidates: Vec, - /// The consensus slot of the block. - pub slot: Slot, - /// The session of the block. - pub session: SessionIndex, -} +/// A list of primitives introduced by v2. +pub mod v2 { + use parity_scale_codec::{Decode, Encode}; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + use std::ops::BitOr; -/// Errors that can occur during the approvals protocol. -#[derive(Debug, thiserror::Error)] -#[allow(missing_docs)] -pub enum ApprovalError { - #[error("Schnorrkel signature error")] - SchnorrkelSignature(schnorrkel::errors::SignatureError), - #[error("Authority index {0} out of bounds")] - AuthorityOutOfBounds(usize), -} + use bitvec::{prelude::Lsb0, vec::BitVec}; + use polkadot_primitives::{ + CandidateIndex, CoreIndex, Hash, ValidatorIndex, ValidatorSignature, + }; -/// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. -pub struct UnsafeVRFOutput { - vrf_output: VrfOutput, - slot: Slot, - authority_index: u32, -} + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2"; + /// A static context associated with producing randomness for v2 multi-core assignments. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED v2"; + /// A static context used for all relay-vrf-modulo VRFs for v2 multi-core assignments. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD v2"; + /// A read-only bitvec wrapper + #[derive(Clone, Debug, Encode, Decode, Hash, PartialEq, Eq)] + pub struct Bitfield(BitVec, std::marker::PhantomData); -impl UnsafeVRFOutput { - /// Get the slot. - pub fn slot(&self) -> Slot { - self.slot + /// A `read-only`, `non-zero` bitfield. + /// Each 1 bit identifies a candidate by the bitfield bit index. + pub type CandidateBitfield = Bitfield; + /// A bitfield of core assignments. + pub type CoreBitfield = Bitfield; + + /// Errors that can occur when creating and manipulating bitfields. + #[derive(Debug)] + pub enum BitfieldError { + /// All bits are zero. + NullAssignment, } - /// Compute the randomness associated with this VRF output. - pub fn compute_randomness( - self, - authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], - randomness: &babe_primitives::Randomness, - epoch_index: u64, - ) -> Result { - let author = match authorities.get(self.authority_index as usize) { - None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), - Some(x) => &x.0, - }; + /// A bit index in `Bitfield`. + #[cfg_attr(test, derive(PartialEq, Clone))] + pub struct BitIndex(pub usize); + + /// Helper trait to convert primitives to `BitIndex`. + pub trait AsBitIndex { + /// Returns the index of the corresponding bit in `Bitfield`. + fn as_bit_index(&self) -> BitIndex; + } + + impl Bitfield { + /// Returns the bit value at specified `index`. If `index` is greater than bitfield size, + /// returns `false`. + pub fn bit_at(&self, index: BitIndex) -> bool { + if self.0.len() <= index.0 { + false + } else { + self.0[index.0] + } + } + + /// Returns number of bits. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns the number of 1 bits. + pub fn count_ones(&self) -> usize { + self.0.count_ones() + } + + /// Returns the index of the first 1 bit. + pub fn first_one(&self) -> Option { + self.0.first_one() + } - let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) - .map_err(ApprovalError::SchnorrkelSignature)?; + /// Returns an iterator over inner bits. + pub fn iter_ones(&self) -> bitvec::slice::IterOnes { + self.0.iter_ones() + } - let transcript = sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + /// For testing purpose, we want a inner mutable ref. + #[cfg(test)] + pub fn inner_mut(&mut self) -> &mut BitVec { + &mut self.0 + } - let inout = self - .vrf_output - .0 - .attach_input_hash(&pubkey, transcript.0) - .map_err(ApprovalError::SchnorrkelSignature)?; - Ok(RelayVRFStory(inout.make_bytes(RELAY_VRF_STORY_CONTEXT))) + /// Returns the inner bitfield and consumes `self`. + pub fn into_inner(self) -> BitVec { + self.0 + } } -} -/// Extract the slot number and relay VRF from a header. -/// -/// This fails if either there is no BABE `PreRuntime` digest or -/// the digest has type `SecondaryPlain`, which Substrate nodes do -/// not produce or accept anymore. -pub fn babe_unsafe_vrf_info(header: &Header) -> Option { - use babe_primitives::digests::CompatibleDigestItem; - - for digest in &header.digest.logs { - if let Some(pre) = digest.as_babe_pre_digest() { - let slot = pre.slot(); - let authority_index = pre.authority_index(); - - return pre.vrf_signature().map(|sig| UnsafeVRFOutput { - vrf_output: sig.output.clone(), - slot, - authority_index, + impl AsBitIndex for CandidateIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self as usize) + } + } + + impl AsBitIndex for CoreIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(self.0 as usize) + } + } + + impl AsBitIndex for usize { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self) + } + } + + impl From for Bitfield + where + T: AsBitIndex, + { + fn from(value: T) -> Self { + Self( + { + let mut bv = bitvec::bitvec![u8, Lsb0; 0; value.as_bit_index().0 + 1]; + bv.set(value.as_bit_index().0, true); + bv + }, + Default::default(), + ) + } + } + + impl TryFrom> for Bitfield + where + T: Into>, + { + type Error = BitfieldError; + + fn try_from(mut value: Vec) -> Result { + if value.is_empty() { + return Err(BitfieldError::NullAssignment) + } + + let initial_bitfield = + value.pop().expect("Just checked above it's not empty; qed").into(); + + Ok(Self( + value.into_iter().fold(initial_bitfield.0, |initial_bitfield, element| { + let mut bitfield: Bitfield = element.into(); + bitfield + .0 + .resize(std::cmp::max(initial_bitfield.len(), bitfield.0.len()), false); + bitfield.0.bitor(initial_bitfield) + }), + Default::default(), + )) + } + } + + /// Certificate is changed compared to `AssignmentCertKind`: + /// - introduced RelayVRFModuloCompact + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block where the + /// candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 0)] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + #[codec(index = 1)] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 2)] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCertV2 { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKindV2, + /// The VRF showing the criterion is met. + pub vrf: VrfSignature, + } + + impl From for AssignmentCertV2 { + fn from(cert: super::v1::AssignmentCert) -> Self { + Self { + kind: match cert.kind { + super::v1::AssignmentCertKind::RelayVRFDelay { core_index } => + AssignmentCertKindV2::RelayVRFDelay { core_index }, + super::v1::AssignmentCertKind::RelayVRFModulo { sample } => + AssignmentCertKindV2::RelayVRFModulo { sample }, + }, + vrf: cert.vrf, + } + } + } + + /// Errors that can occur when trying to convert to/from assignment v1/v2 + #[derive(Debug)] + pub enum AssignmentConversionError { + /// Assignment certificate is not supported in v1. + CertificateNotSupported, + } + + impl TryFrom for super::v1::AssignmentCert { + type Error = AssignmentConversionError; + fn try_from(cert: AssignmentCertV2) -> Result { + Ok(Self { + kind: match cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => + super::v1::AssignmentCertKind::RelayVRFDelay { core_index }, + AssignmentCertKindV2::RelayVRFModulo { sample } => + super::v1::AssignmentCertKind::RelayVRFModulo { sample }, + // Not supported + _ => return Err(AssignmentConversionError::CertificateNotSupported), + }, + vrf: cert.vrf, }) } } - None + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCertV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCertV2, + } + + impl From for IndirectAssignmentCertV2 { + fn from(indirect_cert: super::v1::IndirectAssignmentCert) -> Self { + Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.into(), + } + } + } + + impl TryFrom for super::v1::IndirectAssignmentCert { + type Error = AssignmentConversionError; + fn try_from( + indirect_cert: IndirectAssignmentCertV2, + ) -> Result { + Ok(Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.try_into()?, + }) + } + } + + impl From for IndirectSignedApprovalVoteV2 { + fn from(value: super::v1::IndirectSignedApprovalVote) -> Self { + Self { + block_hash: value.block_hash, + validator: value.validator, + candidate_indices: value.candidate_index.into(), + signature: value.signature, + } + } + } + + /// Errors that can occur when trying to convert to/from approvals v1/v2 + #[derive(Debug)] + pub enum ApprovalConversionError { + /// More than one candidate was signed. + MoreThanOneCandidate(usize), + } + + impl TryFrom for super::v1::IndirectSignedApprovalVote { + type Error = ApprovalConversionError; + + fn try_from(value: IndirectSignedApprovalVoteV2) -> Result { + if value.candidate_indices.count_ones() != 1 { + return Err(ApprovalConversionError::MoreThanOneCandidate( + value.candidate_indices.count_ones(), + )) + } + Ok(Self { + block_hash: value.block_hash, + validator: value.validator, + candidate_index: value.candidate_indices.first_one().expect("Qed we checked above") + as u32, + signature: value.signature, + }) + } + } + + /// A signed approval vote which references the candidate indirectly via the block. + /// + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVoteV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_indices: CandidateBitfield, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } +} + +#[cfg(test)] +mod test { + use super::v2::{BitIndex, Bitfield}; + + use polkadot_primitives::{CandidateIndex, CoreIndex}; + + #[test] + fn test_assignment_bitfield_from_vec() { + let candidate_indices = vec![1u32, 7, 3, 10, 45, 8, 200, 2]; + let max_index = *candidate_indices.iter().max().unwrap(); + let bitfield = Bitfield::try_from(candidate_indices.clone()).unwrap(); + let candidate_indices = + candidate_indices.into_iter().map(|i| BitIndex(i as usize)).collect::>(); + + // Test 1 bits. + for index in candidate_indices.clone() { + assert!(bitfield.bit_at(index)); + } + + // Test 0 bits. + for index in 0..max_index { + if candidate_indices.contains(&BitIndex(index as usize)) { + continue + } + assert!(!bitfield.bit_at(BitIndex(index as usize))); + } + } + + #[test] + fn test_assignment_bitfield_invariant_msb() { + let core_indices = vec![CoreIndex(1), CoreIndex(3), CoreIndex(10), CoreIndex(20)]; + let mut bitfield = Bitfield::try_from(core_indices.clone()).unwrap(); + assert!(bitfield.inner_mut().pop().unwrap()); + + for i in 0..1024 { + assert!(Bitfield::try_from(CoreIndex(i)).unwrap().inner_mut().pop().unwrap()); + assert!(Bitfield::try_from(i).unwrap().inner_mut().pop().unwrap()); + } + } + + #[test] + fn test_assignment_bitfield_basic() { + let bitfield = Bitfield::try_from(CoreIndex(0)).unwrap(); + assert!(bitfield.bit_at(BitIndex(0))); + assert!(!bitfield.bit_at(BitIndex(1))); + assert_eq!(bitfield.len(), 1); + + let mut bitfield = Bitfield::try_from(20 as CandidateIndex).unwrap(); + assert!(bitfield.bit_at(BitIndex(20))); + assert_eq!(bitfield.inner_mut().count_ones(), 1); + assert_eq!(bitfield.len(), 21); + } } diff --git a/node/primitives/src/disputes/message.rs b/node/primitives/src/disputes/message.rs index 89d3ea6c0af9..31fe73a7ba1c 100644 --- a/node/primitives/src/disputes/message.rs +++ b/node/primitives/src/disputes/message.rs @@ -170,7 +170,7 @@ impl DisputeMessage { let valid_vote = ValidDisputeVote { validator_index: valid_index, signature: valid_statement.validator_signature().clone(), - kind: *valid_kind, + kind: valid_kind.clone(), }; let invalid_vote = InvalidDisputeVote { diff --git a/node/primitives/src/disputes/mod.rs b/node/primitives/src/disputes/mod.rs index 5e8e5815258d..040a92575e4e 100644 --- a/node/primitives/src/disputes/mod.rs +++ b/node/primitives/src/disputes/mod.rs @@ -46,6 +46,15 @@ pub struct SignedDisputeStatement { session_index: SessionIndex, } +/// Errors encountered while signing a dispute statement +#[derive(Debug)] +pub enum SignedDisputeStatementError { + /// Encountered a keystore error while signing + KeyStoreError(KeystoreError), + /// Could not generate signing payload + PayloadError, +} + /// Tracked votes on candidates, for the purposes of dispute resolution. #[derive(Debug, Clone)] pub struct CandidateVotes { @@ -107,8 +116,9 @@ impl ValidCandidateVotes { ValidDisputeStatementKind::BackingValid(_) | ValidDisputeStatementKind::BackingSeconded(_) => false, ValidDisputeStatementKind::Explicit | - ValidDisputeStatementKind::ApprovalChecking => { - occupied.insert((kind, sig)); + ValidDisputeStatementKind::ApprovalChecking | + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_) => { + occupied.insert((kind.clone(), sig)); kind != occupied.get().0 }, }, @@ -213,16 +223,19 @@ impl SignedDisputeStatement { candidate_hash: CandidateHash, session_index: SessionIndex, validator_public: ValidatorId, - ) -> Result, KeystoreError> { + ) -> Result, SignedDisputeStatementError> { let dispute_statement = if valid { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) } else { DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) }; - let data = dispute_statement.payload_data(candidate_hash, session_index); + let data = dispute_statement + .payload_data(candidate_hash, session_index) + .map_err(|_| SignedDisputeStatementError::PayloadError)?; let signature = keystore - .sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data)? + .sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data) + .map_err(SignedDisputeStatementError::KeyStoreError)? .map(|sig| Self { dispute_statement, candidate_hash, diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index 8cd9e4434bbd..167d0e8ff634 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -220,3 +220,4 @@ runtime-metrics = [ "polkadot-runtime?/runtime-metrics", "polkadot-runtime-parachains/runtime-metrics" ] +network-protocol-staging = ["polkadot-node-network-protocol/network-protocol-staging"] \ No newline at end of file diff --git a/node/service/src/fake_runtime_api.rs b/node/service/src/fake_runtime_api.rs index d9553afa024b..b8c40da41858 100644 --- a/node/service/src/fake_runtime_api.rs +++ b/node/service/src/fake_runtime_api.rs @@ -23,12 +23,12 @@ use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefyS use grandpa_primitives::AuthorityId as GrandpaId; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - runtime_api, slashing, AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, - CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, - DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, - ValidatorId, ValidatorIndex, ValidatorSignature, + runtime_api, slashing, vstaging::ApprovalVotingParams, AccountId, AuthorityDiscoveryId, + Balance, Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Nonce, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_core::OpaqueMetadata; use sp_runtime::{ @@ -116,6 +116,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(6)] impl runtime_api::ParachainHost for Runtime { fn validators() -> Vec { unimplemented!() @@ -229,6 +230,10 @@ sp_api::impl_runtime_apis! { ) -> Option<()> { unimplemented!() } + + fn approval_voting_params() -> ApprovalVotingParams { + unimplemented!() + } } impl beefy_primitives::BeefyApi for Runtime { diff --git a/node/service/src/parachains_db/mod.rs b/node/service/src/parachains_db/mod.rs index 519afbe0ccd1..92f3f167f22f 100644 --- a/node/service/src/parachains_db/mod.rs +++ b/node/service/src/parachains_db/mod.rs @@ -41,7 +41,15 @@ pub(crate) mod columns { pub const COL_SESSION_WINDOW_DATA: u32 = 5; } + // Version 4 only changed structures in approval voting, so we can re-export the v4 definitions. pub mod v3 { + pub use super::v4::{ + COL_APPROVAL_DATA, COL_AVAILABILITY_DATA, COL_AVAILABILITY_META, + COL_CHAIN_SELECTION_DATA, COL_DISPUTE_COORDINATOR_DATA, NUM_COLUMNS, ORDERED_COL, + }; + } + + pub mod v4 { pub const NUM_COLUMNS: u32 = 5; pub const COL_AVAILABILITY_DATA: u32 = 0; pub const COL_AVAILABILITY_META: u32 = 1; @@ -73,14 +81,14 @@ pub struct ColumnsConfig { /// The real columns used by the parachains DB. #[cfg(any(test, feature = "full-node"))] pub const REAL_COLUMNS: ColumnsConfig = ColumnsConfig { - col_availability_data: columns::v3::COL_AVAILABILITY_DATA, - col_availability_meta: columns::v3::COL_AVAILABILITY_META, - col_approval_data: columns::v3::COL_APPROVAL_DATA, - col_chain_selection_data: columns::v3::COL_CHAIN_SELECTION_DATA, - col_dispute_coordinator_data: columns::v3::COL_DISPUTE_COORDINATOR_DATA, + col_availability_data: columns::v4::COL_AVAILABILITY_DATA, + col_availability_meta: columns::v4::COL_AVAILABILITY_META, + col_approval_data: columns::v4::COL_APPROVAL_DATA, + col_chain_selection_data: columns::v4::COL_CHAIN_SELECTION_DATA, + col_dispute_coordinator_data: columns::v4::COL_DISPUTE_COORDINATOR_DATA, }; -#[derive(PartialEq)] +#[derive(PartialEq, Copy, Clone)] pub(crate) enum DatabaseKind { ParityDB, RocksDB, @@ -125,28 +133,28 @@ pub fn open_creating_rocksdb( let path = root.join("parachains").join("db"); - let mut db_config = DatabaseConfig::with_columns(columns::v3::NUM_COLUMNS); + let mut db_config = DatabaseConfig::with_columns(columns::v4::NUM_COLUMNS); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_DATA, cache_sizes.availability_data); + .insert(columns::v4::COL_AVAILABILITY_DATA, cache_sizes.availability_data); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_META, cache_sizes.availability_meta); + .insert(columns::v4::COL_AVAILABILITY_META, cache_sizes.availability_meta); let _ = db_config .memory_budget - .insert(columns::v3::COL_APPROVAL_DATA, cache_sizes.approval_data); + .insert(columns::v4::COL_APPROVAL_DATA, cache_sizes.approval_data); let path_str = path .to_str() .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB, upgrade::CURRENT_VERSION)?; let db = Database::open(&db_config, &path_str)?; let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) @@ -164,14 +172,14 @@ pub fn open_creating_paritydb( .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB, upgrade::CURRENT_VERSION)?; let db = parity_db::Db::open_or_create(&upgrade::paritydb_version_3_config(&path)) .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?; let db = polkadot_node_subsystem_util::database::paritydb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) } diff --git a/node/service/src/parachains_db/upgrade.rs b/node/service/src/parachains_db/upgrade.rs index 54ef97afd71c..9f7d35a712f8 100644 --- a/node/service/src/parachains_db/upgrade.rs +++ b/node/service/src/parachains_db/upgrade.rs @@ -22,13 +22,17 @@ use std::{ str::FromStr, }; +use polkadot_node_core_approval_voting::approval_db::v2::{ + migration_helpers::v1_to_v2, Config as ApprovalDbConfig, +}; type Version = u32; /// Version file name. const VERSION_FILE_NAME: &'static str = "parachain_db_version"; /// Current db version. -const CURRENT_VERSION: Version = 3; +/// Version 4 changes approval db format for `OurAssignment`. +pub(crate) const CURRENT_VERSION: Version = 4; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -38,6 +42,10 @@ pub enum Error { CorruptedVersionFile, #[error("Parachains DB has a future version (expected {current:?}, found {got:?})")] FutureVersion { current: Version, got: Version }, + #[error("Parachain DB migration failed")] + MigrationFailed, + #[error("Parachain DB migration would take forever")] + MigrationLoop, } impl From for io::Error { @@ -49,10 +57,35 @@ impl From for io::Error { } } -/// Try upgrading parachain's database to the current version. -pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +/// Try upgrading parachain's database to a target version. +pub(crate) fn try_upgrade_db( + db_path: &Path, + db_kind: DatabaseKind, + target_version: Version, +) -> Result<(), Error> { + // Ensure we don't loop forever below befcause of a bug. + const MAX_MIGRATIONS: u32 = 30; + + // Loop migrations until we reach the target version. + for _ in 0..MAX_MIGRATIONS { + let version = try_upgrade_db_to_next_version(db_path, db_kind)?; + if version == target_version { + return Ok(()) + } + } + + Err(Error::MigrationLoop) +} + +/// Try upgrading parachain's database to the next version. +/// If successfull, it returns the current version. +pub(crate) fn try_upgrade_db_to_next_version( + db_path: &Path, + db_kind: DatabaseKind, +) -> Result { let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none()); - if !is_empty { + + let new_version = if !is_empty { match get_db_version(db_path)? { // 0 -> 1 migration Some(0) => migrate_from_version_0_to_1(db_path, db_kind)?, @@ -60,21 +93,26 @@ pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<() Some(1) => migrate_from_version_1_to_2(db_path, db_kind)?, // 2 -> 3 migration Some(2) => migrate_from_version_2_to_3(db_path, db_kind)?, + // 3 -> 4 migration + Some(3) => migrate_from_version_3_to_4(db_path, db_kind)?, // Already at current version, do nothing. - Some(CURRENT_VERSION) => (), + Some(CURRENT_VERSION) => CURRENT_VERSION, // This is an arbitrary future version, we don't handle it. Some(v) => return Err(Error::FutureVersion { current: CURRENT_VERSION, got: v }), // No version file. For `RocksDB` we dont need to do anything. - None if db_kind == DatabaseKind::RocksDB => (), + None if db_kind == DatabaseKind::RocksDB => CURRENT_VERSION, // No version file. `ParityDB` did not previously have a version defined. // We handle this as a `0 -> 1` migration. None if db_kind == DatabaseKind::ParityDB => migrate_from_version_0_to_1(db_path, db_kind)?, None => unreachable!(), } - } + } else { + CURRENT_VERSION + }; - update_version(db_path) + update_version(db_path, new_version)?; + Ok(new_version) } /// Reads current database version from the file at given path. @@ -91,9 +129,9 @@ fn get_db_version(path: &Path) -> Result, Error> { /// Writes current database version to the file. /// Creates a new file if the version file does not exist yet. -fn update_version(path: &Path) -> Result<(), Error> { +fn update_version(path: &Path, new_version: Version) -> Result<(), Error> { fs::create_dir_all(path)?; - fs::write(version_file_path(path), CURRENT_VERSION.to_string()).map_err(Into::into) + fs::write(version_file_path(path), new_version.to_string()).map_err(Into::into) } /// Returns the version file path. @@ -103,7 +141,7 @@ fn version_file_path(path: &Path) -> PathBuf { file_path } -fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 0 to version 1 ..."); match db_kind { @@ -116,7 +154,7 @@ fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 1 to version 2 ..."); match db_kind { @@ -129,7 +167,48 @@ fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +// Migrade approval voting database. `OurAssignment` has been changed to support the v2 assignments. +// As these are backwards compatible, we'll convert the old entries in the new format. +fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result { + gum::info!(target: LOG_TARGET, "Migrating parachains db from version 3 to version 4 ..."); + use polkadot_node_subsystem_util::database::{ + kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, + }; + use std::sync::Arc; + + let approval_db_config = + ApprovalDbConfig { col_approval_data: super::REAL_COLUMNS.col_approval_data }; + + let _result = match db_kind { + DatabaseKind::ParityDB => { + let db = ParityDbAdapter::new( + parity_db::Db::open(&paritydb_version_3_config(path)) + .map_err(|e| other_io_error(format!("Error opening db {:?}", e)))?, + super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + DatabaseKind::RocksDB => { + let db_path = path + .to_str() + .ok_or_else(|| super::other_io_error("Invalid database path".into()))?; + let db_cfg = + kvdb_rocksdb::DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + let db = RocksDbAdapter::new( + kvdb_rocksdb::Database::open(&db_cfg, db_path)?, + &super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + }; + + gum::info!(target: LOG_TARGET, "Migration complete! "); + Ok(CURRENT_VERSION) +} + +fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 2 to version 3 ..."); match db_kind { DatabaseKind::ParityDB => paritydb_migrate_from_version_2_to_3(path), @@ -143,7 +222,7 @@ fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), /// Migration from version 0 to version 1: /// * the number of columns has changed from 3 to 5; -fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -155,12 +234,12 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { db.add_column()?; db.add_column()?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// * the number of columns has changed from 5 to 6; -fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -171,10 +250,10 @@ fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { db.add_column()?; - Ok(()) + Ok(2) } -fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -185,7 +264,7 @@ fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { db.remove_last_column()?; - Ok(()) + Ok(3) } // This currently clears columns which had their configs altered between versions. @@ -249,7 +328,7 @@ fn paritydb_fix_columns( pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v1::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -260,7 +339,7 @@ pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { pub(crate) fn paritydb_version_2_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v2::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -278,48 +357,59 @@ pub(crate) fn paritydb_version_3_config(path: &Path) -> parity_db::Options { options } +/// Database configuration for version 0. This is useful just for testing. +#[cfg(test)] +pub(crate) fn paritydb_version_0_config(path: &Path) -> parity_db::Options { + let mut options = + parity_db::Options::with_columns(&path, super::columns::v0::NUM_COLUMNS as u8); + options.columns[super::columns::v4::COL_AVAILABILITY_META as usize].btree_index = true; + + options +} + /// Migration from version 0 to version 1. /// Cases covered: /// - upgrading from v0.9.23 or earlier -> the `dispute coordinator column` was changed /// - upgrading from v0.9.24+ -> this is a no op assuming the DB has been manually fixed as per /// release notes -fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result { // Delete the `dispute coordinator` column if needed (if column configuration is changed). paritydb_fix_columns( path, paritydb_version_1_config(path), - vec![super::columns::v3::COL_DISPUTE_COORDINATOR_DATA], + vec![super::columns::v4::COL_DISPUTE_COORDINATOR_DATA], )?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// - add a new column for session information storage -fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result { let mut options = paritydb_version_1_config(path); // Adds the session info column. parity_db::Db::add_column(&mut options, Default::default()) .map_err(|e| other_io_error(format!("Error adding column {:?}", e)))?; - Ok(()) + Ok(2) } /// Migration from version 2 to version 3: /// - drop the column used by `RollingSessionWindow` -fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result { parity_db::Db::drop_last_column(&mut paritydb_version_2_config(path)) .map_err(|e| other_io_error(format!("Error removing COL_SESSION_WINDOW_DATA {:?}", e)))?; - Ok(()) + Ok(3) } #[cfg(test)] mod tests { use super::{ - columns::{v2::COL_SESSION_WINDOW_DATA, v3::*}, + columns::{v2::COL_SESSION_WINDOW_DATA, v4::*}, *, }; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_fill_test_data; #[test] fn test_rocksdb_migrate_1_to_2() { @@ -338,7 +428,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "1").expect("Failed to write DB version"); { - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); db.write(DBTransaction { ops: vec![DBOp::Insert { col: COL_DISPUTE_COORDINATOR_DATA, @@ -349,14 +439,14 @@ mod tests { .unwrap(); } - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 2).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS); - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); assert_eq!( db.get(COL_DISPUTE_COORDINATOR_DATA, b"1234").unwrap(), @@ -380,6 +470,77 @@ mod tests { ); } + #[test] + fn test_migrate_3_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_sanity_check; + use polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + let db_cfg: DatabaseConfig = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + + let approval_cfg = ApprovalDbConfig { + col_approval_data: crate::parachains_db::REAL_COLUMNS.col_approval_data, + }; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(db_dir.path()), "3").expect("Failed to write DB version"); + let expected_candidates = { + let db = Database::open(&db_cfg, db_path.clone()).unwrap(); + assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); + let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + // Fill the approval voting column with test data. + v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg.clone()).unwrap() + }; + + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); + + v1_to_v2_sanity_check(std::sync::Arc::new(db), approval_cfg.clone(), expected_candidates) + .unwrap(); + } + + #[test] + fn test_rocksdb_migrate_0_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + + fs::write(version_file_path(db_dir.path()), "0").expect("Failed to write DB version"); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS); + } + + #[test] + fn test_paritydb_migrate_0_to_4() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "0").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v0::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 4).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS as u8); + } + #[test] fn test_rocksdb_migrate_2_to_3() { use kvdb_rocksdb::{Database, DatabaseConfig}; @@ -387,6 +548,7 @@ mod tests { let db_dir = tempfile::tempdir().unwrap(); let db_path = db_dir.path().to_str().unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); + { let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS as u32); @@ -395,7 +557,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "2").expect("Failed to write DB version"); - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 3).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); diff --git a/node/service/src/tests.rs b/node/service/src/tests.rs index 86119662d9bc..26c8083185d8 100644 --- a/node/service/src/tests.rs +++ b/node/service/src/tests.rs @@ -17,7 +17,7 @@ use super::{relay_chain_selection::*, *}; use futures::channel::oneshot::Receiver; -use polkadot_node_primitives::approval::VrfSignature; +use polkadot_node_primitives::approval::v2::VrfSignature; use polkadot_node_subsystem::messages::{AllMessages, BlockDescription}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; diff --git a/node/subsystem-types/Cargo.toml b/node/subsystem-types/Cargo.toml index 1fb9ac83b780..7c5546741dff 100644 --- a/node/subsystem-types/Cargo.toml +++ b/node/subsystem-types/Cargo.toml @@ -24,3 +24,4 @@ smallvec = "1.8.0" substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" } thiserror = "1.0.31" async-trait = "0.1.57" +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index d5dcea7a2565..fa7f1dd4ec01 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -32,19 +32,23 @@ use polkadot_node_network_protocol::{ self as net_protocol, peer_set::PeerSet, request_response::Requests, PeerId, }; use polkadot_node_primitives::{ - approval::{BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::BlockApprovalMeta, + v2::{CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, + }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, CollationSecondedSignal, DisputeMessage, DisputeStatus, ErasureChunk, PoV, SignedDisputeStatement, SignedFullStatement, ValidationResult, }; use polkadot_primitives::{ - slashing, AuthorityDiscoveryId, BackedCandidate, BlockNumber, CandidateEvent, CandidateHash, - CandidateIndex, CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreState, - DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Header as BlockHeader, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, PvfExecTimeoutKind, - SessionIndex, SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, + slashing, vstaging::ApprovalVotingParams, AuthorityDiscoveryId, BackedCandidate, BlockNumber, + CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, CollatorId, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupIndex, + GroupRotationInfo, Hash, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, MultiDisputeStatementSet, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, PvfExecTimeoutKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, + SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, + ValidatorSignature, }; use polkadot_statement_table::v2::Misbehavior; use std::{ @@ -654,6 +658,8 @@ pub enum RuntimeApiRequest { slashing::OpaqueKeyOwnershipProof, RuntimeApiSender>, ), + /// Approval voting params + ApprovalVotingParams(RuntimeApiSender), } impl RuntimeApiRequest { @@ -771,6 +777,8 @@ pub enum AssignmentCheckError { InvalidCert(ValidatorIndex, String), #[error("Internal state mismatch: {0:?}, {1:?}")] Internal(Hash, CandidateHash), + #[error("Oversized candidate or core bitfield >= {0}")] + InvalidBitfield(usize), } /// The result type of [`ApprovalVotingMessage::CheckAndImportApproval`] request. @@ -836,15 +844,15 @@ pub enum ApprovalVotingMessage { /// Check if the assignment is valid and can be accepted by our view of the protocol. /// Should not be sent unless the block hash is known. CheckAndImportAssignment( - IndirectAssignmentCert, - CandidateIndex, + IndirectAssignmentCertV2, + CandidateBitfield, oneshot::Sender, ), /// Check if the approval vote is valid and can be accepted by our view of the /// protocol. /// /// Should not be sent unless the block hash within the indirect vote is known. - CheckAndImportApproval(IndirectSignedApprovalVote, oneshot::Sender), + CheckAndImportApproval(IndirectSignedApprovalVoteV2, oneshot::Sender), /// Returns the highest possible ancestor hash of the provided block hash which is /// acceptable to vote on finality for. /// The `BlockNumber` provided is the number of the block's ancestor which is the @@ -860,7 +868,7 @@ pub enum ApprovalVotingMessage { /// requires calling into `approval-distribution`: Calls should be infrequent and bounded. GetApprovalSignaturesForCandidate( CandidateHash, - oneshot::Sender>, + oneshot::Sender, ValidatorSignature)>>, ), } @@ -872,11 +880,11 @@ pub enum ApprovalDistributionMessage { NewBlocks(Vec), /// Distribute an assignment cert from the local validator. The cert is assumed /// to be valid, relevant, and for the given relay-parent and validator index. - DistributeAssignment(IndirectAssignmentCert, CandidateIndex), + DistributeAssignment(IndirectAssignmentCertV2, CandidateBitfield), /// Distribute an approval vote for the local validator. The approval vote is assumed to be /// valid, relevant, and the corresponding approval already issued. /// If not, the subsystem is free to drop the message. - DistributeApproval(IndirectSignedApprovalVote), + DistributeApproval(IndirectSignedApprovalVoteV2), /// An update from the network bridge. #[from] NetworkBridgeUpdate(NetworkBridgeEvent), @@ -884,7 +892,7 @@ pub enum ApprovalDistributionMessage { /// Get all approval signatures for all chains a candidate appeared in. GetApprovalSignatures( HashSet<(Hash, CandidateIndex)>, - oneshot::Sender>, + oneshot::Sender, ValidatorSignature)>>, ), /// Approval checking lag update measured in blocks. ApprovalCheckingLagUpdate(BlockNumber), diff --git a/node/subsystem-types/src/runtime_client.rs b/node/subsystem-types/src/runtime_client.rs index 4d8eddde73e9..dc4132305236 100644 --- a/node/subsystem-types/src/runtime_client.rs +++ b/node/subsystem-types/src/runtime_client.rs @@ -16,12 +16,13 @@ use async_trait::async_trait; use polkadot_primitives::{ - runtime_api::ParachainHost, vstaging, Block, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + runtime_api::ParachainHost, + vstaging::{self, ApprovalVotingParams}, + Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_api::{ApiError, ApiExt, ProvideRuntimeApi}; @@ -231,6 +232,9 @@ pub trait RuntimeApiSubsystemClient { &self, at: Hash, ) -> std::result::Result, ApiError>; + + /// Approval voting configuration parameters + async fn approval_voting_params(&self, at: Hash) -> Result; } /// Default implementation of [`RuntimeApiSubsystemClient`] using the client. @@ -456,4 +460,9 @@ where runtime_api.submit_report_dispute_lost(at, dispute_proof, key_ownership_proof) } + + /// Approval voting configuration parameters + async fn approval_voting_params(&self, at: Hash) -> Result { + self.client.runtime_api().approval_voting_params(at) + } } diff --git a/primitives/src/runtime_api.rs b/primitives/src/runtime_api.rs index c3a150a642e0..73578b95a055 100644 --- a/primitives/src/runtime_api.rs +++ b/primitives/src/runtime_api.rs @@ -114,10 +114,11 @@ //! separated from the stable primitives. use crate::{ - vstaging, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, + CoreState, DisputeState, ExecutorParams, GroupRotationInfo, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use parity_scale_codec::{Decode, Encode}; use polkadot_core_primitives as pcp; @@ -239,5 +240,9 @@ sp_api::decl_runtime_apis! { dispute_proof: vstaging::slashing::DisputeProof, key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; + + /// Approval voting configuration parameters + #[api_version(6)] + fn approval_voting_params() -> ApprovalVotingParams; } } diff --git a/primitives/src/v5/mod.rs b/primitives/src/v5/mod.rs index c973bb05bb48..445ab9976231 100644 --- a/primitives/src/v5/mod.rs +++ b/primitives/src/v5/mod.rs @@ -354,6 +354,13 @@ pub mod well_known_keys { /// Unique identifier for the Parachains Inherent pub const PARACHAINS_INHERENT_IDENTIFIER: InherentIdentifier = *b"parachn0"; +// /// TODO: Make this two a parachain host configuration. +// /// Maximum allowed candidates to be signed withing a signle approval votes. +// pub const MAX_APPROVAL_COALESCE_COUNT: u64 = 6; +// /// The maximum time we await for an approval to be coalesced with other approvals +// /// before we sign it and distribute to our peers +// pub const MAX_APPROVAL_COALESCE_WAIT_MILLIS: u64 = 500; + /// The key type ID for parachain assignment key. pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); @@ -1115,6 +1122,26 @@ impl ApprovalVote { } } +/// A vote of approvalf for multiple candidates. +#[derive(Clone, RuntimeDebug)] +pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); + +impl<'a> ApprovalVoteMultipleCandidates<'a> { + /// Yields the signing payload for this approval vote. + pub fn signing_payload(&self, session_index: SessionIndex) -> Vec { + const MAGIC: [u8; 4] = *b"APPR"; + // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the signature + // will look the same. + // This gives us the nice benefit that old nodes can still check signatures when len is 1 and the + // new node can check the signature coming from old nodes. + if self.0.len() == 1 { + (MAGIC, self.0.first().expect("QED: we just checked"), session_index).encode() + } else { + (MAGIC, &self.0, session_index).encode() + } + } +} + /// Custom validity errors used in Polkadot while validating transactions. #[repr(u8)] pub enum ValidityError { @@ -1289,25 +1316,39 @@ pub enum DisputeStatement { impl DisputeStatement { /// Get the payload data for this type of dispute statement. - pub fn payload_data(&self, candidate_hash: CandidateHash, session: SessionIndex) -> Vec { - match *self { + pub fn payload_data( + &self, + candidate_hash: CandidateHash, + session: SessionIndex, + ) -> Result, ()> { + match self { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => - ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(), + Ok(ExplicitDisputeStatement { valid: true, candidate_hash, session } + .signing_payload()), DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded( inclusion_parent, - )) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { + )) => Ok(CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, - }), + parent_hash: *inclusion_parent, + })), DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => - CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { + Ok(CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, - }), + parent_hash: *inclusion_parent, + })), DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => - ApprovalVote(candidate_hash).signing_payload(session), + Ok(ApprovalVote(candidate_hash).signing_payload(session)), + DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes), + ) => + if candidate_hashes.contains(&candidate_hash) { + Ok(ApprovalVoteMultipleCandidates(candidate_hashes).signing_payload(session)) + } else { + Err(()) + }, DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => - ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(), + Ok(ExplicitDisputeStatement { valid: false, candidate_hash, session } + .signing_payload()), } } @@ -1319,7 +1360,7 @@ impl DisputeStatement { session: SessionIndex, validator_signature: &ValidatorSignature, ) -> Result<(), ()> { - let payload = self.payload_data(candidate_hash, session); + let payload = self.payload_data(candidate_hash, session)?; if validator_signature.verify(&payload[..], &validator_public) { Ok(()) @@ -1351,13 +1392,14 @@ impl DisputeStatement { Self::Valid(ValidDisputeStatementKind::BackingValid(_)) => true, Self::Valid(ValidDisputeStatementKind::Explicit) | Self::Valid(ValidDisputeStatementKind::ApprovalChecking) | + Self::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_)) | Self::Invalid(_) => false, } } } /// Different kinds of statements of validity on a candidate. -#[derive(Encode, Decode, Copy, Clone, PartialEq, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)] pub enum ValidDisputeStatementKind { /// An explicit statement issued as part of a dispute. #[codec(index = 0)] @@ -1371,6 +1413,11 @@ pub enum ValidDisputeStatementKind { /// An approval vote from the approval checking phase. #[codec(index = 3)] ApprovalChecking, + /// An approval vote from the new version. + /// TODO: Fixme this probably means we can't create this version + /// untill all nodes have been updated to support it. + #[codec(index = 4)] + ApprovalCheckingMultipleCandidates(Vec), } /// Different kinds of statements of invalidity on a candidate. diff --git a/primitives/src/vstaging/mod.rs b/primitives/src/vstaging/mod.rs index 0dbfa8c34cf6..bc1b80296f5d 100644 --- a/primitives/src/vstaging/mod.rs +++ b/primitives/src/vstaging/mod.rs @@ -50,3 +50,23 @@ pub struct AsyncBackingParams { /// When async backing is disabled, the only valid value is 0. pub allowed_ancestry_len: u32, } + +/// Approval voting configuration parameters +#[derive( + RuntimeDebug, + Copy, + Clone, + PartialEq, + Encode, + Decode, + TypeInfo, + serde::Serialize, + serde::Deserialize, +)] +pub struct ApprovalVotingParams { + /// The maximum number of candidates `approval-voting` can vote for with + /// a single signatures. + /// + /// Setting it to 1, means we send the approval as soon as we have it available. + pub max_approval_coalesce_count: u32, +} diff --git a/roadmap/implementers-guide/src/node/approval/approval-voting.md b/roadmap/implementers-guide/src/node/approval/approval-voting.md index 88744e50cf79..ca811cf6b7f4 100644 --- a/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -2,7 +2,7 @@ Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the aims of this subsystem. -Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they broadcast an approval vote. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". +Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least `MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of `DelayTranche`s, which measure the number of ticks elapsed since a block was produced. We track metadata for all un-finalized but included candidates. We compute our local assignments to check each candidate, as well as which `DelayTranche` those assignments may be minimally triggered at. As the same candidate may appear in more than one block, we must produce our potential assignments for each (Block, Candidate) pair. The timing loop is based on waiting for assignments to become no-shows or waiting to broadcast and begin our own assignment to check. @@ -49,22 +49,28 @@ struct TrancheEntry { assignments: Vec<(ValidatorIndex, Tick)>, } -struct OurAssignment { - cert: AssignmentCert, - tranche: DelayTranche, - validator_index: ValidatorIndex, - triggered: bool, +pub struct OurAssignment { + /// Our assignment certificate. + cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + triggered: bool, } -struct ApprovalEntry { - tranches: Vec, // sorted ascending by tranche number. - backing_group: GroupIndex, - our_assignment: Option, - our_approval_sig: Option, - assignments: Bitfield, // n_validators bits - approved: bool, +pub struct ApprovalEntry { + tranches: Vec, + backing_group: GroupIndex, + our_assignment: Option, + our_approval_sig: Option, + // `n_validators` bits. + assigned_validators: Bitfield, + approved: bool, } + struct CandidateEntry { candidate: CandidateReceipt, session: SessionIndex, @@ -89,6 +95,13 @@ struct BlockEntry { // this block. The block can be considered approved has all bits set to 1 approved_bitfield: Bitfield, children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + candidates_pending_signature: BTreeMap, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + distributed_assignments: Bitfield, } // slot_duration * 2 + DelayTranche gives the number of delay tranches since the @@ -200,6 +213,8 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. * Check the assignment cert * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ sample.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. + * If the cert kind is `RelayVRFModuloCompact`, then the certificate is valid as long as the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the VRF Output. The assignment is considered a valid tranche0 assignment for all claimed candidates if all `core_bitfield` indices match the core indices where the claimed candidates were included at. + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. @@ -217,10 +232,10 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che On receiving a `CheckAndImportApproval(indirect_approval_vote, response_channel)` message: * Fetch the `BlockEntry` from the indirect approval vote's `block_hash`. If none, return `ApprovalCheckResult::Bad`. - * Fetch the `CandidateEntry` from the indirect approval vote's `candidate_index`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. - * Construct a `SignedApprovalVote` using the candidate hash and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. + * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. + * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. * Send `ApprovalCheckResult::Accepted` - * [Import the checked approval vote](#import-checked-approval) + * [Import the checked approval vote](#import-checked-approval) for all candidates #### `ApprovalVotingMessage::ApprovedAncestor` @@ -288,10 +303,24 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: #### Issue Approval Vote * Fetch the block entry and candidate entry. Ignore if `None` - we've probably just lost a race with finality. - * Construct a `SignedApprovalVote` with the validator index for the session. * [Import the checked approval vote](#import-checked-approval). It is "checked" as we've just issued the signature. - * Construct a `IndirectSignedApprovalVote` using the information about the vote. - * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * IF `MAX_APPROVAL_COALESCE_COUNT` candidates are in the waiting queue + * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. + * Construct a `IndirectSignedApprovalVote` using the information about the vote. + * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * ELSE + * Queue the candidate in the `BlockEntry::candidates_pending_signature` + * Arm a per BlockEntry timer with latest tick we can send the vote. + +### Delayed vote distribution + * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the vote immediately. + * When the timer wakes up it will either: + * IF there is a candidate in the queue past its sending tick: + * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. + * Construct a `IndirectSignedApprovalVote` using the information about the vote. + * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * ELSE + * Re-arm the timer with latest tick we have the send a the vote. ### Determining Approval of Candidate diff --git a/roadmap/implementers-guide/src/protocol-approval.md b/roadmap/implementers-guide/src/protocol-approval.md index 693822ce0797..63345032cbb0 100644 --- a/roadmap/implementers-guide/src/protocol-approval.md +++ b/roadmap/implementers-guide/src/protocol-approval.md @@ -98,12 +98,14 @@ We want checkers for candidate equivocations that lie outside our preferred rela Assignment criteria compute actual assignments using stories and the validators' secret approval assignment key. Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. -Assignment criteria come in three flavors, `RelayVRFModulo`, `RelayVRFDelay` and `RelayEquivocation`. Among these, both `RelayVRFModulo` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. +Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: `RelayVRFModulo` runs several distinct samples whose VRF input is the `RelayVRFStory` and the sample number. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core", reduces this number modulo the number of availability cores, and outputs the candidate just declared available by, and included by aka leaving, that availability core. We drop any samples that return no candidate because no candidate was leaving the sampled availability core in this relay chain block. We choose three samples initially, but we could make polkadot more secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. +`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up to `relay_vrf_modulo_samples` availability core indices. + There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. `RelayVRFDelay` and `RelayEquivocation` both compute their output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Tranche" and reduce the result modulo `num_delay_tranches + zeroth_delay_tranche_width`, and consolidate results 0 through `zeroth_delay_tranche_width` to be 0. In this way, they ensure the zeroth delay tranche has `zeroth_delay_tranche_width+1` times as many assignments as any other tranche. @@ -114,9 +116,9 @@ As future work (or TODO?), we should merge assignment notices with the same dela We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` assignments. +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` and `RelayVRFModuloCompact` assignments. In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`, and begin reconstruction and validation for 'C. If however `C` reached enough assignments, then validators with later assignments skip announcing their assignments. @@ -150,6 +152,12 @@ We strongly prefer if postponements come from tranches higher aka less important TODO: When? Is this optimal for the network? etc. +## Approval coalescing +To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay the sending of it untill one of three things happen: +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. +- `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. +- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in turn my trigger other assignments. + ## On-chain verification We should verify approval on-chain to reward approval checkers. We therefore require the "no show" timeout to be longer than a relay chain slot so that we can witness "no shows" on-chain, which helps with this goal. The major challenge with an on-chain record of the off-chain process is adversarial block producers who may either censor votes or publish votes to the chain which cause other votes to be ignored and unrewarded (reward stealing). @@ -164,7 +172,7 @@ We need the chain to win in this case, but doing this requires imposing an annoy ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` although assignment levels have never been properly analyzed. +We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` or `RelayVRFModuloCompact` although assignment levels have never been properly analyzed. Our delay criteria `RelayVRFDelay` and `RelayEquivocation` both have two primary paramaters, expected checkers per tranche and the zeroth delay tranche width. diff --git a/roadmap/implementers-guide/src/types/approval.md b/roadmap/implementers-guide/src/types/approval.md index b58e0a8187e1..2cc9b34cc700 100644 --- a/roadmap/implementers-guide/src/types/approval.md +++ b/roadmap/implementers-guide/src/types/approval.md @@ -20,6 +20,35 @@ enum AssignmentCertKind { } } +enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block where the + /// candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, +} + struct AssignmentCert { // The criterion which is claimed to be met by this cert. kind: AssignmentCertKind, diff --git a/roadmap/implementers-guide/src/types/runtime.md b/roadmap/implementers-guide/src/types/runtime.md index 55c0a571b6c8..81d4f6a7ce75 100644 --- a/roadmap/implementers-guide/src/types/runtime.md +++ b/roadmap/implementers-guide/src/types/runtime.md @@ -55,7 +55,7 @@ struct HostConfiguration { pub zeroth_delay_tranche_width: u32, /// The number of validators needed to approve a block. pub needed_approvals: u32, - /// The number of samples to do of the RelayVRFModulo approval assignment criterion. + /// The number of samples to use in `RelayVRFModulo` or `RelayVRFModuloCompact` approval assignment criterions. pub relay_vrf_modulo_samples: u32, /// Total number of individual messages allowed in the parachain -> relay-chain message queue. pub max_upward_queue_count: u32, diff --git a/runtime/kusama/src/lib.rs b/runtime/kusama/src/lib.rs index ae1925e898a2..07d63c907695 100644 --- a/runtime/kusama/src/lib.rs +++ b/runtime/kusama/src/lib.rs @@ -23,12 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, @@ -1692,6 +1692,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(6)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1822,6 +1823,11 @@ sp_api::impl_runtime_apis! { key_ownership_proof, ) } + + /// Approval voting configuration parameters + fn approval_voting_params() -> ApprovalVotingParams { + runtime_parachains::runtime_api_impl::vstaging::approval_voting_params:: () + } } impl beefy_primitives::BeefyApi for Runtime { diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 4921af5bedda..029bd0d68e74 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -634,7 +634,7 @@ impl BenchBuilder { } else { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) }; - let data = dispute_statement.payload_data(candidate_hash, session); + let data = dispute_statement.payload_data(candidate_hash.clone(), session).unwrap(); let statement_sig = validator_public.sign(&data).unwrap(); (dispute_statement, ValidatorIndex(validator_index), statement_sig) diff --git a/runtime/parachains/src/configuration.rs b/runtime/parachains/src/configuration.rs index 03d1ae420495..f84a2420733c 100644 --- a/runtime/parachains/src/configuration.rs +++ b/runtime/parachains/src/configuration.rs @@ -24,8 +24,9 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_parachain::primitives::{MAX_HORIZONTAL_MESSAGE_NUM, MAX_UPWARD_MESSAGE_NUM}; use primitives::{ - vstaging::AsyncBackingParams, Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, - MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + vstaging::{ApprovalVotingParams, AsyncBackingParams}, + Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use sp_runtime::{traits::Zero, Perbill}; use sp_std::prelude::*; @@ -242,6 +243,10 @@ pub struct HostConfiguration { /// /// This value should be greater than [`paras_availability_period`]. pub minimum_validation_upgrade_delay: BlockNumber, + + /// Params used by approval-voting + /// TODO: fixme this is not correctly migrated + pub approval_voting_params: ApprovalVotingParams, } impl> Default for HostConfiguration { @@ -292,6 +297,7 @@ impl> Default for HostConfiguration crate::hrmp::HRMP_MAX_INBOUND_CHANNELS_BOUND { return Err(MaxHrmpInboundChannelsExceeded) } - + // TODO: add consistency check for approval-voting-params Ok(()) } @@ -1150,6 +1156,21 @@ pub mod pallet { config.on_demand_ttl = new; }) } + /// Set approval-voting parameters. + #[pallet::call_index(52)] + #[pallet::weight(( + T::WeightInfo::set_config_with_executor_params(), + DispatchClass::Operational, + ))] + pub fn set_approval_voting_params( + origin: OriginFor, + new: ApprovalVotingParams, + ) -> DispatchResult { + ensure_root(origin)?; + Self::schedule_config_update(|config| { + config.approval_voting_params = new; + }) + } } #[pallet::hooks] diff --git a/runtime/parachains/src/configuration/migration/v8.rs b/runtime/parachains/src/configuration/migration/v8.rs index 7f7cc1cdefcd..94b415056d33 100644 --- a/runtime/parachains/src/configuration/migration/v8.rs +++ b/runtime/parachains/src/configuration/migration/v8.rs @@ -23,7 +23,7 @@ use frame_support::{ weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::SessionIndex; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; use sp_runtime::Perbill; use sp_std::vec::Vec; @@ -150,6 +150,9 @@ on_demand_base_fee : 10_000_000u128, on_demand_fee_variability : Perbill::from_percent(3), on_demand_target_queue_utilization : Perbill::from_percent(25), on_demand_ttl : 5u32.into(), +approval_voting_params : ApprovalVotingParams { + max_approval_coalesce_count: 1, + } } }; diff --git a/runtime/parachains/src/configuration/tests.rs b/runtime/parachains/src/configuration/tests.rs index b2a81894a939..31c293686033 100644 --- a/runtime/parachains/src/configuration/tests.rs +++ b/runtime/parachains/src/configuration/tests.rs @@ -317,6 +317,7 @@ fn setting_pending_config_members() { on_demand_fee_variability: Perbill::from_percent(3), on_demand_target_queue_utilization: Perbill::from_percent(25), on_demand_ttl: 5u32, + approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, }; Configuration::set_validation_upgrade_cooldown( diff --git a/runtime/parachains/src/disputes.rs b/runtime/parachains/src/disputes.rs index cf2e99e7359a..e0ff68f79063 100644 --- a/runtime/parachains/src/disputes.rs +++ b/runtime/parachains/src/disputes.rs @@ -25,11 +25,11 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_runtime_metrics::get_current_time; use primitives::{ - byzantine_threshold, supermajority_threshold, ApprovalVote, CandidateHash, - CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CompactStatement, ConsensusLog, - DisputeState, DisputeStatement, DisputeStatementSet, ExplicitDisputeStatement, - InvalidDisputeStatementKind, MultiDisputeStatementSet, SessionIndex, SigningContext, - ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature, + byzantine_threshold, supermajority_threshold, vstaging::ApprovalVoteMultipleCandidates, + ApprovalVote, CandidateHash, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, + CompactStatement, ConsensusLog, DisputeState, DisputeStatement, DisputeStatementSet, + ExplicitDisputeStatement, InvalidDisputeStatementKind, MultiDisputeStatementSet, SessionIndex, + SigningContext, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature, }; use scale_info::TypeInfo; use sp_runtime::{ @@ -1016,6 +1016,8 @@ impl Pallet { statement, signature, ) { + log::warn!("Failed to check dispute signature"); + importer.undo(undo); filter.remove_index(i); continue @@ -1261,21 +1263,29 @@ fn check_signature( statement: &DisputeStatement, validator_signature: &ValidatorSignature, ) -> Result<(), ()> { - let payload = match *statement { + let payload = match statement { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(), DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded(inclusion_parent)) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, + parent_hash: *inclusion_parent, }), DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, + parent_hash: *inclusion_parent, }), DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => ApprovalVote(candidate_hash).signing_payload(session), + DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates( + candidates, + )) => + if candidates.contains(&candidate_hash) { + ApprovalVoteMultipleCandidates(candidates).signing_payload(session) + } else { + return Err(()) + }, DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(), }; diff --git a/runtime/parachains/src/runtime_api_impl/vstaging.rs b/runtime/parachains/src/runtime_api_impl/vstaging.rs index d01b543630c3..2de20b0e087e 100644 --- a/runtime/parachains/src/runtime_api_impl/vstaging.rs +++ b/runtime/parachains/src/runtime_api_impl/vstaging.rs @@ -15,3 +15,12 @@ // along with Polkadot. If not, see . //! Put implementations of functions from staging APIs here. + +use primitives::vstaging::ApprovalVotingParams; + +use crate::{configuration, initializer}; + +pub fn approval_voting_params() -> ApprovalVotingParams { + let config = >::config(); + config.approval_voting_params +} diff --git a/runtime/polkadot/src/lib.rs b/runtime/polkadot/src/lib.rs index da8cf7cb5a61..436dbe80d1df 100644 --- a/runtime/polkadot/src/lib.rs +++ b/runtime/polkadot/src/lib.rs @@ -60,12 +60,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, }; use sp_core::OpaqueMetadata; use sp_mmr_primitives as mmr; @@ -1673,6 +1673,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(6)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1803,6 +1804,11 @@ sp_api::impl_runtime_apis! { key_ownership_proof, ) } + + /// Approval voting configuration parameters + fn approval_voting_params() -> ApprovalVotingParams { + runtime_parachains::runtime_api_impl::vstaging::approval_voting_params:: () + } } impl beefy_primitives::BeefyApi for Runtime { diff --git a/runtime/rococo/src/lib.rs b/runtime/rococo/src/lib.rs index a9811872b6af..49d3d64c0e96 100644 --- a/runtime/rococo/src/lib.rs +++ b/runtime/rococo/src/lib.rs @@ -23,11 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, impls::ToAuthor, @@ -1715,7 +1716,7 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] + #[api_version(6)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1846,6 +1847,10 @@ sp_api::impl_runtime_apis! { key_ownership_proof, ) } + /// Approval voting configuration parameters + fn approval_voting_params() -> ApprovalVotingParams { + runtime_parachains::runtime_api_impl::vstaging::approval_voting_params:: () + } } #[api_version(3)] diff --git a/runtime/westend/src/lib.rs b/runtime/westend/src/lib.rs index e6fa4afc9388..40d6b2464c86 100644 --- a/runtime/westend/src/lib.rs +++ b/runtime/westend/src/lib.rs @@ -39,12 +39,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, elections::OnChainAccuracy, impl_runtime_weights, @@ -1427,7 +1427,7 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] + #[api_version(6)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1558,6 +1558,11 @@ sp_api::impl_runtime_apis! { key_ownership_proof, ) } + + /// Approval voting configuration parameters + fn approval_voting_params() -> ApprovalVotingParams { + runtime_parachains::runtime_api_impl::vstaging::approval_voting_params:: () + } } impl beefy_primitives::BeefyApi for Runtime { diff --git a/scripts/ci/gitlab/pipeline/zombienet.yml b/scripts/ci/gitlab/pipeline/zombienet.yml index 6d023489c073..cdb963a58248 100644 --- a/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/scripts/ci/gitlab/pipeline/zombienet.yml @@ -64,6 +64,36 @@ zombienet-tests-parachains-pvf: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-approval-coalescing: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0006-approval-voting-coalescing.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + zombienet-tests-parachains-disputes: stage: zombienet image: "${ZOMBIENET_IMAGE}" @@ -160,6 +190,34 @@ zombienet-tests-parachains-disputes-past-session: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-max-tranche0-approvals: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: publish-malus-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0005-parachains-max-tranche0.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test zombienet-test-parachains-upgrade-smoke-test: stage: zombienet @@ -182,7 +240,7 @@ zombienet-test-parachains-upgrade-smoke-test: - echo "${GH_DIR}" - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + - export COL_IMAGE="docker.io/parity/polkadot-parachain:latest" # Use cumulus lastest image script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" diff --git a/zombienet_tests/functional/0001-parachains-pvf.zndsl b/zombienet_tests/functional/0001-parachains-pvf.zndsl index 46bb8bcdf72b..7859dd6d1156 100644 --- a/zombienet_tests/functional/0001-parachains-pvf.zndsl +++ b/zombienet_tests/functional/0001-parachains-pvf.zndsl @@ -32,6 +32,8 @@ alice: parachain 2005 block height is at least 10 within 300 seconds alice: parachain 2006 block height is at least 10 within 300 seconds alice: parachain 2007 block height is at least 10 within 300 seconds +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + # Check preparation time is under 10s. # Check all buckets <= 10. alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds diff --git a/zombienet_tests/functional/0002-parachains-disputes.toml b/zombienet_tests/functional/0002-parachains-disputes.toml index a0a87d60d4e3..5c649ceaf75f 100644 --- a/zombienet_tests/functional/0002-parachains-disputes.toml +++ b/zombienet_tests/functional/0002-parachains-disputes.toml @@ -5,6 +5,10 @@ timeout = 1000 max_validators_per_core = 5 needed_approvals = 8 +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] + max_approval_coalesce_count = 5 + + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/zombienet_tests/functional/0005-parachains-max-tranche0.toml b/zombienet_tests/functional/0005-parachains-max-tranche0.toml new file mode 100644 index 000000000000..ab7fa0195d13 --- /dev/null +++ b/zombienet_tests/functional/0005-parachains-max-tranche0.toml @@ -0,0 +1,40 @@ +[settings] +timeout = 1000 +bootnode = true + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + max_validators_per_core = 1 + needed_approvals = 7 + relay_vrf_modulo_samples = 5 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" +default_command = "polkadot" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.node_groups]] + name = "some-validator" + count = 8 + args = ["-lparachain=debug,runtime=debug"] + +{% for id in range(2000,2005) %} +[[parachains]] +id = {{id}} +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size={{10000*(id-1999)}} --pvf-complexity={{id - 1999}}" + [parachains.collator] + image = "{{COL_IMAGE}}" + name = "collator" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size={{10000*(id-1999)}}", "--parachain-id={{id}}", "--pvf-complexity={{id - 1999}}"] +{% endfor %} + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" diff --git a/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl b/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl new file mode 100644 index 000000000000..48a56f60c9bc --- /dev/null +++ b/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl @@ -0,0 +1,27 @@ +Description: Test if parachains make progress with most of approvals being tranch0 +Network: ./0005-parachains-max-tranche0.toml +Creds: config + +# Check authority status. +some-validator-0: reports node_roles is 4 +some-validator-1: reports node_roles is 4 +some-validator-3: reports node_roles is 4 +some-validator-4: reports node_roles is 4 +some-validator-5: reports node_roles is 4 +some-validator-6: reports node_roles is 4 +some-validator-7: reports node_roles is 4 + +some-validator-0: parachain 2000 block height is at least 5 within 180 seconds +some-validator-1: parachain 2001 block height is at least 5 within 180 seconds +some-validator-2: parachain 2002 block height is at least 5 within 180 seconds +some-validator-3: parachain 2003 block height is at least 5 within 180 seconds +some-validator-4: parachain 2004 block height is at least 5 within 180 seconds + +some-validator-0: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-1: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-2: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-3: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-4: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-5: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-6: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-7: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 diff --git a/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/zombienet_tests/functional/0006-approval-voting-coalescing.toml new file mode 100644 index 000000000000..444f56e70b34 --- /dev/null +++ b/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -0,0 +1,195 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + needed_approvals = 5 + relay_vrf_modulo_samples = 3 + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] + max_approval_coalesce_count = 6 + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + args = [ "--alice", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "charlie" + args = [ "--charlie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "dave" + args = [ "--dave", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "ferdie" + args = [ "--ferdie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "eve" + args = [ "--eve", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "one" + args = [ "--one", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "two" + args = [ "--two", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "nine" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eleven" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twelve" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "thirteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "fourteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "fifteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "sixteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "seventeen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eithteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "nineteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twenty" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twentyone" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twentytwo" + args = ["-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=1" + + [parachains.collator] + name = "collator01" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=1", "--parachain-id=2000"] + +[[parachains]] +id = 2001 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=10" + + [parachains.collator] + name = "collator02" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2001", "--pvf-complexity=10"] + +[[parachains]] +id = 2002 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=100" + + [parachains.collator] + name = "collator03" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2002", "--pvf-complexity=100"] + +[[parachains]] +id = 2003 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=300" + + [parachains.collator] + name = "collator04" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--parachain-id=2003", "--pvf-complexity=300"] + +[[parachains]] +id = 2004 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator05" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2004", "--pvf-complexity=300"] + +[[parachains]] +id = 2005 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=400" + + [parachains.collator] + name = "collator06" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--pvf-complexity=400", "--parachain-id=2005"] + +[[parachains]] +id = 2006 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator07" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2006"] + +[[parachains]] +id = 2007 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator08" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2007"] + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" \ No newline at end of file diff --git a/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl b/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl new file mode 100644 index 000000000000..272ac4fa2640 --- /dev/null +++ b/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl @@ -0,0 +1,51 @@ +Description: Approval voting coalescing does not lag finality +Network: ./0006-approval-voting-coalescing.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 +bob: reports node_roles is 4 +charlie: reports node_roles is 4 +dave: reports node_roles is 4 +eve: reports node_roles is 4 +ferdie: reports node_roles is 4 +one: reports node_roles is 4 +two: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds +bob: parachain 2001 is registered within 60 seconds +charlie: parachain 2002 is registered within 60 seconds +dave: parachain 2003 is registered within 60 seconds +ferdie: parachain 2004 is registered within 60 seconds +eve: parachain 2005 is registered within 60 seconds +one: parachain 2006 is registered within 60 seconds +two: parachain 2007 is registered within 60 seconds + +# Ensure parachains made progress. +alice: parachain 2000 block height is at least 10 within 300 seconds +alice: parachain 2001 block height is at least 10 within 300 seconds +alice: parachain 2002 block height is at least 10 within 300 seconds +alice: parachain 2003 block height is at least 10 within 300 seconds +alice: parachain 2004 block height is at least 10 within 300 seconds +alice: parachain 2005 block height is at least 10 within 300 seconds +alice: parachain 2006 block height is at least 10 within 300 seconds +alice: parachain 2007 block height is at least 10 within 300 seconds + +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +bob: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +charlie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +dave: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +eve: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +ferdie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + +alice: reports polkadot_parachain_approval_checking_finality_lag is 0 +bob: reports polkadot_parachain_approval_checking_finality_lag is 0 +charlie: reports polkadot_parachain_approval_checking_finality_lag is 0 +dave: reports polkadot_parachain_approval_checking_finality_lag is 0 +ferdie: reports polkadot_parachain_approval_checking_finality_lag is 0 +eve: reports polkadot_parachain_approval_checking_finality_lag is 0 +one: reports polkadot_parachain_approval_checking_finality_lag is 0 +two: reports polkadot_parachain_approval_checking_finality_lag is 0 \ No newline at end of file diff --git a/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml b/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml index 0becb408550a..88b789f37fa1 100644 --- a/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml +++ b/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml @@ -31,7 +31,7 @@ cumulus_based = true [parachains.collator] name = "collator01" image = "{{COL_IMAGE}}" - command = "polkadot-collator" + command = "polkadot-parachain" [[parachains.collator.env]] name = "RUST_LOG"