Prospective Parachains: track candidate parent nodes in a fragment tree

node/core/backing/src/lib.rs

@@ -87,8 +87,9 @@ use polkadot_node_subsystem::{
 	messages::{
 		AvailabilityDistributionMessage, AvailabilityStoreMessage, CandidateBackingMessage,
 		CandidateValidationMessage, CollatorProtocolMessage, DisputeCoordinatorMessage,
-		HypotheticalDepthRequest, ProspectiveParachainsMessage, ProvisionableData,
-		ProvisionerMessage, RuntimeApiMessage, RuntimeApiRequest, StatementDistributionMessage,
+		HypotheticalDepthRequest, ProspectiveCandidateParentState, ProspectiveParachainsMessage,
+		ProvisionableData, ProvisionerMessage, RuntimeApiMessage, RuntimeApiRequest,
+		StatementDistributionMessage,
 	},
 	overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError,
 };
@@ -1175,7 +1176,8 @@ async fn seconding_sanity_check<Context>(
 					// The leaf is already occupied.
 					return SecondingAllowed::No
 				}
-				responses.push(futures::future::ok((vec![0], head, leaf_state)).boxed());
+				let depths = vec![(0, ProspectiveCandidateParentState::Backed)];
+				responses.push(futures::future::ok((depths, head, leaf_state)).boxed());
 			}
 		}
 	}
@@ -1195,7 +1197,7 @@ async fn seconding_sanity_check<Context>(
 				return SecondingAllowed::No
 			},
 			Ok((depths, head, leaf_state)) => {
-				for depth in &depths {
+				for (depth, _) in &depths {
 					if leaf_state
 						.seconded_at_depth
 						.get(&candidate_para)
@@ -1212,6 +1214,7 @@ async fn seconding_sanity_check<Context>(
 						return SecondingAllowed::No
 					}
 				}
+				let depths = depths.into_iter().map(|(depth, _)| depth).collect();
 
 				membership.push((*head, depths));
 			},

node/core/backing/src/tests/prospective_parachains.rs

@@ -282,7 +282,10 @@ async fn assert_hypothetical_depth_requests(
 						request
 					),
 				};
-				let resp = std::mem::take(&mut expected_requests[idx].1);
+				let resp = std::mem::take(&mut expected_requests[idx].1)
+					.into_iter()
+					.map(|d| (d, ProspectiveCandidateParentState::Backed))
+					.collect();
 				tx.send(resp).unwrap();
 
 				expected_requests.remove(idx);

node/core/prospective-parachains/Cargo.toml

@@ -10,7 +10,6 @@ gum = { package = "tracing-gum", path = "../../gum" }
 parity-scale-codec = "2"
 thiserror = "1.0.30"
 fatality = "0.0.6"
-bitvec = "1"
 
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-primitives = { path = "../../primitives" }

node/core/prospective-parachains/src/fragment_tree.rs

@@ -57,7 +57,6 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
 
 use super::LOG_TARGET;
-use bitvec::prelude::*;
 use polkadot_node_subsystem_util::inclusion_emulator::staging::{
 	ConstraintModifications, Constraints, Fragment, ProspectiveCandidate, RelayChainBlockInfo,
 };
@@ -314,9 +313,10 @@ pub(crate) struct FragmentTree {
 	// the top-level children.
 	nodes: Vec<FragmentNode>,
 
-	// The candidates stored in this tree, mapped to a bitvec indicating the depths
-	// where the candidate is stored.
-	candidates: HashMap<CandidateHash, BitVec<u16, Msb0>>,
+	// The candidates stored in this tree, mapped to its membership in a tree:
+	// depth -> parent candidate hash. Candidate parent for 0-depth should always
+	// be backed, for convention we store zero-hash as a placeholder.
+	candidates: HashMap<CandidateHash, BTreeMap<usize, CandidateHash>>,
 }
 
 impl FragmentTree {
@@ -349,18 +349,14 @@ impl FragmentTree {
 		let pointer = NodePointer::Storage(self.nodes.len());
 		let parent_pointer = node.parent;
 		let candidate_hash = node.candidate_hash;
+		let node_depth = node.depth;
 
-		let max_depth = self.scope.max_depth;
-
-		self.candidates
-			.entry(candidate_hash)
-			.or_insert_with(|| bitvec![u16, Msb0; 0; max_depth + 1])
-			.set(node.depth, true);
-
-		match parent_pointer {
+		let parent_hash = match parent_pointer {
 			NodePointer::Storage(ptr) => {
 				self.nodes.push(node);
-				self.nodes[ptr].children.push((pointer, candidate_hash))
+				let parent_node = &mut self.nodes[ptr];
+				parent_node.children.push((pointer, candidate_hash));
+				parent_node.candidate_hash
 			},
 			NodePointer::Root => {
 				// Maintain the invariant of node storage beginning with depth-0.
@@ -371,8 +367,14 @@ impl FragmentTree {
 						self.nodes.iter().take_while(|n| n.parent == NodePointer::Root).count();
 					self.nodes.insert(pos, node);
 				}
+				CandidateHash::default()
 			},
-		}
+		};
+
+		self.candidates
+			.entry(candidate_hash)
+			.or_default()
+			.insert(node_depth, parent_hash);
 	}
 
 	fn node_has_candidate_child(
@@ -409,7 +411,7 @@ impl FragmentTree {
 
 	/// Whether the candidate exists and at what depths.
 	pub(crate) fn candidate(&self, candidate: &CandidateHash) -> Option<Vec<usize>> {
-		self.candidates.get(candidate).map(|d| d.iter_ones().collect())
+		self.candidates.get(candidate).map(|d| d.keys().copied().collect())
 	}
 
 	/// Add a candidate and recursively populate from storage.
@@ -454,10 +456,10 @@ impl FragmentTree {
 		hash: CandidateHash,
 		parent_head_data_hash: Hash,
 		candidate_relay_parent: Hash,
-	) -> Vec<usize> {
+	) -> Vec<(usize, CandidateHash)> {
 		// if known.
 		if let Some(depths) = self.candidates.get(&hash) {
-			return depths.iter_ones().collect()
+			return depths.iter().map(|(&depth, &parent)| (depth, parent)).collect()
 		}
 
 		// if out of scope.
@@ -471,7 +473,7 @@ impl FragmentTree {
 			};
 
 		let max_depth = self.scope.max_depth;
-		let mut depths = bitvec![u16, Msb0; 0; max_depth + 1];
+		let mut depths = BTreeMap::new();
 
 		// iterate over all nodes < max_depth where parent head-data matches,
 		// relay-parent number is <= candidate, and depth < max_depth.
@@ -483,16 +485,17 @@ impl FragmentTree {
 				continue
 			}
 			if node.head_data_hash == parent_head_data_hash {
-				depths.set(node.depth + 1, true);
+				depths.insert(node.depth + 1, node.candidate_hash);
 			}
 		}
 
 		// compare against root as well.
 		if self.scope.base_constraints.required_parent.hash() == parent_head_data_hash {
-			depths.set(0, true);
+			// use a placeholder value.
+			depths.insert(0, CandidateHash::default());
 		}
 
-		depths.iter_ones().collect()
+		depths.into_iter().collect()
 	}
 
 	/// Select a candidate after the given `required_path` which pass
@@ -1296,7 +1299,7 @@ mod tests {
 				HeadData::from(vec![0x0a]).hash(),
 				relay_parent_a,
 			),
-			vec![0, 2, 4],
+			vec![(0, CandidateHash::default()), (2, candidate_b_hash), (4, candidate_b_hash)],
 		);
 
 		assert_eq!(
@@ -1305,7 +1308,7 @@ mod tests {
 				HeadData::from(vec![0x0b]).hash(),
 				relay_parent_a,
 			),
-			vec![1, 3],
+			vec![(1, candidate_a_hash), (3, candidate_a_hash)],
 		);
 
 		assert_eq!(
@@ -1314,7 +1317,7 @@ mod tests {
 				HeadData::from(vec![0x0a]).hash(),
 				relay_parent_a,
 			),
-			vec![0, 2, 4],
+			vec![(0, CandidateHash::default()), (2, candidate_b_hash), (4, candidate_b_hash)],
 		);
 
 		assert_eq!(
@@ -1323,7 +1326,7 @@ mod tests {
 				HeadData::from(vec![0x0b]).hash(),
 				relay_parent_a,
 			),
-			vec![1, 3]
+			vec![(1, candidate_a_hash), (3, candidate_a_hash)],
 		);
 	}
 }

node/core/prospective-parachains/src/lib.rs

@@ -33,9 +33,9 @@ use futures::{channel::oneshot, prelude::*};
 
 use polkadot_node_subsystem::{
 	messages::{
-		ChainApiMessage, FragmentTreeMembership, HypotheticalDepthRequest,
-		ProspectiveParachainsMessage, ProspectiveValidationDataRequest, RuntimeApiMessage,
-		RuntimeApiRequest,
+		ChainApiMessage, CollatorProtocolMessage, FragmentTreeMembership, HypotheticalDepthRequest,
+		ProspectiveCandidateParentState, ProspectiveParachainsMessage,
+		ProspectiveValidationDataRequest, RuntimeApiMessage, RuntimeApiRequest,
 	},
 	overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError,
 };
@@ -326,7 +326,7 @@ async fn handle_candidate_seconded<Context>(
 
 #[overseer::contextbounds(ProspectiveParachains, prefix = self::overseer)]
 async fn handle_candidate_backed<Context>(
-	_ctx: &mut Context,
+	ctx: &mut Context,
 	view: &mut View,
 	para: ParaId,
 	candidate_hash: CandidateHash,
@@ -368,6 +368,9 @@ async fn handle_candidate_backed<Context>(
 	}
 
 	storage.mark_backed(&candidate_hash);
+
+	ctx.send_message(CollatorProtocolMessage::Backed(para, candidate_hash)).await;
+
 	Ok(())
 }
 
@@ -429,22 +432,35 @@ fn answer_get_backable_candidate(
 fn answer_hypothetical_depths_request(
 	view: &View,
 	request: HypotheticalDepthRequest,
-	tx: oneshot::Sender<Vec<usize>>,
+	tx: oneshot::Sender<Vec<(usize, ProspectiveCandidateParentState)>>,
 ) {
-	match view
+	let fragment_tree = view
 		.active_leaves
 		.get(&request.fragment_tree_relay_parent)
-		.and_then(|l| l.fragment_trees.get(&request.candidate_para))
-	{
-		Some(fragment_tree) => {
-			let depths = fragment_tree.hypothetical_depths(
-				request.candidate_hash,
-				request.parent_head_data_hash,
-				request.candidate_relay_parent,
-			);
+		.and_then(|l| l.fragment_trees.get(&request.candidate_para));
+	let candidate_storage = view.candidate_storage.get(&request.candidate_para);
+	match (fragment_tree, candidate_storage) {
+		(Some(fragment_tree), Some(candidate_storage)) => {
+			let depths = fragment_tree
+				.hypothetical_depths(
+					request.candidate_hash,
+					request.parent_head_data_hash,
+					request.candidate_relay_parent,
+				)
+				.into_iter()
+				.map(|(depth, parent)| {
+					let parent_status = if depth == 0 || candidate_storage.is_backed(&parent) {
+						ProspectiveCandidateParentState::Backed
+					} else {
+						ProspectiveCandidateParentState::Seconded(parent)
+					};
+					(depth, parent_status)
+				})
+				.collect();
+
 			let _ = tx.send(depths);
 		},
-		None => {
+		_ => {
 			let _ = tx.send(Vec::new());
 		},
 	}

node/network/collator-protocol/src/validator_side/mod.rs

@@ -1145,6 +1145,7 @@ async fn process_msg<Context>(
 				);
 			}
 		},
+		Backed(..) => {},
 		Invalid(parent, candidate_receipt) => {
 			let id = match state.pending_candidates.entry(parent) {
 				Entry::Occupied(entry)

node/overseer/src/lib.rs

@@ -606,6 +606,7 @@ pub struct Overseer<SupportsParachains> {
 	#[subsystem(ProspectiveParachainsMessage, sends: [
 		RuntimeApiMessage,
 		ChainApiMessage,
+		CollatorProtocolMessage,
 	])]
 	prospective_parachains: ProspectiveParachains,
 

node/subsystem-types/src/messages.rs

@@ -213,6 +213,9 @@ pub enum CollatorProtocolMessage {
 	///
 	/// The hash is the relay parent.
 	Seconded(Hash, SignedFullStatement),
+	/// Inform the Subsystem that a previously seconded candidate has been backed.
+	/// This message is forwarded from Prospective Parachains.
+	Backed(ParaId, CandidateHash),
 }
 
 impl Default for CollatorProtocolMessage {
@@ -956,6 +959,19 @@ pub struct HypotheticalDepthRequest {
 	pub fragment_tree_relay_parent: Hash,
 }
 
+/// State of candidate's parent node in a fragment tree of some para.
+///
+/// Validators should only second candidates for which there exists a backed
+/// parent in a fragment tree, including root.
+#[derive(Debug, Copy, Clone)]
+pub enum ProspectiveCandidateParentState {
+	/// Parent node candidate is seconded but not backed yet.
+	Seconded(CandidateHash),
+	/// Candidate's parent is backed, its hash is of
+	/// no interest.
+	Backed,
+}
+
 /// A request for the persisted validation data stored in the prospective
 /// parachains subsystem.
 #[derive(Debug)]
@@ -1004,7 +1020,10 @@ pub enum ProspectiveParachainsMessage {
 	///
 	/// Returns an empty vector either if there is no such depth or the fragment tree relay-parent
 	/// is unknown.
-	GetHypotheticalDepth(HypotheticalDepthRequest, oneshot::Sender<Vec<usize>>),
+	GetHypotheticalDepth(
+		HypotheticalDepthRequest,
+		oneshot::Sender<Vec<(usize, ProspectiveCandidateParentState)>>,
+	),
 	/// Get the membership of the candidate in all fragment trees.
 	GetTreeMembership(ParaId, CandidateHash, oneshot::Sender<FragmentTreeMembership>),
 	/// Get the minimum accepted relay-parent number for each para in the fragment tree