Skip to content

Commit

Permalink
Custom RPC request management for sync (#3029)
Browse files Browse the repository at this point in the history 
## Proposed Changes
Make `lighthouse_network` generic over request ids, now usable by sync
  • Loading branch information
@divagant-martian
divagant-martian committed Mar 2, 2022
1 parent e88b18b commit 4bf1af4
Show file tree
Hide file tree
Showing 18 changed files with 568 additions and 519 deletions.
116 changes: 85 additions & 31 deletions beacon_node/lighthouse_network/src/behaviour/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,16 @@ pub type PeerRequestId = (ConnectionId, SubstreamId);
pub type SubscriptionFilter = MaxCountSubscriptionFilter<WhitelistSubscriptionFilter>;
pub type Gossipsub = BaseGossipsub<SnappyTransform, SubscriptionFilter>;

/// Identifier of a request.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RequestId<AppReqId> {
Application(AppReqId),
Behaviour,
}

/// The types of events than can be obtained from polling the behaviour.
#[derive(Debug)]
pub enum BehaviourEvent<TSpec: EthSpec> {
pub enum BehaviourEvent<AppReqId: ReqId, TSpec: EthSpec> {
/// We have successfully dialed and connected to a peer.
PeerConnectedOutgoing(PeerId),
/// A peer has successfully dialed and connected to us.
Expand All @@ -86,7 +93,7 @@ pub enum BehaviourEvent<TSpec: EthSpec> {
/// An RPC Request that was sent failed.
RPCFailed {
/// The id of the failed request.
id: RequestId,
id: AppReqId,
/// The peer to which this request was sent.
peer_id: PeerId,
},
Expand All @@ -102,7 +109,7 @@ pub enum BehaviourEvent<TSpec: EthSpec> {
/// Peer that sent the response.
peer_id: PeerId,
/// Id of the request to which the peer is responding.
id: RequestId,
id: AppReqId,
/// Response the peer sent.
response: Response<TSpec>,
},
Expand Down Expand Up @@ -134,16 +141,16 @@ enum InternalBehaviourMessage {
/// behaviours.
#[derive(NetworkBehaviour)]
#[behaviour(
out_event = "BehaviourEvent<TSpec>",
out_event = "BehaviourEvent<AppReqId, TSpec>",
poll_method = "poll",
event_process = true
)]
pub struct Behaviour<TSpec: EthSpec> {
pub struct Behaviour<AppReqId: ReqId, TSpec: EthSpec> {
/* Sub-Behaviours */
/// The routing pub-sub mechanism for eth2.
gossipsub: Gossipsub,
/// The Eth2 RPC specified in the wire-0 protocol.
eth2_rpc: RPC<TSpec>,
eth2_rpc: RPC<RequestId<AppReqId>, TSpec>,
/// Discv5 Discovery protocol.
discovery: Discovery<TSpec>,
/// Keep regular connection to peers and disconnect if absent.
Expand All @@ -156,7 +163,7 @@ pub struct Behaviour<TSpec: EthSpec> {
/* Auxiliary Fields */
/// The output events generated by this behaviour to be consumed in the swarm poll.
#[behaviour(ignore)]
events: VecDeque<BehaviourEvent<TSpec>>,
events: VecDeque<BehaviourEvent<AppReqId, TSpec>>,
/// Internal behaviour events, the NBAction type is composed of sub-behaviours, so we use a
/// custom type here to avoid having to specify the concrete type.
#[behaviour(ignore)]
Expand Down Expand Up @@ -192,7 +199,7 @@ pub struct Behaviour<TSpec: EthSpec> {
}

/// Implements the combined behaviour for the libp2p service.
impl<TSpec: EthSpec> Behaviour<TSpec> {
impl<AppReqId: ReqId, TSpec: EthSpec> Behaviour<AppReqId, TSpec> {
pub async fn new(
local_key: &Keypair,
ctx: ServiceContext<'_>,
Expand Down Expand Up @@ -562,9 +569,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
/* Eth2 RPC behaviour functions */

/// Send a request to a peer over RPC.
pub fn send_request(&mut self, peer_id: PeerId, request_id: RequestId, request: Request) {
pub fn send_request(&mut self, peer_id: PeerId, request_id: AppReqId, request: Request) {
self.eth2_rpc
.send_request(peer_id, request_id, request.into())
.send_request(peer_id, RequestId::Application(request_id), request.into())
}

/// Send a successful response to a peer over RPC.
Expand Down Expand Up @@ -718,12 +725,12 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
}

/// Sends a Ping request to the peer.
fn ping(&mut self, id: RequestId, peer_id: PeerId) {
fn ping(&mut self, peer_id: PeerId) {
let ping = crate::rpc::Ping {
data: *self.network_globals.local_metadata.read().seq_number(),
};
trace!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => %peer_id);

trace!(self.log, "Sending Ping"; "peer_id" => %peer_id);
let id = RequestId::Behaviour;
self.eth2_rpc
.send_request(peer_id, id, OutboundRequest::Ping(ping));
}
Expand Down Expand Up @@ -761,13 +768,19 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {

// RPC Propagation methods
/// Queues the response to be sent upwards as long at it was requested outside the Behaviour.
fn propagate_response(&mut self, id: RequestId, peer_id: PeerId, response: Response<TSpec>) {
if !matches!(id, RequestId::Behaviour) {
self.add_event(BehaviourEvent::ResponseReceived {
fn propagate_response(
&mut self,
id: RequestId<AppReqId>,
peer_id: PeerId,
response: Response<TSpec>,
) {
match id {
RequestId::Application(id) => self.add_event(BehaviourEvent::ResponseReceived {
peer_id,
id,
response,
});
}),
RequestId::Behaviour => {}
}
}

Expand All @@ -793,7 +806,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
}

/// Adds an event to the queue waking the current task to process it.
fn add_event(&mut self, event: BehaviourEvent<TSpec>) {
fn add_event(&mut self, event: BehaviourEvent<AppReqId, TSpec>) {
self.events.push_back(event);
if let Some(waker) = &self.waker {
waker.wake_by_ref();
Expand Down Expand Up @@ -869,7 +882,11 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
*/

// Gossipsub
impl<TSpec: EthSpec> NetworkBehaviourEventProcess<GossipsubEvent> for Behaviour<TSpec> {
impl<AppReqId, TSpec> NetworkBehaviourEventProcess<GossipsubEvent> for Behaviour<AppReqId, TSpec>
where
AppReqId: ReqId,
TSpec: EthSpec,
{
fn inject_event(&mut self, event: GossipsubEvent) {
match event {
GossipsubEvent::Message {
Expand Down Expand Up @@ -961,8 +978,13 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<GossipsubEvent> for Behaviour<
}

// RPC
impl<TSpec: EthSpec> NetworkBehaviourEventProcess<RPCMessage<TSpec>> for Behaviour<TSpec> {
fn inject_event(&mut self, event: RPCMessage<TSpec>) {
impl<AppReqId, TSpec> NetworkBehaviourEventProcess<RPCMessage<RequestId<AppReqId>, TSpec>>
for Behaviour<AppReqId, TSpec>
where
AppReqId: ReqId,
TSpec: EthSpec,
{
fn inject_event(&mut self, event: RPCMessage<RequestId<AppReqId>, TSpec>) {
let peer_id = event.peer_id;

if !self.peer_manager.is_connected(&peer_id) {
Expand Down Expand Up @@ -1006,7 +1028,7 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<RPCMessage<TSpec>> for Behavio
ConnectionDirection::Outgoing,
);
// inform failures of requests comming outside the behaviour
if !matches!(id, RequestId::Behaviour) {
if let RequestId::Application(id) = id {
self.add_event(BehaviourEvent::RPCFailed { peer_id, id });
}
}
Expand Down Expand Up @@ -1090,7 +1112,11 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<RPCMessage<TSpec>> for Behavio
}

// Discovery
impl<TSpec: EthSpec> NetworkBehaviourEventProcess<DiscoveryEvent> for Behaviour<TSpec> {
impl<AppReqId, TSpec> NetworkBehaviourEventProcess<DiscoveryEvent> for Behaviour<AppReqId, TSpec>
where
AppReqId: ReqId,
TSpec: EthSpec,
{
fn inject_event(&mut self, event: DiscoveryEvent) {
match event {
DiscoveryEvent::SocketUpdated(socket_addr) => {
Expand Down Expand Up @@ -1119,7 +1145,11 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<DiscoveryEvent> for Behaviour<
}

// Identify
impl<TSpec: EthSpec> NetworkBehaviourEventProcess<IdentifyEvent> for Behaviour<TSpec> {
impl<AppReqId, TSpec> NetworkBehaviourEventProcess<IdentifyEvent> for Behaviour<AppReqId, TSpec>
where
TSpec: EthSpec,
AppReqId: ReqId,
{
fn inject_event(&mut self, event: IdentifyEvent) {
match event {
IdentifyEvent::Received { peer_id, mut info } => {
Expand All @@ -1140,15 +1170,20 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<IdentifyEvent> for Behaviour<T
}
}

impl<TSpec: EthSpec> Behaviour<TSpec> {
type BehaviourHandler<AppReqId, TSpec> =
<Behaviour<AppReqId, TSpec> as NetworkBehaviour>::ConnectionHandler;

impl<AppReqId, TSpec> Behaviour<AppReqId, TSpec>
where
TSpec: EthSpec,
AppReqId: ReqId,
{
/// Consumes the events list and drives the Lighthouse global NetworkBehaviour.
fn poll(
&mut self,
cx: &mut Context,
_: &mut impl PollParameters,
) -> Poll<
NBAction<BehaviourEvent<TSpec>, <Behaviour<TSpec> as NetworkBehaviour>::ConnectionHandler>,
> {
) -> Poll<NBAction<BehaviourEvent<AppReqId, TSpec>, BehaviourHandler<AppReqId, TSpec>>> {
if let Some(waker) = &self.waker {
if waker.will_wake(cx.waker()) {
self.waker = Some(cx.waker().clone());
Expand Down Expand Up @@ -1207,7 +1242,9 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
}
}

impl<TSpec: EthSpec> NetworkBehaviourEventProcess<PeerManagerEvent> for Behaviour<TSpec> {
impl<AppReqId: ReqId, TSpec: EthSpec> NetworkBehaviourEventProcess<PeerManagerEvent>
for Behaviour<AppReqId, TSpec>
{
fn inject_event(&mut self, event: PeerManagerEvent) {
match event {
PeerManagerEvent::PeerConnectedIncoming(peer_id) => {
Expand Down Expand Up @@ -1242,7 +1279,7 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<PeerManagerEvent> for Behaviou
}
PeerManagerEvent::Ping(peer_id) => {
// send a ping request to this peer
self.ping(RequestId::Behaviour, peer_id);
self.ping(peer_id);
}
PeerManagerEvent::MetaData(peer_id) => {
self.send_meta_data_request(peer_id);
Expand All @@ -1251,7 +1288,8 @@ impl<TSpec: EthSpec> NetworkBehaviourEventProcess<PeerManagerEvent> for Behaviou
debug!(self.log, "Peer Manager disconnecting peer";
"peer_id" => %peer_id, "reason" => %reason);
// send one goodbye
self.eth2_rpc.shutdown(peer_id, reason);
self.eth2_rpc
.shutdown(peer_id, RequestId::Behaviour, reason);
}
}
}
Expand Down Expand Up @@ -1335,3 +1373,19 @@ pub fn save_metadata_to_disk<E: EthSpec>(dir: &Path, metadata: MetaData<E>, log:
}
}
}

impl<AppReqId: std::fmt::Debug> slog::Value for RequestId<AppReqId> {
fn serialize(
&self,
record: &slog::Record,
key: slog::Key,
serializer: &mut dyn slog::Serializer,
) -> slog::Result {
match self {
RequestId::Behaviour => slog::Value::serialize("Behaviour", record, key, serializer),
RequestId::Application(ref id) => {
slog::Value::serialize(&format_args!("{:?}", id), record, key, serializer)
}
}
}
}
6 changes: 5 additions & 1 deletion beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,13 @@ impl<TSpec: EthSpec> NetworkBehaviour for PeerManager<TSpec> {
_connection_id: &ConnectionId,
endpoint: &ConnectedPoint,
_failed_addresses: Option<&Vec<Multiaddr>>,
_other_established: usize,
other_established: usize,
) {
debug!(self.log, "Connection established"; "peer_id" => %peer_id, "connection" => ?endpoint.to_endpoint());
if other_established == 0 {
self.events.push(PeerManagerEvent::MetaData(*peer_id));
}

// Check NAT if metrics are enabled
if self.network_globals.local_enr.read().udp().is_some() {
metrics::check_nat();
Expand Down
Loading

0 comments on commit 4bf1af4

Please sign in to comment.