From 61d4ae13af06366fb4123af081234812d2b5f3ab Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 2 Aug 2023 17:01:02 +1000 Subject: [PATCH 01/70] Initial quic support --- Cargo.lock | 308 +++++++++++++----- beacon_node/lighthouse_network/Cargo.toml | 3 +- beacon_node/lighthouse_network/src/config.rs | 70 ++-- .../lighthouse_network/src/listen_addr.rs | 31 +- .../lighthouse_network/src/service/utils.rs | 27 +- beacon_node/network/src/nat.rs | 59 ++-- beacon_node/network/src/service.rs | 35 +- beacon_node/src/cli.rs | 48 ++- 8 files changed, 397 insertions(+), 184 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2e8188eec4..16dae7f6f59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,45 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror", + "time 0.3.24", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "asn1_der" version = "0.7.6" @@ -275,7 +314,7 @@ checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.10", + "pin-project-lite", ] [[package]] @@ -321,7 +360,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.10", + "pin-project-lite", ] [[package]] @@ -393,7 +432,7 @@ dependencies = [ "memchr", "mime", "percent-encoding", - "pin-project-lite 0.2.10", + "pin-project-lite", "rustversion", "serde", "serde_json", @@ -916,9 +955,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "51f1226cd9da55587234753d1245dd5b132343ea240f26b6a9003d68706141ba" +dependencies = [ + "libc", +] [[package]] name = "cexpr" @@ -1515,6 +1557,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.3.6" @@ -1706,6 +1762,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "dtoa" version = "1.0.9" @@ -2809,7 +2876,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.10", + "pin-project-lite", "waker-fn", ] @@ -2826,13 +2893,12 @@ dependencies = [ [[package]] name = "futures-rustls" -version = "0.22.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd" +checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" dependencies = [ "futures-io", - "rustls 0.20.8", - "webpki 0.22.0", + "rustls 0.21.5", ] [[package]] @@ -2877,7 +2943,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.10", + "pin-project-lite", "pin-utils", "slab", ] @@ -3256,7 +3322,7 @@ checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.10", + "pin-project-lite", ] [[package]] @@ -3361,7 +3427,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.10", + "pin-project-lite", "socket2 0.4.9", "tokio", "tower-service", @@ -3928,7 +3994,6 @@ dependencies = [ "libp2p-plaintext", "libp2p-swarm", "libp2p-tcp", - "libp2p-websocket", "libp2p-yamux", "multiaddr 0.18.0", "pin-project", @@ -4155,6 +4220,28 @@ dependencies = [ "unsigned-varint 0.7.1", ] +[[package]] +name = "libp2p-quic" +version = "0.9.0-alpha" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af35e582535de00bbac3d62412d53c1f07c80f1b6ea1451716b9e4ed5002498d" +dependencies = [ + "bytes", + "futures", + "futures-timer", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-tls", + "log", + "parking_lot 0.12.1", + "quinn", + "rand 0.8.5", + "rustls 0.21.5", + "thiserror", + "tokio", +] + [[package]] name = "libp2p-swarm" version = "0.43.2" @@ -4209,23 +4296,22 @@ dependencies = [ ] [[package]] -name = "libp2p-websocket" -version = "0.42.0" +name = "libp2p-tls" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956d981ebc84abc3377e5875483c06d94ff57bc6b25f725047f9fd52592f72d4" +checksum = "ec63209ec17ffb354a5fdfde7c36f14d168367c5f115fdb5a177a342414d5a55" dependencies = [ - "either", "futures", "futures-rustls", "libp2p-core", "libp2p-identity", - "log", - "parking_lot 0.12.1", - "quicksink", - "rw-stream-sink", - "soketto", - "url", - "webpki-roots 0.23.1", + "rcgen", + "ring", + "rustls 0.21.5", + "thiserror", + "webpki 0.22.0", + "x509-parser", + "yasna", ] [[package]] @@ -4376,6 +4462,7 @@ dependencies = [ "hex", "lazy_static", "libp2p", + "libp2p-quic", "lighthouse_metrics", "lighthouse_version", "lru 0.7.8", @@ -5238,6 +5325,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -5602,12 +5698,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "pin-project-lite" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" - [[package]] name = "pin-project-lite" version = "0.2.10" @@ -5698,7 +5788,7 @@ dependencies = [ "concurrent-queue", "libc", "log", - "pin-project-lite 0.2.10", + "pin-project-lite", "windows-sys", ] @@ -6014,14 +6104,51 @@ dependencies = [ ] [[package]] -name = "quicksink" -version = "0.1.2" +name = "quinn" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77de3c815e5a160b1539c6592796801df2043ae35e123b46d73380cfa57af858" +checksum = "21252f1c0fc131f1b69182db8f34837e8a69737b8251dff75636a9be0518c324" dependencies = [ - "futures-core", - "futures-sink", - "pin-project-lite 0.1.12", + "bytes", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.21.5", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85af4ed6ee5a89f26a26086e9089a6643650544c025158449a3626ebf72884b3" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls 0.21.5", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df19e284d93757a9fb91d63672f7741b129246a669db09d1c0063071debc0c0" +dependencies = [ + "bytes", + "libc", + "socket2 0.5.3", + "tracing", + "windows-sys", ] [[package]] @@ -6168,6 +6295,18 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring", + "time 0.3.24", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -6265,7 +6404,7 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.10", + "pin-project-lite", "rustls 0.21.5", "rustls-pemfile", "serde", @@ -6281,7 +6420,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots", "winreg 0.10.1", ] @@ -6433,6 +6572,15 @@ dependencies = [ "semver 1.0.18", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustix" version = "0.37.23" @@ -6493,7 +6641,7 @@ checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" dependencies = [ "log", "ring", - "rustls-webpki 0.101.2", + "rustls-webpki", "sct 0.7.0", ] @@ -6506,16 +6654,6 @@ dependencies = [ "base64 0.21.2", ] -[[package]] -name = "rustls-webpki" -version = "0.100.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.101.2" @@ -7296,21 +7434,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "soketto" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" -dependencies = [ - "base64 0.13.1", - "bytes", - "futures", - "httparse", - "log", - "rand 0.8.5", - "sha-1 0.9.8", -] - [[package]] name = "spin" version = "0.5.2" @@ -7896,7 +8019,7 @@ dependencies = [ "mio", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.10", + "pin-project-lite", "signal-hook-registry", "socket2 0.4.9", "tokio-macros", @@ -7909,7 +8032,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ - "pin-project-lite 0.2.10", + "pin-project-lite", "tokio", ] @@ -7950,7 +8073,7 @@ dependencies = [ "parking_lot 0.12.1", "percent-encoding", "phf", - "pin-project-lite 0.2.10", + "pin-project-lite", "postgres-protocol", "postgres-types", "socket2 0.5.3", @@ -7997,7 +8120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", - "pin-project-lite 0.2.10", + "pin-project-lite", "tokio", "tokio-util 0.7.8", ] @@ -8028,7 +8151,7 @@ dependencies = [ "tokio-rustls 0.23.4", "tungstenite 0.17.3", "webpki 0.22.0", - "webpki-roots 0.22.6", + "webpki-roots", ] [[package]] @@ -8042,7 +8165,7 @@ dependencies = [ "futures-io", "futures-sink", "log", - "pin-project-lite 0.2.10", + "pin-project-lite", "slab", "tokio", ] @@ -8056,7 +8179,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.10", + "pin-project-lite", "slab", "tokio", "tracing", @@ -8114,7 +8237,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project", - "pin-project-lite 0.2.10", + "pin-project-lite", "tokio", "tower-layer", "tower-service", @@ -8141,7 +8264,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", - "pin-project-lite 0.2.10", + "pin-project-lite", "tracing-attributes", "tracing-core", ] @@ -8936,15 +9059,6 @@ dependencies = [ "webpki 0.22.0", ] -[[package]] -name = "webpki-roots" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.1", -] - [[package]] name = "which" version = "4.4.0" @@ -9208,6 +9322,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x509-parser" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab0c2f54ae1d92f4fcb99c0b7ccf0b1e3451cbd395e5f115ccbdbcb18d4f634" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "rusticata-macros", + "thiserror", + "time 0.3.24", +] + [[package]] name = "xml-rs" version = "0.8.16" @@ -9246,6 +9377,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time 0.3.24", +] + [[package]] name = "zeroize" version = "1.6.0" diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index f71845fed25..b86749913fc 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -44,11 +44,12 @@ prometheus-client = "0.21.0" unused_port = { path = "../../common/unused_port" } delay_map = "0.3.0" void = "1" +libp2p-quic= { version = "0.9.0-alpha", features=["tokio"]} [dependencies.libp2p] version = "0.52" default-features = false -features = ["websocket", "identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"] +features = ["identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"] [dev-dependencies] slog-term = "2.6.0" diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 6c8f20a24b9..b1119f31966 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -107,6 +107,9 @@ pub struct Config { /// Disables the discovery protocol from starting. pub disable_discovery: bool, + /// Disables quic support. + pub disable_quic_support: bool, + /// Attempt to construct external port mappings with UPnP. pub upnp_enabled: bool, @@ -154,27 +157,41 @@ impl Config { /// Sets the listening address to use an ipv4 address. The discv5 ip_mode and table filter are /// adjusted accordingly to ensure addresses that are present in the enr are globally /// reachable. - pub fn set_ipv4_listening_address(&mut self, addr: Ipv4Addr, tcp_port: u16, udp_port: u16) { + pub fn set_ipv4_listening_address( + &mut self, + addr: Ipv4Addr, + tcp_port: u16, + disc_port: u16, + quic_port: u16, + ) { self.listen_addresses = ListenAddress::V4(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, }); - self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), udp_port); + self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); self.discv5_config.table_filter = |enr| enr.ip4().as_ref().map_or(false, is_global_ipv4) } /// Sets the listening address to use an ipv6 address. The discv5 ip_mode and table filter is /// adjusted accordingly to ensure addresses that are present in the enr are globally /// reachable. - pub fn set_ipv6_listening_address(&mut self, addr: Ipv6Addr, tcp_port: u16, udp_port: u16) { + pub fn set_ipv6_listening_address( + &mut self, + addr: Ipv6Addr, + tcp_port: u16, + disc_port: u16, + quic_port: u16, + ) { self.listen_addresses = ListenAddress::V6(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, }); - self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), udp_port); + self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); self.discv5_config.table_filter = |enr| enr.ip6().as_ref().map_or(false, is_global_ipv6) } @@ -185,26 +202,30 @@ impl Config { &mut self, v4_addr: Ipv4Addr, tcp4_port: u16, - udp4_port: u16, + disc4_port: u16, + quic4_port: u16, v6_addr: Ipv6Addr, tcp6_port: u16, - udp6_port: u16, + disc6_port: u16, + quic6_port: u16, ) { self.listen_addresses = ListenAddress::DualStack( ListenAddr { addr: v4_addr, - udp_port: udp4_port, + disc_port: disc4_port, + quic_port: quic4_port, tcp_port: tcp4_port, }, ListenAddr { addr: v6_addr, - udp_port: udp6_port, + disc_port: disc6_port, + quic_port: quic6_port, tcp_port: tcp6_port, }, ); self.discv5_config.listen_config = discv5::ListenConfig::default() - .with_ipv4(v4_addr, udp4_port) - .with_ipv6(v6_addr, udp6_port); + .with_ipv4(v4_addr, disc4_port) + .with_ipv6(v6_addr, disc6_port); self.discv5_config.table_filter = |enr| match (&enr.ip4(), &enr.ip6()) { (None, None) => false, @@ -218,27 +239,32 @@ impl Config { match listen_addr { ListenAddress::V4(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, - }) => self.set_ipv4_listening_address(addr, tcp_port, udp_port), + }) => self.set_ipv4_listening_address(addr, tcp_port, disc_port, quic_port), ListenAddress::V6(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, - }) => self.set_ipv6_listening_address(addr, tcp_port, udp_port), + }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port, quic_port), ListenAddress::DualStack( ListenAddr { addr: ip4addr, - udp_port: udp4_port, + disc_port: disc4_port, + quic_port: quic4_port, tcp_port: tcp4_port, }, ListenAddr { addr: ip6addr, - udp_port: udp6_port, + disc_port: disc6_port, + quic_port: quic6_port, tcp_port: tcp6_port, }, ) => self.set_ipv4_ipv6_listening_addresses( - ip4addr, tcp4_port, udp4_port, ip6addr, tcp6_port, udp6_port, + ip4addr, tcp4_port, disc4_port, quic4_port, ip6addr, tcp6_port, disc6_port, + quic6_port, ), } } @@ -277,7 +303,8 @@ impl Default for Config { ); let listen_addresses = ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, - udp_port: 9000, + disc_port: 9000, + quic_port: 9001, tcp_port: 9000, }); @@ -325,6 +352,7 @@ impl Default for Config { disable_peer_scoring: false, client_version: lighthouse_version::version_with_platform(), disable_discovery: false, + disable_quic_support: false, upnp_enabled: true, network_load: 3, private: false, @@ -560,4 +588,4 @@ pub const fn is_global_ipv6(addr: &Ipv6Addr) -> bool { || is_documentation(addr) || is_unique_local(addr) || is_unicast_link_local(addr)) -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 20d87d403cd..646df708efd 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -6,17 +6,26 @@ use serde::{Deserialize, Serialize}; /// A listening address composed by an Ip, an UDP port and a TCP port. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ListenAddr { + /// The IP address we will listen on. pub addr: Ip, - pub udp_port: u16, + /// The UDP port that discovery will listen on. + pub disc_port: u16, + /// The UDP port that QUIC will listen on. + pub quic_port: u16, + /// The TCP port that libp2p will listen on. pub tcp_port: u16, } impl + Clone> ListenAddr { - pub fn udp_socket_addr(&self) -> SocketAddr { - (self.addr.clone().into(), self.udp_port).into() + pub fn discovery_socket_addr(&self) -> SocketAddr { + (self.addr.clone().into(), self.disc_port).into() } - pub fn tcp_socket_addr(&self) -> SocketAddr { + pub fn quic_socket_addr(&self) -> SocketAddr { + (self.addr.clone().into(), self.quic_port).into() + } + + pub fn libp2p_socket_addr(&self) -> SocketAddr { (self.addr.clone().into(), self.tcp_port).into() } } @@ -61,7 +70,8 @@ impl ListenAddress { pub fn unused_v4_ports() -> Self { ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, - udp_port: unused_port::unused_udp4_port().unwrap(), + disc_port: unused_port::unused_udp4_port().unwrap(), + quic_port: unused_port::unused_udp4_port().unwrap(), tcp_port: unused_port::unused_tcp4_port().unwrap(), }) } @@ -70,7 +80,8 @@ impl ListenAddress { pub fn unused_v6_ports() -> Self { ListenAddress::V6(ListenAddr { addr: Ipv6Addr::UNSPECIFIED, - udp_port: unused_port::unused_udp6_port().unwrap(), + disc_port: unused_port::unused_udp6_port().unwrap(), + quic_port: unused_port::unused_udp6_port().unwrap(), tcp_port: unused_port::unused_tcp6_port().unwrap(), }) } @@ -84,14 +95,16 @@ impl slog::KV for ListenAddress { ) -> slog::Result { if let Some(v4_addr) = self.v4() { serializer.emit_arguments("ip4_address", &format_args!("{}", v4_addr.addr))?; - serializer.emit_u16("udp4_port", v4_addr.udp_port)?; + serializer.emit_u16("disc4_port", v4_addr.disc_port)?; + serializer.emit_u16("quic4_port", v4_addr.quic_port)?; serializer.emit_u16("tcp4_port", v4_addr.tcp_port)?; } if let Some(v6_addr) = self.v6() { serializer.emit_arguments("ip6_address", &format_args!("{}", v6_addr.addr))?; - serializer.emit_u16("udp6_port", v6_addr.udp_port)?; + serializer.emit_u16("disc6_port", v6_addr.disc_port)?; + serializer.emit_u16("quic6_port", v6_addr.quic_port)?; serializer.emit_u16("tcp6_port", v6_addr.tcp_port)?; } slog::Result::Ok(()) } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 21fd09b6b0f..6ceea7bec57 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -9,6 +9,7 @@ use libp2p::core::{multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxe use libp2p::gossipsub; use libp2p::identity::{secp256k1, Keypair}; use libp2p::{core, noise, yamux, PeerId, Transport, TransportExt}; +use libp2p_quic; use prometheus_client::registry::Registry; use slog::{debug, warn}; use ssz::Decode; @@ -37,18 +38,15 @@ pub struct Context<'a> { type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; -/// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and -/// mplex as the multiplexing layer. +/// The implementation supports TCP/IP, QUIC (experimental) over UDP, noise as the encryption layer, and +/// mplex/yamux as the multiplexing layer (when using TCP). pub fn build_transport( local_private_key: Keypair, ) -> std::io::Result<(BoxedTransport, Arc)> { + // Creates the TCP transport layer let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)); + // Enables DNS over the TCP transport. let transport = libp2p::dns::TokioDnsConfig::system(tcp)?; - #[cfg(feature = "libp2p-websocket")] - let transport = { - let trans_clone = transport.clone(); - transport.or_transport(libp2p::websocket::WsConfig::new(trans_clone)) - }; // yamux config let mut yamux_config = yamux::Config::default(); @@ -58,10 +56,19 @@ pub fn build_transport( .authenticate(generate_noise_config(&local_private_key)) .multiplex(yamux_config) .timeout(Duration::from_secs(10)) - .boxed() .with_bandwidth_logging(); - // Authentication + // Enables Quic + /* + // The default quic configuration suits us for now. + let quic_config = libp2p_quic::Config::new(&local_private_key); + let transport = transport.or_transport(libp2p_quic::tokio::Transport::new(quic_config)); + + // TODO: Get quick to support bandwidth measurements. + */ + + let transport = transport.boxed(); + Ok((transport, bandwidth)) } @@ -267,4 +274,4 @@ pub(crate) fn save_metadata_to_disk( ); } } -} +} \ No newline at end of file diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index 9bf123e8dec..99b43db0334 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -12,20 +12,26 @@ use types::EthSpec; /// Configuration required to construct the UPnP port mappings. pub struct UPnPConfig { - /// The local tcp port. + /// The local TCP port. tcp_port: u16, - /// The local udp port. - udp_port: u16, + /// The local UDP discovery port. + disc_port: u16, + /// The local UDP quic port. + quic_port: u16, /// Whether discovery is enabled or not. disable_discovery: bool, + /// Whether quic is enabled or not. + disable_quic_support: bool, } impl UPnPConfig { pub fn from_config(config: &NetworkConfig) -> Option { config.listen_addrs().v4().map(|v4_addr| UPnPConfig { tcp_port: v4_addr.tcp_port, - udp_port: v4_addr.udp_port, + disc_port: v4_addr.disc_port, + quic_port: v4_addr.quic_port, disable_discovery: config.disable_discovery, + disable_quic_support: config.disable_quic_support, }) } } @@ -88,26 +94,37 @@ pub fn construct_upnp_mappings( external_socket }).ok(); - let udp_socket = if !config.disable_discovery { - let discovery_socket = SocketAddrV4::new(address, config.udp_port); + let set_udp_mapping = |udp_port| { + let udp_socket = SocketAddrV4::new(address, udp_port); add_port_mapping( &gateway, igd::PortMappingProtocol::UDP, - discovery_socket, + udp_socket, "udp", &log, - ).and_then(|_| { - let external_socket = external_ip - .map(|ip| SocketAddr::new(ip.into(), config.udp_port)).map_err(|_| ()); - info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.udp_port)); - external_socket - }).ok() - } else { - None + ).map(|_| { + info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_ip.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), udp_port)); + }) }; + let mut udp_sockets = Vec::new(); + + // Set the discovery UDP port mapping + if !config.disable_discovery { + if set_udp_mapping(config.disc_port).is_ok() { + udp_sockets.push(config.disc_port); + } + } + + // Set the quic UDP port mapping + if !config.disable_quic_support { + if set_udp_mapping(config.quic_port).is_ok() { + udp_sockets.push(config.quic_port) + } + } + // report any updates to the network service. - network_send.send(NetworkMessage::UPnPMappingEstablished{ tcp_socket, udp_socket }) + network_send.send(NetworkMessage::UPnPMappingEstablished{ tcp_socket, udp_sockets }) .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e)); } _ => debug!(log, "UPnP no routes constructed. IPv6 not supported"), @@ -161,8 +178,8 @@ fn add_port_mapping( } /// Removes the specified TCP and UDP port mappings. -pub fn remove_mappings(tcp_port: Option, udp_port: Option, log: &slog::Logger) { - if tcp_port.is_some() || udp_port.is_some() { +pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Logger) { + if tcp_port.is_some() || !udp_ports.is_empty() { debug!(log, "Removing UPnP port mappings"); match igd::search_gateway(Default::default()) { Ok(gateway) => { @@ -174,8 +191,8 @@ pub fn remove_mappings(tcp_port: Option, udp_port: Option, log: &slog: } } } - if let Some(udp_port) = udp_port { - match gateway.remove_port(igd::PortMappingProtocol::UDP, udp_port) { + for udp_port in udp_ports { + match gateway.remove_port(igd::PortMappingProtocol::UDP, *udp_port) { Ok(()) => debug!(log, "UPnP Removed UDP port mapping"; "port" => udp_port), Err(e) => { debug!(log, "UPnP Failed to remove UDP port mapping"; "port" => udp_port, "error" => %e) @@ -186,4 +203,4 @@ pub fn remove_mappings(tcp_port: Option, udp_port: Option, log: &slog: Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), } } -} +} \ No newline at end of file diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index c355c671e80..aedcdb45918 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -97,8 +97,8 @@ pub enum NetworkMessage { UPnPMappingEstablished { /// The external TCP address has been updated. tcp_socket: Option, - /// The external UDP address has been updated. - udp_socket: Option, + /// The external UDP sockets have been established. + udp_sockets: Vec, }, /// Reports a peer to the peer manager for performing an action. ReportPeer { @@ -190,11 +190,8 @@ pub struct NetworkService { /// A collection of global variables, accessible outside of the network service. network_globals: Arc>, /// Stores potentially created UPnP mappings to be removed on shutdown. (TCP port and UDP - /// port). - upnp_mappings: (Option, Option), - /// Keeps track of if discovery is auto-updating or not. This is used to inform us if we should - /// update the UDP socket of discovery if the UPnP mappings get established. - discovery_auto_update: bool, + /// ports). + upnp_mappings: (Option, Vec), /// A delay that expires when a new fork takes place. next_fork_update: Pin>>, /// A delay that expires when we need to subscribe to a new fork's topics. @@ -359,8 +356,7 @@ impl NetworkService { router_send, store, network_globals: network_globals.clone(), - upnp_mappings: (None, None), - discovery_auto_update: config.discv5_config.enr_update, + upnp_mappings: (None, Vec::new()), next_fork_update, next_fork_subscriptions, next_unsubscribe, @@ -618,9 +614,9 @@ impl NetworkService { } NetworkMessage::UPnPMappingEstablished { tcp_socket, - udp_socket, + udp_sockets, } => { - self.upnp_mappings = (tcp_socket.map(|s| s.port()), udp_socket.map(|s| s.port())); + self.upnp_mappings = (tcp_socket.map(|s| s.port()), udp_sockets); // If there is an external TCP port update, modify our local ENR. if let Some(tcp_socket) = tcp_socket { if let Err(e) = self @@ -631,19 +627,6 @@ impl NetworkService { warn!(self.log, "Failed to update ENR"; "error" => e); } } - // if the discovery service is not auto-updating, update it with the - // UPnP mappings - if !self.discovery_auto_update { - if let Some(udp_socket) = udp_socket { - if let Err(e) = self - .libp2p - .discovery_mut() - .update_enr_udp_socket(udp_socket) - { - warn!(self.log, "Failed to update ENR"; "error" => e); - } - } - } } NetworkMessage::ValidationResult { propagation_source, @@ -994,8 +977,8 @@ impl Drop for NetworkService { } // attempt to remove port mappings - crate::nat::remove_mappings(self.upnp_mappings.0, self.upnp_mappings.1, &self.log); + crate::nat::remove_mappings(self.upnp_mappings.0, &(self.upnp_mappings.1), &self.log); info!(self.log, "Network service shutdown"); } -} +} \ No newline at end of file diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index e46c3d8ca11..a20e33c23f1 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -75,11 +75,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("The address lighthouse will listen for UDP and TCP connections. To listen \ over IpV4 and IpV6 set this flag twice with the different values.\n\ Examples:\n\ - - --listen-address '0.0.0.0' will listen over Ipv4.\n\ - - --listen-address '::' will listen over Ipv6.\n\ + - --listen-address '0.0.0.0' will listen over IPv4.\n\ + - --listen-address '::' will listen over IPv6.\n\ - --listen-address '0.0.0.0' --listen-address '::' will listen over both \ - Ipv4 and Ipv6. The order of the given addresses is not relevant. However, \ - multiple Ipv4, or multiple Ipv6 addresses will not be accepted.") + IPv4 and IPv6. The order of the given addresses is not relevant. However, \ + multiple IPv4, or multiple IPv6 addresses will not be accepted.") .multiple(true) .max_values(2) .default_value("0.0.0.0") @@ -89,9 +89,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("port") .long("port") .value_name("PORT") - .help("The TCP/UDP port to listen on. The UDP port can be modified by the \ - --discovery-port flag. If listening over both Ipv4 and Ipv6 the --port flag \ - will apply to the Ipv4 address and --port6 to the Ipv6 address.") + .help("The TCP/UDP ports to listen on. There are two UDP ports. The discovery UDP port will be set to this value and the Quic UDP port will be set to his value + 1. The discovery port can be modified by the \ + --discovery-port flag and the quic port can be modified by the --quic-port flag. If listening over both IPv4 and IPv6 the --port flag \ + will apply to the IPv4 address and --port6 to the IPv6 address.") .default_value("9000") .takes_value(true), ) @@ -99,8 +99,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("port6") .long("port6") .value_name("PORT") - .help("The TCP/UDP port to listen on over IpV6 when listening over both Ipv4 and \ - Ipv6. Defaults to 9090 when required.") + .help("The TCP/UDP ports to listen on over IPv6 when listening over both IPv4 and \ + IPv6. Defaults to 9090 when required. The Quic UDP port will be set to this value + 1.") .default_value("9090") .takes_value(true), ) @@ -111,12 +111,27 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("The UDP port that discovery will listen on. Defaults to `port`") .takes_value(true), ) + .arg( + Arg::with_name("quic-port") + .long("quic-port") + .value_name("PORT") + .help("The UDP port that quic will listen on. Defaults to `port` + 1") + .takes_value(true), + ) .arg( Arg::with_name("discovery-port6") .long("discovery-port6") .value_name("PORT") - .help("The UDP port that discovery will listen on over IpV6 if listening over \ - both Ipv4 and IpV6. Defaults to `port6`") + .help("The UDP port that discovery will listen on over IPv6 if listening over \ + both IPv4 and IPv6. Defaults to `port6`") + .takes_value(true), + ) + .arg( + Arg::with_name("quic-port6") + .long("quic-port6") + .value_name("PORT") + .help("The UDP port that quic will listen on over IPv6 if listening over \ + both IPv4 and IPv6. Defaults to `port6` + 1") .takes_value(true), ) .arg( @@ -225,11 +240,20 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { without an ENR.") .takes_value(true), ) + // NOTE: This is hidden because it is primarily a developer feature for testnets and + // debugging. We remove it from the list to avoid clutter. .arg( Arg::with_name("disable-discovery") .long("disable-discovery") .help("Disables the discv5 discovery protocol. The node will not search for new peers or participate in the discovery protocol.") .takes_value(false), + .hidden(true) + ) + .arg( + Arg::with_name("disable-quic") + .long("disable-quic") + .help("Disables the quic transport. The node will rely solely on the TCP transport for libp2p connections.") + .takes_value(false), ) .arg( Arg::with_name("disable-peer-scoring") @@ -1141,4 +1165,4 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .possible_values(ProgressiveBalancesMode::VARIANTS) ) -} +} \ No newline at end of file From 3eb493757b1431658251db1e748aad1afc3ebb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 2 Aug 2023 16:56:50 +0100 Subject: [PATCH 02/70] enable Quic transport on libp2p --- .../lighthouse_network/src/service/utils.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 6ceea7bec57..8b7c3cfbc7c 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -4,6 +4,7 @@ use crate::types::{ error, EnrAttestationBitfield, EnrSyncCommitteeBitfield, GossipEncoding, GossipKind, }; use crate::{GossipTopic, NetworkConfig}; +use futures::future::Either; use libp2p::bandwidth::BandwidthSinks; use libp2p::core::{multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed}; use libp2p::gossipsub; @@ -51,25 +52,24 @@ pub fn build_transport( // yamux config let mut yamux_config = yamux::Config::default(); yamux_config.set_window_update_mode(yamux::WindowUpdateMode::on_read()); - let (transport, bandwidth) = transport + let transport = transport .upgrade(core::upgrade::Version::V1) .authenticate(generate_noise_config(&local_private_key)) .multiplex(yamux_config) - .timeout(Duration::from_secs(10)) - .with_bandwidth_logging(); + .timeout(Duration::from_secs(10)); // Enables Quic - /* // The default quic configuration suits us for now. let quic_config = libp2p_quic::Config::new(&local_private_key); - let transport = transport.or_transport(libp2p_quic::tokio::Transport::new(quic_config)); - - // TODO: Get quick to support bandwidth measurements. - */ - - let transport = transport.boxed(); + let (transport, bandwidth) = transport + .or_transport(libp2p_quic::tokio::Transport::new(quic_config)) + .map(|either_output, _| match either_output { + Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + }) + .with_bandwidth_logging(); - Ok((transport, bandwidth)) + Ok((transport.boxed(), bandwidth)) } // Useful helper functions for debugging. Currently not used in the client. @@ -274,4 +274,4 @@ pub(crate) fn save_metadata_to_disk( ); } } -} \ No newline at end of file +} From 0ebdc792071e2e0d7e70da0ff37f8b84431e41ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 2 Aug 2023 16:08:57 +0100 Subject: [PATCH 03/70] allow support for the two different UDP ports, Discovery and Quic. Listen on both when starting th swarm. --- beacon_node/lighthouse_network/src/config.rs | 55 +++++++------- .../lighthouse_network/src/discovery/enr.rs | 4 +- .../lighthouse_network/src/listen_addr.rs | 54 +++++++++---- .../lighthouse_network/src/service/mod.rs | 4 +- .../lighthouse_network/tests/common.rs | 4 +- beacon_node/network/src/nat.rs | 6 +- beacon_node/src/cli.rs | 7 +- beacon_node/src/config.rs | 75 +++++++++++++------ boot_node/src/config.rs | 14 ++-- lcli/src/generate_bootnode_enr.rs | 4 +- testing/node_test_rig/src/lib.rs | 2 +- testing/simulator/src/local_network.rs | 9 ++- 12 files changed, 152 insertions(+), 86 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index b1119f31966..ae8bd6072e8 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -63,14 +63,14 @@ pub struct Config { /// that no discovery address has been set in the CLI args. pub enr_address: (Option, Option), - /// The udp4 port to broadcast to peers in order to reach back for discovery. - pub enr_udp4_port: Option, + /// The disc4 port to broadcast to peers in order to reach back for discovery. + pub enr_disc4_port: Option, /// The tcp4 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp4_port: Option, - /// The udp6 port to broadcast to peers in order to reach back for discovery. - pub enr_udp6_port: Option, + /// The disc6 port to broadcast to peers in order to reach back for discovery. + pub enr_disc6_port: Option, /// The tcp6 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp6_port: Option, @@ -167,7 +167,7 @@ impl Config { self.listen_addresses = ListenAddress::V4(ListenAddr { addr, disc_port, - quic_port, + quic_port: Some(quic_port), tcp_port, }); self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); @@ -177,17 +177,11 @@ impl Config { /// Sets the listening address to use an ipv6 address. The discv5 ip_mode and table filter is /// adjusted accordingly to ensure addresses that are present in the enr are globally /// reachable. - pub fn set_ipv6_listening_address( - &mut self, - addr: Ipv6Addr, - tcp_port: u16, - disc_port: u16, - quic_port: u16, - ) { + pub fn set_ipv6_listening_address(&mut self, addr: Ipv6Addr, tcp_port: u16, disc_port: u16) { self.listen_addresses = ListenAddress::V6(ListenAddr { addr, disc_port, - quic_port, + quic_port: None, tcp_port, }); @@ -198,6 +192,8 @@ impl Config { /// Sets the listening address to use both an ipv4 and ipv6 address. The discv5 ip_mode and /// table filter is adjusted accordingly to ensure addresses that are present in the enr are /// globally reachable. + /// TODO: add Quic support IPV6 port when https://github.com/libp2p/rust-libp2p/issues/4165 + /// gets addressed. pub fn set_ipv4_ipv6_listening_addresses( &mut self, v4_addr: Ipv4Addr, @@ -207,19 +203,18 @@ impl Config { v6_addr: Ipv6Addr, tcp6_port: u16, disc6_port: u16, - quic6_port: u16, ) { self.listen_addresses = ListenAddress::DualStack( ListenAddr { addr: v4_addr, disc_port: disc4_port, - quic_port: quic4_port, + quic_port: Some(quic4_port), tcp_port: tcp4_port, }, ListenAddr { addr: v6_addr, disc_port: disc6_port, - quic_port: quic6_port, + quic_port: None, tcp_port: tcp6_port, }, ); @@ -242,13 +237,18 @@ impl Config { disc_port, quic_port, tcp_port, - }) => self.set_ipv4_listening_address(addr, tcp_port, disc_port, quic_port), + }) => self.set_ipv4_listening_address( + addr, + tcp_port, + disc_port, + quic_port.expect("Quic port should exist on an IPV4 address"), + ), ListenAddress::V6(ListenAddr { addr, disc_port, - quic_port, + quic_port: _, tcp_port, - }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port, quic_port), + }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port), ListenAddress::DualStack( ListenAddr { addr: ip4addr, @@ -259,12 +259,17 @@ impl Config { ListenAddr { addr: ip6addr, disc_port: disc6_port, - quic_port: quic6_port, + quic_port: _quic6_port, tcp_port: tcp6_port, }, ) => self.set_ipv4_ipv6_listening_addresses( - ip4addr, tcp4_port, disc4_port, quic4_port, ip6addr, tcp6_port, disc6_port, - quic6_port, + ip4addr, + tcp4_port, + disc4_port, + quic4_port.expect("Quic port should exist on an IPV4 address"), + ip6addr, + tcp6_port, + disc6_port, ), } } @@ -304,7 +309,7 @@ impl Default for Config { let listen_addresses = ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, disc_port: 9000, - quic_port: 9001, + quic_port: Some(9001), tcp_port: 9000, }); @@ -338,9 +343,9 @@ impl Default for Config { listen_addresses, enr_address: (None, None), - enr_udp4_port: None, + enr_disc4_port: None, enr_tcp4_port: None, - enr_udp6_port: None, + enr_disc6_port: None, enr_tcp6_port: None, target_peers: 50, gs_config, diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index ef22f816a77..0ec061eac7a 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -155,11 +155,11 @@ pub fn create_enr_builder_from_config( builder.ip6(*ip); } - if let Some(udp4_port) = config.enr_udp4_port { + if let Some(udp4_port) = config.enr_disc4_port { builder.udp4(udp4_port); } - if let Some(udp6_port) = config.enr_udp6_port { + if let Some(udp6_port) = config.enr_disc6_port { builder.udp6(udp6_port); } diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 646df708efd..777dd178e89 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -11,7 +11,9 @@ pub struct ListenAddr { /// The UDP port that discovery will listen on. pub disc_port: u16, /// The UDP port that QUIC will listen on. - pub quic_port: u16, + /// NB: Quic port is optional as it's not yet supported with IPV6 + /// See https://github.com/libp2p/rust-libp2p/issues/4165 + pub quic_port: Option, /// The TCP port that libp2p will listen on. pub tcp_port: u16, } @@ -21,8 +23,20 @@ impl + Clone> ListenAddr { (self.addr.clone().into(), self.disc_port).into() } - pub fn quic_socket_addr(&self) -> SocketAddr { - (self.addr.clone().into(), self.quic_port).into() + pub fn quic_socket_addr(&self) -> Option { + let addr: IpAddr = self.addr.clone().into(); + match addr { + IpAddr::V4(ip) => { + let addr = ( + ip, + self.quic_port + .expect("Quic port should exist on an IPV4 address"), + ) + .into(); + Some(addr) + } + IpAddr::V6(_) => None, + } } pub fn libp2p_socket_addr(&self) -> SocketAddr { @@ -55,15 +69,24 @@ impl ListenAddress { } } - /// Returns the TCP addresses. - pub fn tcp_addresses(&self) -> impl Iterator + '_ { - let v4_multiaddr = self + /// Returns the Listen addresses. + pub fn listen_addresses(&self) -> impl Iterator { + let v4_tcp_multiaddrs = self + .v4() + .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); + + let v4_quic_multiaddrs = self .v4() .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); - let v6_multiaddr = self + + let v6_tcp_multiaddrs = self .v6() .map(|v6_addr| Multiaddr::from(v6_addr.addr).with(Protocol::Tcp(v6_addr.tcp_port))); - v4_multiaddr.into_iter().chain(v6_multiaddr) + + v4_tcp_multiaddrs + .into_iter() + .chain(v4_quic_multiaddrs) + .chain(v6_tcp_multiaddrs) } #[cfg(test)] @@ -71,7 +94,7 @@ impl ListenAddress { ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, disc_port: unused_port::unused_udp4_port().unwrap(), - quic_port: unused_port::unused_udp4_port().unwrap(), + quic_port: unused_port::unused_udp4_port().transpose().unwrap(), tcp_port: unused_port::unused_tcp4_port().unwrap(), }) } @@ -81,7 +104,7 @@ impl ListenAddress { ListenAddress::V6(ListenAddr { addr: Ipv6Addr::UNSPECIFIED, disc_port: unused_port::unused_udp6_port().unwrap(), - quic_port: unused_port::unused_udp6_port().unwrap(), + quic_port: None, tcp_port: unused_port::unused_tcp6_port().unwrap(), }) } @@ -96,15 +119,20 @@ impl slog::KV for ListenAddress { if let Some(v4_addr) = self.v4() { serializer.emit_arguments("ip4_address", &format_args!("{}", v4_addr.addr))?; serializer.emit_u16("disc4_port", v4_addr.disc_port)?; - serializer.emit_u16("quic4_port", v4_addr.quic_port)?; + serializer.emit_u16( + "quic4_port", + v4_addr + .quic_port + .expect("Quic port should exist on an IPV4 address"), + )?; serializer.emit_u16("tcp4_port", v4_addr.tcp_port)?; } if let Some(v6_addr) = self.v6() { serializer.emit_arguments("ip6_address", &format_args!("{}", v6_addr.addr))?; serializer.emit_u16("disc6_port", v6_addr.disc_port)?; - serializer.emit_u16("quic6_port", v6_addr.quic_port)?; + serializer.emit_none("quic6_port")?; serializer.emit_u16("tcp6_port", v6_addr.tcp_port)?; } slog::Result::Ok(()) } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 1a25beee0a7..04ed3a8b5b2 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -399,7 +399,7 @@ impl Network { info!(self.log, "Libp2p Starting"; "peer_id" => %enr.peer_id(), "bandwidth_config" => format!("{}-{}", config.network_load, NetworkLoad::from(config.network_load).name)); debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery); - for listen_multiaddr in config.listen_addrs().tcp_addresses() { + for listen_multiaddr in config.listen_addrs().listen_addresses() { match self.swarm.listen_on(listen_multiaddr.clone()) { Ok(_) => { let mut log_address = listen_multiaddr; @@ -1569,4 +1569,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index b48891335cc..9fccf006ea1 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -75,8 +75,8 @@ pub fn build_config(port: u16, mut boot_nodes: Vec) -> NetworkConfig { .tempdir() .unwrap(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port); - config.enr_udp4_port = Some(port); + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port, port + 1); + config.enr_disc4_port = Some(port); config.enr_address = (Some(std::net::Ipv4Addr::LOCALHOST), None); config.boot_nodes_enr.append(&mut boot_nodes); config.network_dir = path.into_path(); diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index 99b43db0334..f20ad8c542a 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -29,7 +29,9 @@ impl UPnPConfig { config.listen_addrs().v4().map(|v4_addr| UPnPConfig { tcp_port: v4_addr.tcp_port, disc_port: v4_addr.disc_port, - quic_port: v4_addr.quic_port, + quic_port: v4_addr + .quic_port + .expect("Quic port should exist on an IPV4 address"), disable_discovery: config.disable_discovery, disable_quic_support: config.disable_quic_support, }) @@ -203,4 +205,4 @@ pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Log Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), } } -} \ No newline at end of file +} diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index a20e33c23f1..3c126bc727e 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -89,7 +89,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("port") .long("port") .value_name("PORT") - .help("The TCP/UDP ports to listen on. There are two UDP ports. The discovery UDP port will be set to this value and the Quic UDP port will be set to his value + 1. The discovery port can be modified by the \ + .help("The TCP/UDP ports to listen on. There are two UDP ports. \ + The discovery UDP port will be set to this value and the Quic UDP port will be set to this value + 1. The discovery port can be modified by the \ --discovery-port flag and the quic port can be modified by the --quic-port flag. If listening over both IPv4 and IPv6 the --port flag \ will apply to the IPv4 address and --port6 to the IPv6 address.") .default_value("9000") @@ -246,7 +247,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("disable-discovery") .long("disable-discovery") .help("Disables the discv5 discovery protocol. The node will not search for new peers or participate in the discovery protocol.") - .takes_value(false), + .takes_value(false) .hidden(true) ) .arg( @@ -1165,4 +1166,4 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .possible_values(ProgressiveBalancesMode::VARIANTS) ) -} \ No newline at end of file +} diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 4abf649bfbb..76633290e3a 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -869,15 +869,15 @@ pub fn parse_listening_addresses( .map_err(|parse_error| format!("Failed to parse --port6 as an integer: {parse_error}"))? .unwrap_or(9090); - // parse the possible udp ports - let maybe_udp_port = cli_args + // parse the possible discovery ports. + let maybe_disc_port = cli_args .value_of("discovery-port") .map(str::parse::) .transpose() .map_err(|parse_error| { format!("Failed to parse --discovery-port as an integer: {parse_error}") })?; - let maybe_udp6_port = cli_args + let maybe_disc6_port = cli_args .value_of("discovery-port6") .map(str::parse::) .transpose() @@ -885,6 +885,15 @@ pub fn parse_listening_addresses( format!("Failed to parse --discovery-port6 as an integer: {parse_error}") })?; + // parse the possible quic port. + let maybe_quic_port = cli_args + .value_of("quic-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic-port as an integer: {parse_error}") + })?; + // Now put everything together let listening_addresses = match (maybe_ipv4, maybe_ipv6) { (None, None) => { @@ -903,20 +912,21 @@ pub fn parse_listening_addresses( .transpose()? .unwrap_or(port); - if maybe_udp6_port.is_some() { + if maybe_disc6_port.is_some() { warn!(log, "When listening only over IpV6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") } // use zero ports if required. If not, use the specific udp port. If none given, use // the tcp port. - let udp_port = use_zero_ports + let disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? - .or(maybe_udp_port) + .or(maybe_disc_port) .unwrap_or(port); ListenAddress::V6(lighthouse_network::ListenAddr { addr: ipv6, - udp_port, + quic_port: None, + disc_port, tcp_port, }) } @@ -928,16 +938,25 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp4_port) .transpose()? .unwrap_or(port); - // use zero ports if required. If not, use the specific udp port. If none given, use + // use zero ports if required. If not, use the specific discovery port. If none given, use // the tcp port. - let udp_port = use_zero_ports + let disc_port = use_zero_ports .then(unused_port::unused_udp4_port) .transpose()? - .or(maybe_udp_port) + .or(maybe_disc_port) .unwrap_or(port); + // use zero ports if required. If not, use the specific quic port. If none given, use + // the tcp port + 1. + let quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + ListenAddress::V4(lighthouse_network::ListenAddr { addr: ipv4, - udp_port, + disc_port, + quic_port: Some(quic_port), tcp_port, }) } @@ -946,31 +965,39 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp4_port) .transpose()? .unwrap_or(port); - let ipv4_udp_port = use_zero_ports + let ipv4_disc_port = use_zero_ports .then(unused_port::unused_udp4_port) .transpose()? - .or(maybe_udp_port) + .or(maybe_disc_port) .unwrap_or(ipv4_tcp_port); + let ipv4_quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); // Defaults to 9090 when required let ipv6_tcp_port = use_zero_ports .then(unused_port::unused_tcp6_port) .transpose()? .unwrap_or(port6); - let ipv6_udp_port = use_zero_ports + let ipv6_disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? - .or(maybe_udp6_port) + .or(maybe_disc6_port) .unwrap_or(ipv6_tcp_port); + ListenAddress::DualStack( lighthouse_network::ListenAddr { addr: ipv4, - udp_port: ipv4_udp_port, + disc_port: ipv4_disc_port, + quic_port: Some(ipv4_quic_port), tcp_port: ipv4_tcp_port, }, lighthouse_network::ListenAddr { addr: ipv6, - udp_port: ipv6_udp_port, + disc_port: ipv6_disc_port, + quic_port: None, tcp_port: ipv6_tcp_port, }, ) @@ -1079,7 +1106,7 @@ pub fn set_network_config( } if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp-port") { - config.enr_udp4_port = Some( + config.enr_disc4_port = Some( enr_udp_port_str .parse::() .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?, @@ -1095,7 +1122,7 @@ pub fn set_network_config( } if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp6-port") { - config.enr_udp6_port = Some( + config.enr_disc6_port = Some( enr_udp_port_str .parse::() .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?, @@ -1121,7 +1148,7 @@ pub fn set_network_config( ipv4_addr.addr }; config.enr_address.0 = Some(ipv4_enr_addr); - config.enr_udp4_port = Some(ipv4_addr.udp_port); + config.enr_disc4_port = Some(ipv4_addr.disc_port); } if let Some(ipv6_addr) = config.listen_addrs().v6().cloned() { @@ -1131,7 +1158,7 @@ pub fn set_network_config( ipv6_addr.addr }; config.enr_address.1 = Some(ipv6_enr_addr); - config.enr_udp6_port = Some(ipv6_addr.udp_port); + config.enr_disc6_port = Some(ipv6_addr.disc_port); } } @@ -1164,11 +1191,11 @@ pub fn set_network_config( // actually matters. Just use the udp port. let port = match config.listen_addrs() { - ListenAddress::V4(v4_addr) => v4_addr.udp_port, - ListenAddress::V6(v6_addr) => v6_addr.udp_port, + ListenAddress::V4(v4_addr) => v4_addr.disc_port, + ListenAddress::V6(v6_addr) => v6_addr.disc_port, ListenAddress::DualStack(v4_addr, _v6_addr) => { // NOTE: slight preference for ipv4 that I don't think is of importance. - v4_addr.udp_port + v4_addr.disc_port } }; diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index d006156bf9d..7c5f2dd0f52 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -57,20 +57,20 @@ impl BootNodeConfig { set_network_config(&mut network_config, matches, &data_dir, &logger)?; - // Set the Enr UDP ports to the listening ports if not present. + // Set the Enr Discovery ports to the listening ports if not present. if let Some(listening_addr_v4) = network_config.listen_addrs().v4() { - network_config.enr_udp4_port = Some( + network_config.enr_disc4_port = Some( network_config - .enr_udp4_port - .unwrap_or(listening_addr_v4.udp_port), + .enr_disc4_port + .unwrap_or(listening_addr_v4.disc_port), ) }; if let Some(listening_addr_v6) = network_config.listen_addrs().v6() { - network_config.enr_udp6_port = Some( + network_config.enr_disc6_port = Some( network_config - .enr_udp6_port - .unwrap_or(listening_addr_v6.udp_port), + .enr_disc6_port + .unwrap_or(listening_addr_v6.disc_port), ) }; diff --git a/lcli/src/generate_bootnode_enr.rs b/lcli/src/generate_bootnode_enr.rs index 0584cd65496..54294f2b42c 100644 --- a/lcli/src/generate_bootnode_enr.rs +++ b/lcli/src/generate_bootnode_enr.rs @@ -27,7 +27,7 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { let mut config = NetworkConfig::default(); config.enr_address = (Some(ip), None); - config.enr_udp4_port = Some(udp_port); + config.enr_disc4_port = Some(udp_port); config.enr_tcp6_port = Some(tcp_port); let secp256k1_keypair = secp256k1::Keypair::generate(); @@ -57,4 +57,4 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { .map_err(|e| format!("Unable to write key to {}: {:?}", NETWORK_KEY_FILENAME, e))?; Ok(()) -} +} \ No newline at end of file diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index 62db67b8c57..746992f3207 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -98,7 +98,7 @@ pub fn testing_client_config() -> ClientConfig { // Setting ports to `0` means that the OS will choose some available port. client_config .network - .set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 0, 0); + .set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 0, 0, 0); client_config.network.upnp_enabled = false; client_config.http_api.enabled = true; client_config.http_api.listen_port = 0; diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index e35870d126c..9d03de14716 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -14,6 +14,7 @@ use std::{sync::Arc, time::Duration}; use types::{Epoch, EthSpec}; const BOOTNODE_PORT: u16 = 42424; +const QUIC_PORT: u16 = BOOTNODE_PORT + 1; pub const INVALID_ADDRESS: &str = "http://127.0.0.1:42423"; pub const EXECUTION_PORT: u16 = 4000; @@ -63,8 +64,9 @@ impl LocalNetwork { std::net::Ipv4Addr::UNSPECIFIED, BOOTNODE_PORT, BOOTNODE_PORT, + QUIC_PORT, ); - beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT); + beacon_config.network.enr_disc4_port = Some(BOOTNODE_PORT); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT); beacon_config.network.discv5_config.table_filter = |_| true; @@ -154,8 +156,9 @@ impl LocalNetwork { std::net::Ipv4Addr::UNSPECIFIED, BOOTNODE_PORT + count, BOOTNODE_PORT + count, + BOOTNODE_PORT + count + 1, ); - beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT + count); + beacon_config.network.enr_disc4_port = Some(BOOTNODE_PORT + count); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT + count); beacon_config.network.discv5_config.table_filter = |_| true; beacon_config.network.proposer_only = is_proposer; @@ -312,4 +315,4 @@ impl LocalNetwork { ); genesis_time - now } -} +} \ No newline at end of file From 29db04d2e807c4245b45491e7e9189762e66cfb5 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 3 Aug 2023 16:16:53 +1000 Subject: [PATCH 04/70] Add ENR structure for quick and specify listening criteria --- beacon_node/lighthouse_network/src/config.rs | 41 +- .../src/discovery/enr_ext.rs | 64 ++- beacon_node/lighthouse_network/src/lib.rs | 2 +- .../lighthouse_network/src/listen_addr.rs | 48 +- .../lighthouse_network/src/service/mod.rs | 27 +- .../lighthouse_network/src/service/utils.rs | 27 +- beacon_node/network/src/nat.rs | 6 +- beacon_node/src/config.rs | 424 ++++++++++-------- 8 files changed, 374 insertions(+), 265 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index ae8bd6072e8..b0ed8b6dcf6 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -167,7 +167,7 @@ impl Config { self.listen_addresses = ListenAddress::V4(ListenAddr { addr, disc_port, - quic_port: Some(quic_port), + quic_port, tcp_port, }); self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); @@ -177,11 +177,17 @@ impl Config { /// Sets the listening address to use an ipv6 address. The discv5 ip_mode and table filter is /// adjusted accordingly to ensure addresses that are present in the enr are globally /// reachable. - pub fn set_ipv6_listening_address(&mut self, addr: Ipv6Addr, tcp_port: u16, disc_port: u16) { + pub fn set_ipv6_listening_address( + &mut self, + addr: Ipv6Addr, + tcp_port: u16, + disc_port: u16, + quic_port: u16, + ) { self.listen_addresses = ListenAddress::V6(ListenAddr { addr, disc_port, - quic_port: None, + quic_port, tcp_port, }); @@ -202,19 +208,20 @@ impl Config { quic4_port: u16, v6_addr: Ipv6Addr, tcp6_port: u16, + quic6_port: u16, disc6_port: u16, ) { self.listen_addresses = ListenAddress::DualStack( ListenAddr { addr: v4_addr, disc_port: disc4_port, - quic_port: Some(quic4_port), + quic_port: quic4_port, tcp_port: tcp4_port, }, ListenAddr { addr: v6_addr, disc_port: disc6_port, - quic_port: None, + quic_port: quic6_port, tcp_port: tcp6_port, }, ); @@ -237,18 +244,13 @@ impl Config { disc_port, quic_port, tcp_port, - }) => self.set_ipv4_listening_address( - addr, - tcp_port, - disc_port, - quic_port.expect("Quic port should exist on an IPV4 address"), - ), + }) => self.set_ipv4_listening_address(addr, tcp_port, disc_port, quic_port), ListenAddress::V6(ListenAddr { addr, disc_port, - quic_port: _, + quic_port, tcp_port, - }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port), + }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port, quic_port), ListenAddress::DualStack( ListenAddr { addr: ip4addr, @@ -259,17 +261,12 @@ impl Config { ListenAddr { addr: ip6addr, disc_port: disc6_port, - quic_port: _quic6_port, + quic_port: quic6_port, tcp_port: tcp6_port, }, ) => self.set_ipv4_ipv6_listening_addresses( - ip4addr, - tcp4_port, - disc4_port, - quic4_port.expect("Quic port should exist on an IPV4 address"), - ip6addr, - tcp6_port, - disc6_port, + ip4addr, tcp4_port, disc4_port, quic4_port, ip6addr, tcp6_port, disc6_port, + quic6_port, ), } } @@ -309,7 +306,7 @@ impl Default for Config { let listen_addresses = ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, disc_port: 9000, - quic_port: Some(9001), + quic_port: 9001, tcp_port: 9000, }); diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 753da6292ca..96304c011c8 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -6,12 +6,15 @@ use libp2p::core::multiaddr::Protocol; use libp2p::identity::{ed25519, secp256k1, KeyType, Keypair, PublicKey}; use tiny_keccak::{Hasher, Keccak}; +const QUIC_ENR_KEY: &'static str = "quic"; +const QUIC6_ENR_KEY: &'static str = "quic6"; + /// Extend ENR for libp2p types. pub trait EnrExt { /// The libp2p `PeerId` for the record. fn peer_id(&self) -> PeerId; - /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. + /// Returns a list of multiaddrs if the ENR has an `ip` and one of [`tcp`,`udp`,`quic`] key **or** an `ip6` and one of [`tcp6`,`udp6`,`quic6`]. /// The vector remains empty if these fields are not defined. fn multiaddr(&self) -> Vec; @@ -26,6 +29,15 @@ pub trait EnrExt { /// Returns any multiaddrs that contain the TCP protocol. fn multiaddr_tcp(&self) -> Vec; + + /// Returns any QUIC multiaddrs that are registered in this ENR. + fn multiaddr_quic(&self) -> Vec; + + /// Returns the quic port if one is set. + fn quic(&self) -> Option; + + /// Returns the quic6 port if one is set. + fn quic6(&self) -> Option; } /// Extend ENR CombinedPublicKey for libp2p types. @@ -49,6 +61,16 @@ impl EnrExt for Enr { self.public_key().as_peer_id() } + /// Returns the quic port if one is set. + fn quic(&self) -> Option { + self.get_decodable(QUIC_ENR_KEY).and_then(Result::ok) + } + + /// Returns the quic6 port if one is set. + fn quic6(&self) -> Option { + self.get_decodable(QUIC6_ENR_KEY).and_then(Result::ok) + } + /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. /// The vector remains empty if these fields are not defined. fn multiaddr(&self) -> Vec { @@ -59,6 +81,12 @@ impl EnrExt for Enr { multiaddr.push(Protocol::Udp(udp)); multiaddrs.push(multiaddr); } + if let Some(quic) = self.quic() { + let mut multiaddr: Multiaddr = ip.into(); + multiaddr.push(Protocol::Udp(quic)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } if let Some(tcp) = self.tcp4() { let mut multiaddr: Multiaddr = ip.into(); @@ -73,6 +101,13 @@ impl EnrExt for Enr { multiaddrs.push(multiaddr); } + if let Some(quic6) = self.quic6() { + let mut multiaddr: Multiaddr = ip6.into(); + multiaddr.push(Protocol::Udp(quic6)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } + if let Some(tcp6) = self.tcp6() { let mut multiaddr: Multiaddr = ip6.into(); multiaddr.push(Protocol::Tcp(tcp6)); @@ -174,6 +209,31 @@ impl EnrExt for Enr { multiaddrs } + /// Returns a list of multiaddrs if the ENR has an `ip` and a `quic` key **or** an `ip6` and a `quic6`. + /// The vector remains empty if these fields are not defined. + fn multiaddr_quic(&self) -> Vec { + let mut multiaddrs: Vec = Vec::new(); + // Check for quic first as it is less likely + if let Some(quic_port) = self.quic() { + if let Some(ip) = self.ip4() { + let mut multiaddr: Multiaddr = ip.into(); + multiaddr.push(Protocol::Udp(quic_port)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } + } + + if let Some(quic6_port) = self.quic6() { + if let Some(ip6) = self.ip6() { + let mut multiaddr: Multiaddr = ip6.into(); + multiaddr.push(Protocol::Udp(quic6_port)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } + } + multiaddrs + } + /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. /// The vector remains empty if these fields are not defined. fn multiaddr_tcp(&self) -> Vec { @@ -337,4 +397,4 @@ mod tests { assert_eq!(enr.node_id(), node_id); } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/lib.rs b/beacon_node/lighthouse_network/src/lib.rs index 3d539af3b28..85b9f9126a0 100644 --- a/beacon_node/lighthouse_network/src/lib.rs +++ b/beacon_node/lighthouse_network/src/lib.rs @@ -88,4 +88,4 @@ pub use peer_manager::{ // pub use service::{load_private_key, Context, Libp2pEvent, Service, NETWORK_KEY_FILENAME}; pub use service::api_types::{PeerRequestId, Request, Response}; pub use service::utils::*; -pub use service::{Gossipsub, NetworkEvent}; +pub use service::{Gossipsub, NetworkEvent}; \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 777dd178e89..6b4b494a2ea 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -11,9 +11,10 @@ pub struct ListenAddr { /// The UDP port that discovery will listen on. pub disc_port: u16, /// The UDP port that QUIC will listen on. - /// NB: Quic port is optional as it's not yet supported with IPV6 + /// NB: Quic port is not yet supported with IPV6 /// See https://github.com/libp2p/rust-libp2p/issues/4165 - pub quic_port: Option, + /// It will therefore be disabled at the swarm level. + pub quic_port: u16, /// The TCP port that libp2p will listen on. pub tcp_port: u16, } @@ -23,20 +24,8 @@ impl + Clone> ListenAddr { (self.addr.clone().into(), self.disc_port).into() } - pub fn quic_socket_addr(&self) -> Option { - let addr: IpAddr = self.addr.clone().into(); - match addr { - IpAddr::V4(ip) => { - let addr = ( - ip, - self.quic_port - .expect("Quic port should exist on an IPV4 address"), - ) - .into(); - Some(addr) - } - IpAddr::V6(_) => None, - } + pub fn quic_socket_addr(&self) -> SocketAddr { + (self.addr.clone().into(), self.quic_port).into() } pub fn libp2p_socket_addr(&self) -> SocketAddr { @@ -69,20 +58,24 @@ impl ListenAddress { } } - /// Returns the Listen addresses. + /// Returns the addresses the Swarm will listen on, given the setup. pub fn listen_addresses(&self) -> impl Iterator { let v4_tcp_multiaddrs = self .v4() .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); - let v4_quic_multiaddrs = self - .v4() - .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); + let v4_quic_multiaddrs = self.v4().map(|v4_addr| { + Multiaddr::from(v4_addr.addr) + .with(Protocol::Udp(v4_addr.tcp_port)) + .with(Protocol::QuicV1) + }); let v6_tcp_multiaddrs = self .v6() .map(|v6_addr| Multiaddr::from(v6_addr.addr).with(Protocol::Tcp(v6_addr.tcp_port))); + // TODO: Add QUIC IPv6 multiaddr once it is supported + v4_tcp_multiaddrs .into_iter() .chain(v4_quic_multiaddrs) @@ -94,7 +87,7 @@ impl ListenAddress { ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, disc_port: unused_port::unused_udp4_port().unwrap(), - quic_port: unused_port::unused_udp4_port().transpose().unwrap(), + quic_port: unused_port::unused_udp4_port().unwrap(), tcp_port: unused_port::unused_tcp4_port().unwrap(), }) } @@ -104,7 +97,7 @@ impl ListenAddress { ListenAddress::V6(ListenAddr { addr: Ipv6Addr::UNSPECIFIED, disc_port: unused_port::unused_udp6_port().unwrap(), - quic_port: None, + quic_port: unused_port::unused_tcp6_port().unwrap(), tcp_port: unused_port::unused_tcp6_port().unwrap(), }) } @@ -119,20 +112,15 @@ impl slog::KV for ListenAddress { if let Some(v4_addr) = self.v4() { serializer.emit_arguments("ip4_address", &format_args!("{}", v4_addr.addr))?; serializer.emit_u16("disc4_port", v4_addr.disc_port)?; - serializer.emit_u16( - "quic4_port", - v4_addr - .quic_port - .expect("Quic port should exist on an IPV4 address"), - )?; + serializer.emit_u16("quic4_port", v4_addr.quic_port)?; serializer.emit_u16("tcp4_port", v4_addr.tcp_port)?; } if let Some(v6_addr) = self.v6() { serializer.emit_arguments("ip6_address", &format_args!("{}", v6_addr.addr))?; serializer.emit_u16("disc6_port", v6_addr.disc_port)?; - serializer.emit_none("quic6_port")?; + serializer.emit_u16("quic6_port", v6_addr.quic_port)?; serializer.emit_u16("tcp6_port", v6_addr.tcp_port)?; } slog::Result::Ok(()) } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 04ed3a8b5b2..809b05c55d6 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -341,7 +341,7 @@ impl Network { let (swarm, bandwidth) = { // Set up the transport - tcp/ws with noise and mplex - let (transport, bandwidth) = build_transport(local_keypair.clone()) + let (transport, bandwidth) = build_transport(local_keypair.clone(), !config.disable_quic_support) .map_err(|e| format!("Failed to build transport: {:?}", e))?; // use the executor for libp2p @@ -397,9 +397,17 @@ impl Network { async fn start(&mut self, config: &crate::NetworkConfig) -> error::Result<()> { let enr = self.network_globals.local_enr(); info!(self.log, "Libp2p Starting"; "peer_id" => %enr.peer_id(), "bandwidth_config" => format!("{}-{}", config.network_load, NetworkLoad::from(config.network_load).name)); - debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery); + debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery, "quic_enabled" => !config.disable_quic_support); for listen_multiaddr in config.listen_addrs().listen_addresses() { + + // If QUIC is disabled, ignore listening on QUIC ports + if config.disable_quic_support { + if listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) { + continue; + } + } + match self.swarm.listen_on(listen_multiaddr.clone()) { Ok(_) => { let mut log_address = listen_multiaddr; @@ -440,6 +448,21 @@ impl Network { boot_nodes.dedup(); for bootnode_enr in boot_nodes { + // If QUIC is enabled, attempt QUIC connections first + if !config.disable_quic_support { + for quic_multiaddr in &bootnode_enr.multiaddr_quic() { + if !self + .network_globals + .peers + .read() + .is_connected_or_dialing(&bootnode_enr.peer_id()) + { + dial(quic_multiaddr.clone()); + } + } + } + + // for multiaddr in &bootnode_enr.multiaddr() { // ignore udp multiaddr if it exists let components = multiaddr.iter().collect::>(); diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 8b7c3cfbc7c..79c6afc83b1 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -43,6 +43,7 @@ type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; /// mplex/yamux as the multiplexing layer (when using TCP). pub fn build_transport( local_private_key: Keypair, + quic_support: bool, ) -> std::io::Result<(BoxedTransport, Arc)> { // Creates the TCP transport layer let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)); @@ -58,16 +59,20 @@ pub fn build_transport( .multiplex(yamux_config) .timeout(Duration::from_secs(10)); - // Enables Quic - // The default quic configuration suits us for now. - let quic_config = libp2p_quic::Config::new(&local_private_key); - let (transport, bandwidth) = transport - .or_transport(libp2p_quic::tokio::Transport::new(quic_config)) - .map(|either_output, _| match either_output { - Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), - Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), - }) - .with_bandwidth_logging(); + let (transport, bandwidth) = if quic_support { + // Enables Quic + // The default quic configuration suits us for now. + let quic_config = libp2p_quic::Config::new(&local_private_key); + transport + .or_transport(libp2p_quic::tokio::Transport::new(quic_config)) + .map(|either_output, _| match either_output { + Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + }) + .with_bandwidth_logging() + } else { + transport.with_bandwidth_logging() + }; Ok((transport.boxed(), bandwidth)) } @@ -274,4 +279,4 @@ pub(crate) fn save_metadata_to_disk( ); } } -} +} \ No newline at end of file diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index f20ad8c542a..99b43db0334 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -29,9 +29,7 @@ impl UPnPConfig { config.listen_addrs().v4().map(|v4_addr| UPnPConfig { tcp_port: v4_addr.tcp_port, disc_port: v4_addr.disc_port, - quic_port: v4_addr - .quic_port - .expect("Quic port should exist on an IPV4 address"), + quic_port: v4_addr.quic_port, disable_discovery: config.disable_discovery, disable_quic_support: config.disable_quic_support, }) @@ -205,4 +203,4 @@ pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Log Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), } } -} +} \ No newline at end of file diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 76633290e3a..9689b2ab209 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -815,198 +815,6 @@ pub fn get_config( Ok(client_config) } -/// Gets the listening_addresses for lighthouse based on the cli options. -pub fn parse_listening_addresses( - cli_args: &ArgMatches, - log: &Logger, -) -> Result { - let listen_addresses_str = cli_args - .values_of("listen-address") - .expect("--listen_addresses has a default value"); - - let use_zero_ports = cli_args.is_present("zero-ports"); - - // parse the possible ips - let mut maybe_ipv4 = None; - let mut maybe_ipv6 = None; - for addr_str in listen_addresses_str { - let addr = addr_str.parse::().map_err(|parse_error| { - format!("Failed to parse listen-address ({addr_str}) as an Ip address: {parse_error}") - })?; - - match addr { - IpAddr::V4(v4_addr) => match &maybe_ipv4 { - Some(first_ipv4_addr) => { - return Err(format!( - "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ - Got two IpV4 addresses {first_ipv4_addr} and {v4_addr}" - )); - } - None => maybe_ipv4 = Some(v4_addr), - }, - IpAddr::V6(v6_addr) => match &maybe_ipv6 { - Some(first_ipv6_addr) => { - return Err(format!( - "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ - Got two IpV6 addresses {first_ipv6_addr} and {v6_addr}" - )); - } - None => maybe_ipv6 = Some(v6_addr), - }, - } - } - - // parse the possible tcp ports - let port = cli_args - .value_of("port") - .expect("--port has a default value") - .parse::() - .map_err(|parse_error| format!("Failed to parse --port as an integer: {parse_error}"))?; - let port6 = cli_args - .value_of("port6") - .map(str::parse::) - .transpose() - .map_err(|parse_error| format!("Failed to parse --port6 as an integer: {parse_error}"))? - .unwrap_or(9090); - - // parse the possible discovery ports. - let maybe_disc_port = cli_args - .value_of("discovery-port") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --discovery-port as an integer: {parse_error}") - })?; - let maybe_disc6_port = cli_args - .value_of("discovery-port6") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --discovery-port6 as an integer: {parse_error}") - })?; - - // parse the possible quic port. - let maybe_quic_port = cli_args - .value_of("quic-port") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --quic-port as an integer: {parse_error}") - })?; - - // Now put everything together - let listening_addresses = match (maybe_ipv4, maybe_ipv6) { - (None, None) => { - // This should never happen unless clap is broken - return Err("No listening addresses provided".into()); - } - (None, Some(ipv6)) => { - // A single ipv6 address was provided. Set the ports - - if cli_args.is_present("port6") { - warn!(log, "When listening only over IpV6, use the --port flag. The value of --port6 will be ignored.") - } - // use zero ports if required. If not, use the given port. - let tcp_port = use_zero_ports - .then(unused_port::unused_tcp6_port) - .transpose()? - .unwrap_or(port); - - if maybe_disc6_port.is_some() { - warn!(log, "When listening only over IpV6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") - } - // use zero ports if required. If not, use the specific udp port. If none given, use - // the tcp port. - let disc_port = use_zero_ports - .then(unused_port::unused_udp6_port) - .transpose()? - .or(maybe_disc_port) - .unwrap_or(port); - - ListenAddress::V6(lighthouse_network::ListenAddr { - addr: ipv6, - quic_port: None, - disc_port, - tcp_port, - }) - } - (Some(ipv4), None) => { - // A single ipv4 address was provided. Set the ports - - // use zero ports if required. If not, use the given port. - let tcp_port = use_zero_ports - .then(unused_port::unused_tcp4_port) - .transpose()? - .unwrap_or(port); - // use zero ports if required. If not, use the specific discovery port. If none given, use - // the tcp port. - let disc_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_disc_port) - .unwrap_or(port); - // use zero ports if required. If not, use the specific quic port. If none given, use - // the tcp port + 1. - let quic_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_quic_port) - .unwrap_or(port + 1); - - ListenAddress::V4(lighthouse_network::ListenAddr { - addr: ipv4, - disc_port, - quic_port: Some(quic_port), - tcp_port, - }) - } - (Some(ipv4), Some(ipv6)) => { - let ipv4_tcp_port = use_zero_ports - .then(unused_port::unused_tcp4_port) - .transpose()? - .unwrap_or(port); - let ipv4_disc_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_disc_port) - .unwrap_or(ipv4_tcp_port); - let ipv4_quic_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_quic_port) - .unwrap_or(port + 1); - - // Defaults to 9090 when required - let ipv6_tcp_port = use_zero_ports - .then(unused_port::unused_tcp6_port) - .transpose()? - .unwrap_or(port6); - let ipv6_disc_port = use_zero_ports - .then(unused_port::unused_udp6_port) - .transpose()? - .or(maybe_disc6_port) - .unwrap_or(ipv6_tcp_port); - - ListenAddress::DualStack( - lighthouse_network::ListenAddr { - addr: ipv4, - disc_port: ipv4_disc_port, - quic_port: Some(ipv4_quic_port), - tcp_port: ipv4_tcp_port, - }, - lighthouse_network::ListenAddr { - addr: ipv6, - disc_port: ipv6_disc_port, - quic_port: None, - tcp_port: ipv6_tcp_port, - }, - ) - } - }; - - Ok(listening_addresses) -} - /// Sets the network config from the command line arguments. pub fn set_network_config( config: &mut NetworkConfig, @@ -1254,6 +1062,10 @@ pub fn set_network_config( warn!(log, "Discovery is disabled. New peers will not be found"); } + if cli_args.is_present("disable-quic") { + config.disable_quic_support = true; + } + if cli_args.is_present("disable-upnp") { config.upnp_enabled = false; } @@ -1313,6 +1125,232 @@ pub fn set_network_config( Ok(()) } +/// Gets the listening_addresses for lighthouse based on the cli options. +pub fn parse_listening_addresses( + cli_args: &ArgMatches, + log: &Logger, +) -> Result { + let listen_addresses_str = cli_args + .values_of("listen-address") + .expect("--listen_addresses has a default value"); + + let use_zero_ports = cli_args.is_present("zero-ports"); + + // parse the possible ips + let mut maybe_ipv4 = None; + let mut maybe_ipv6 = None; + for addr_str in listen_addresses_str { + let addr = addr_str.parse::().map_err(|parse_error| { + format!("Failed to parse listen-address ({addr_str}) as an Ip address: {parse_error}") + })?; + + match addr { + IpAddr::V4(v4_addr) => match &maybe_ipv4 { + Some(first_ipv4_addr) => { + return Err(format!( + "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ + Got two IpV4 addresses {first_ipv4_addr} and {v4_addr}" + )); + } + None => maybe_ipv4 = Some(v4_addr), + }, + IpAddr::V6(v6_addr) => match &maybe_ipv6 { + Some(first_ipv6_addr) => { + return Err(format!( + "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ + Got two IpV6 addresses {first_ipv6_addr} and {v6_addr}" + )); + } + None => maybe_ipv6 = Some(v6_addr), + }, + } + } + + // parse the possible tcp ports + let port = cli_args + .value_of("port") + .expect("--port has a default value") + .parse::() + .map_err(|parse_error| format!("Failed to parse --port as an integer: {parse_error}"))?; + let port6 = cli_args + .value_of("port6") + .map(str::parse::) + .transpose() + .map_err(|parse_error| format!("Failed to parse --port6 as an integer: {parse_error}"))? + .unwrap_or(9090); + + // parse the possible discovery ports. + let maybe_disc_port = cli_args + .value_of("discovery-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --discovery-port as an integer: {parse_error}") + })?; + let maybe_disc6_port = cli_args + .value_of("discovery-port6") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --discovery-port6 as an integer: {parse_error}") + })?; + + // parse the possible quic port. + let maybe_quic_port = cli_args + .value_of("quic-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic-port as an integer: {parse_error}") + })?; + + // parse the possible quic port. + let maybe_quic6_port = cli_args + .value_of("quic6-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic6-port as an integer: {parse_error}") + })?; + + // Now put everything together + let listening_addresses = match (maybe_ipv4, maybe_ipv6) { + (None, None) => { + // This should never happen unless clap is broken + return Err("No listening addresses provided".into()); + } + (None, Some(ipv6)) => { + // A single ipv6 address was provided. Set the ports + + if cli_args.is_present("port6") { + warn!(log, "When listening only over IPv6, use the --port flag. The value of --port6 will be ignored.") + } + // use zero ports if required. If not, use the given port. + let tcp_port = use_zero_ports + .then(unused_port::unused_tcp6_port) + .transpose()? + .unwrap_or(port); + + if maybe_disc6_port.is_some() { + warn!(log, "When listening only over IPv6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") + } + + // TODO: Remove this warning once https://github.com/libp2p/rust-libp2p/issues/4165 + // is resolved. + if maybe_quic_port.is_some() || maybe_quic6_port.is_some() { + warn!(log, "QUIC is currently disabled over Ipv6.") + } + + // TODO: Once QUIC is supported over IPv6, uncomment this. + /* + if maybe_quic6_port.is_some() { + warn!(log, "When listening only over IPv6, use the --quic-port flag. The value of --quic-port6 will be ignored.") + } + */ + + // use zero ports if required. If not, use the specific udp port. If none given, use + // the tcp port. + let disc_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_disc_port) + .unwrap_or(port); + + let quic_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + + ListenAddress::V6(lighthouse_network::ListenAddr { + addr: ipv6, + quic_port, + disc_port, + tcp_port, + }) + } + (Some(ipv4), None) => { + // A single ipv4 address was provided. Set the ports + + // use zero ports if required. If not, use the given port. + let tcp_port = use_zero_ports + .then(unused_port::unused_tcp4_port) + .transpose()? + .unwrap_or(port); + // use zero ports if required. If not, use the specific discovery port. If none given, use + // the tcp port. + let disc_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_disc_port) + .unwrap_or(port); + // use zero ports if required. If not, use the specific quic port. If none given, use + // the tcp port + 1. + let quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + + ListenAddress::V4(lighthouse_network::ListenAddr { + addr: ipv4, + disc_port, + quic_port, + tcp_port, + }) + } + (Some(ipv4), Some(ipv6)) => { + let ipv4_tcp_port = use_zero_ports + .then(unused_port::unused_tcp4_port) + .transpose()? + .unwrap_or(port); + let ipv4_disc_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_disc_port) + .unwrap_or(ipv4_tcp_port); + let ipv4_quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + + // Defaults to 9090 when required + let ipv6_tcp_port = use_zero_ports + .then(unused_port::unused_tcp6_port) + .transpose()? + .unwrap_or(port6); + let ipv6_disc_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_disc6_port) + .unwrap_or(ipv6_tcp_port); + let ipv6_quic_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_quic6_port) + .unwrap_or(ipv6_disc_port + 1); + + ListenAddress::DualStack( + lighthouse_network::ListenAddr { + addr: ipv4, + disc_port: ipv4_disc_port, + quic_port: ipv4_quic_port, + tcp_port: ipv4_tcp_port, + }, + lighthouse_network::ListenAddr { + addr: ipv6, + disc_port: ipv6_disc_port, + quic_port: ipv6_quic_port, + tcp_port: ipv6_tcp_port, + }, + ) + } + }; + + Ok(listening_addresses) +} + /// Gets the datadir which should be used. pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf { // Read the `--datadir` flag. @@ -1385,4 +1423,4 @@ where .into_iter() .next() .ok_or(format!("Must provide at least one value to {}", flag_name)) -} +} \ No newline at end of file From 5261b8699639e614c994904f3f211de097b90bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 4 Aug 2023 15:38:45 +0100 Subject: [PATCH 05/70] discovery: append quic multiaddrs along with tcp, on handle_pending_outbound_connection --- beacon_node/lighthouse_network/src/discovery/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 0f8ddc53c1b..f2373e273c1 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -968,7 +968,9 @@ impl NetworkBehaviour for Discovery { // ENR's may have multiple Multiaddrs. The multi-addr associated with the UDP // port is removed, which is assumed to be associated with the discv5 protocol (and // therefore irrelevant for other libp2p components). - Ok(enr.multiaddr_tcp()) + let mut addrs = enr.multiaddr_tcp(); + addrs.append(&mut enr.multiaddr_tcp()); + Ok(addrs) } else { Ok(vec![]) } From 73055950e6525b7c5418511fa223743827fbe705 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 7 Aug 2023 17:09:32 +1000 Subject: [PATCH 06/70] Add some documentation for protocol developers --- book/src/SUMMARY.md | 1 + book/src/developers.md | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 book/src/developers.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 7431d223871..18653e2cb10 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -53,3 +53,4 @@ * [Contributing](./contributing.md) * [Development Environment](./setup.md) * [FAQs](./faq.md) +* [Protocol Developers](./developers.md) \ No newline at end of file diff --git a/book/src/developers.md b/book/src/developers.md new file mode 100644 index 00000000000..6ae7f91938d --- /dev/null +++ b/book/src/developers.md @@ -0,0 +1,56 @@ +# For Protocol Developers + +_Documentation for protocol developers._ + +This section lists Lighthouse-specific decisions that are not strictly spec'd and may be useful for +other protocol developers wishing to interact with lighthouse. + + + +## Custom ENR Fields + +Lighthouse currently uses the following ENR fields: + +### Ethereum Consensus Specified + +| Field | Description | +| ---- | ---- | +| `eth2` | The `ENRForkId` in SSZ bytes specifying which fork the node is on | +| `attnets` | An SSZ bitfield which indicates which of the 64 subnets the node is subscribed to for +an extended period of time | +| `syncnets` | An SSZ bitfield which indicates which of the sync committee subnets the node is +subscribed to | + + +### Lighthouse Custom Fields + +Lighthouse is currently using the following custom ENR fields. +| Field | Description | +| ---- | ---- | +| `quic` | The UDP port on which the QUIC transport is listening on IPv4 | +| `quic6` | The UDP port on which the QUIC transport is listening on IPv6 | + + +## Custom RPC Messages + +The specification leaves room for implementation-specific errors. Lighthouse uses the following +custom RPC error messages. + +### Goodbye Reason Codes + +| Code | Message | Description | +| ---- | ---- | ---- | +| 128 | Unable to Verify Network | Teku uses this, so we adopted it. It relates to having a fork +mismatch | +| 129 | Too Many Peers | Lighthouse can close a connection because it has reached its peer-limit and +pruned excess peers | +| 250 | Bad Score | The node has been dropped due to having a bad peer score | +| 251 | Banned | The peer has been banned and disconnected | +| 252 | Banned IP | The IP the node is connected to us with has been banned | + + +### Error Codes + +| Code | Message | Description | +| ---- | ---- | ---- | +| 139 | Rate Limited | The peer has been rate limited so we return this error as a response | \ No newline at end of file From 6c97c5a89e2705b26729f48cb9a3058546c654c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 7 Aug 2023 18:50:23 +0100 Subject: [PATCH 07/70] discovery return Enr's instead of PeerId's on GenerateEvent, and therefore remove unnecessary Enr fetching on Network. --- .../lighthouse_network/src/discovery/mod.rs | 28 ++++++++----------- .../lighthouse_network/src/service/mod.rs | 28 ++++++++----------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index f2373e273c1..4312e37df99 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -75,7 +75,7 @@ const DURATION_DIFFERENCE: Duration = Duration::from_millis(1); /// of the peer if it is specified. #[derive(Debug)] pub struct DiscoveredPeers { - pub peers: HashMap>, + pub peers: HashMap>, } #[derive(Clone, PartialEq)] @@ -787,7 +787,7 @@ impl Discovery { fn process_completed_queries( &mut self, query: QueryResult, - ) -> Option>> { + ) -> Option>> { match query.query_type { QueryType::FindPeers => { self.find_peer_active = false; @@ -798,10 +798,10 @@ impl Discovery { Ok(r) => { debug!(self.log, "Discovery query completed"; "peers_found" => r.len()); let mut results: HashMap<_, Option> = HashMap::new(); - r.iter().for_each(|enr| { + r.into_iter().for_each(|enr| { // cache the found ENR's self.cached_enrs.put(enr.peer_id(), enr.clone()); - results.insert(enr.peer_id(), None); + results.insert(enr, None); }); return Some(results); } @@ -850,17 +850,17 @@ impl Discovery { let subnet_predicate = subnet_predicate::(vec![query.subnet], &self.log); - r.iter() + r.clone() + .into_iter() .filter(|enr| subnet_predicate(enr)) - .map(|enr| enr.peer_id()) - .for_each(|peer_id| { + .for_each(|enr| { if let Some(v) = metrics::get_int_counter( &metrics::SUBNET_PEERS_FOUND, &[query_str], ) { v.inc(); } - let other_min_ttl = mapped_results.get_mut(&peer_id); + let other_min_ttl = mapped_results.get_mut(&enr); // map peer IDs to the min_ttl furthest in the future match (query.min_ttl, other_min_ttl) { @@ -878,15 +878,11 @@ impl Discovery { } // update the mapping if we have a specified min_ttl (Some(min_ttl), Some(None)) => { - mapped_results.insert(peer_id, Some(min_ttl)); + mapped_results.insert(enr, Some(min_ttl)); } // first seen min_ttl for this enr - (Some(min_ttl), None) => { - mapped_results.insert(peer_id, Some(min_ttl)); - } - // first seen min_ttl for this enr - (None, None) => { - mapped_results.insert(peer_id, None); + (min_ttl, None) => { + mapped_results.insert(enr, min_ttl); } (None, Some(Some(_))) => {} // Don't replace the existing specific min_ttl (None, Some(None)) => {} // No-op because this is a duplicate @@ -910,7 +906,7 @@ impl Discovery { } /// Drives the queries returning any results from completed queries. - fn poll_queries(&mut self, cx: &mut Context) -> Option>> { + fn poll_queries(&mut self, cx: &mut Context) -> Option>> { while let Poll::Ready(Some(query_result)) = self.active_queries.poll_next_unpin(cx) { let result = self.process_completed_queries(query_result); if result.is_some() { diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 809b05c55d6..c61c99d6179 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -341,8 +341,9 @@ impl Network { let (swarm, bandwidth) = { // Set up the transport - tcp/ws with noise and mplex - let (transport, bandwidth) = build_transport(local_keypair.clone(), !config.disable_quic_support) - .map_err(|e| format!("Failed to build transport: {:?}", e))?; + let (transport, bandwidth) = + build_transport(local_keypair.clone(), !config.disable_quic_support) + .map_err(|e| format!("Failed to build transport: {:?}", e))?; // use the executor for libp2p struct Executor(task_executor::TaskExecutor); @@ -400,7 +401,6 @@ impl Network { debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery, "quic_enabled" => !config.disable_quic_support); for listen_multiaddr in config.listen_addrs().listen_addresses() { - // If QUIC is disabled, ignore listening on QUIC ports if config.disable_quic_support { if listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) { @@ -1055,20 +1055,18 @@ impl Network { .filter_map(|(peer_id, enr)| { let peers = self.network_globals.peers.read(); if predicate(enr) && peers.should_dial(peer_id) { - Some(*peer_id) + Some(enr) } else { None } }) .collect(); - for peer_id in peers_to_dial { + for enr in peers_to_dial { debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %peer_id); // Remove the ENR from the cache to prevent continual re-dialing on disconnects - self.discovery_mut().remove_cached_enr(&peer_id); - // For any dial event, inform the peer manager - let enr = self.discovery_mut().enr_of_peer(&peer_id); - self.peer_manager_mut().dial_peer(&peer_id, enr); + self.discovery_mut().remove_cached_enr(&enr.peer_id()); + self.peer_manager_mut().dial_peer(&enr.peer_id(), Some(enr)); } } @@ -1359,13 +1357,11 @@ impl Network { &mut self, event: DiscoveredPeers, ) -> Option> { - let DiscoveredPeers { peers } = event; - let to_dial_peers = self.peer_manager_mut().peers_discovered(peers); - for peer_id in to_dial_peers { - debug!(self.log, "Dialing discovered peer"; "peer_id" => %peer_id); + let to_dial_peers = self.peer_manager_mut().peers_discovered(event.peers); + for enr in to_dial_peers { + debug!(self.log, "Dialing discovered peer"; "peer_id" => %enr.peer_id()); // For any dial event, inform the peer manager - let enr = self.discovery_mut().enr_of_peer(&peer_id); - self.peer_manager_mut().dial_peer(&peer_id, enr); + self.peer_manager_mut().dial_peer(&enr.peer_id(), Some(enr)); } None } @@ -1592,4 +1588,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} \ No newline at end of file +} From cd66aeea317d6d1581f1136ca6a9b07ae750e9ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 7 Aug 2023 20:27:02 +0100 Subject: [PATCH 08/70] isolate multiaddresses gathering to dial on PeerManager --- .../lighthouse_network/src/discovery/mod.rs | 21 +--------------- .../src/peer_manager/mod.rs | 24 +++++++++++-------- .../src/peer_manager/network_behaviour.rs | 16 +++++++++---- .../lighthouse_network/src/service/mod.rs | 24 +++++++++---------- 4 files changed, 38 insertions(+), 47 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 4312e37df99..51b3770f9a8 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -953,25 +953,6 @@ impl NetworkBehaviour for Discovery { ) { } - fn handle_pending_outbound_connection( - &mut self, - _connection_id: ConnectionId, - maybe_peer: Option, - _addresses: &[Multiaddr], - _effective_role: libp2p::core::Endpoint, - ) -> Result, libp2p::swarm::ConnectionDenied> { - if let Some(enr) = maybe_peer.and_then(|peer_id| self.enr_of_peer(&peer_id)) { - // ENR's may have multiple Multiaddrs. The multi-addr associated with the UDP - // port is removed, which is assumed to be associated with the discv5 protocol (and - // therefore irrelevant for other libp2p components). - let mut addrs = enr.multiaddr_tcp(); - addrs.append(&mut enr.multiaddr_tcp()); - Ok(addrs) - } else { - Ok(vec![]) - } - } - // Main execution loop to drive the behaviour fn poll( &mut self, @@ -1261,4 +1242,4 @@ mod tests { // when a peer belongs to multiple subnet ids, we use the highest ttl. assert_eq!(results.get(&enr1.peer_id()).unwrap(), &instant1); } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 4f3454f4033..3f5a088f48a 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -1,5 +1,6 @@ //! Implementation of Lighthouse's peer management system. +use crate::discovery::enr_ext::EnrExt; use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode}; use crate::service::TARGET_SUBNET_PEERS; use crate::{error, metrics, Gossipsub}; @@ -13,7 +14,6 @@ use peerdb::{client::ClientKind, BanOperation, BanResult, ScoreUpdateResult}; use rand::seq::SliceRandom; use slog::{debug, error, trace, warn}; use smallvec::SmallVec; -use std::collections::BTreeMap; use std::{ sync::Arc, time::{Duration, Instant}, @@ -78,7 +78,7 @@ pub struct PeerManager { /// The target number of peers we would like to connect to. target_peers: usize, /// Peers queued to be dialed. - peers_to_dial: BTreeMap>, + peers_to_dial: Vec, /// The number of temporarily banned peers. This is used to prevent instantaneous /// reconnection. // NOTE: This just prevents re-connections. The state of the peer is otherwise unaffected. A @@ -317,11 +317,11 @@ impl PeerManager { /// multiaddr here, however this could relate to duplicate PeerId's etc. If the lookup /// proves resource constraining, we should switch to multiaddr dialling here. #[allow(clippy::mutable_key_type)] - pub fn peers_discovered(&mut self, results: HashMap>) -> Vec { + pub fn peers_discovered(&mut self, results: HashMap>) -> Vec { let mut to_dial_peers = Vec::with_capacity(4); let connected_or_dialing = self.network_globals.connected_or_dialing_peers(); - for (peer_id, min_ttl) in results { + for (enr, min_ttl) in results { // There are two conditions in deciding whether to dial this peer. // 1. If we are less than our max connections. Discovery queries are executed to reach // our target peers, so its fine to dial up to our max peers (which will get pruned @@ -333,7 +333,11 @@ impl PeerManager { if (min_ttl.is_some() && connected_or_dialing + to_dial_peers.len() < self.max_priority_peers() || connected_or_dialing + to_dial_peers.len() < self.max_peers()) - && self.network_globals.peers.read().should_dial(&peer_id) + && self + .network_globals + .peers + .read() + .should_dial(&enr.peer_id()) { // This should be updated with the peer dialing. In fact created once the peer is // dialed @@ -341,9 +345,9 @@ impl PeerManager { self.network_globals .peers .write() - .update_min_ttl(&peer_id, min_ttl); + .update_min_ttl(&enr.peer_id(), min_ttl); } - to_dial_peers.push(peer_id); + to_dial_peers.push(enr); } } @@ -407,8 +411,8 @@ impl PeerManager { /* Notifications from the Swarm */ // A peer is being dialed. - pub fn dial_peer(&mut self, peer_id: &PeerId, enr: Option) { - self.peers_to_dial.insert(*peer_id, enr); + pub fn dial_peers(&mut self, mut peers: Vec) { + self.peers_to_dial.append(&mut peers); } /// Reports if a peer is banned or not. @@ -2331,4 +2335,4 @@ mod tests { }) } } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index ce374bb9ab4..eefb0af6b9d 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -12,6 +12,7 @@ use libp2p::swarm::{ConnectionId, NetworkBehaviour, PollParameters, ToSwarm}; use slog::{debug, error}; use types::EthSpec; +use crate::discovery::enr_ext::EnrExt; use crate::metrics; use crate::rpc::GoodbyeReason; use crate::types::SyncState; @@ -95,11 +96,18 @@ impl NetworkBehaviour for PeerManager { self.events.shrink_to_fit(); } - if let Some((peer_id, maybe_enr)) = self.peers_to_dial.pop_first() { - self.inject_peer_connection(&peer_id, ConnectingType::Dialing, maybe_enr); + if let Some(enr) = self.peers_to_dial.pop() { + self.inject_peer_connection(&enr.peer_id(), ConnectingType::Dialing, Some(enr.clone())); + // Prioritize Quic connections over Tcp ones. + let multiaddrs = enr + .multiaddr_quic() + .into_iter() + .chain(enr.multiaddr_tcp()) + .collect(); return Poll::Ready(ToSwarm::Dial { - opts: DialOpts::peer_id(peer_id) + opts: DialOpts::peer_id(enr.peer_id()) .condition(PeerCondition::Disconnected) + .addresses(multiaddrs) .build(), }); } @@ -295,4 +303,4 @@ impl PeerManager { } } } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index c61c99d6179..9de750ea8a5 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1049,25 +1049,26 @@ impl Network { /// in Connected, Dialing or Banned state. fn dial_cached_enrs_in_subnet(&mut self, subnet: Subnet) { let predicate = subnet_predicate::(vec![subnet], &self.log); - let peers_to_dial: Vec = self + let peers_to_dial: Vec = self .discovery() .cached_enrs() .filter_map(|(peer_id, enr)| { let peers = self.network_globals.peers.read(); if predicate(enr) && peers.should_dial(peer_id) { - Some(enr) + Some(enr.clone()) } else { None } }) .collect(); - for enr in peers_to_dial { - debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %peer_id); - // Remove the ENR from the cache to prevent continual re-dialing on disconnects + // Remove the ENR from the cache to prevent continual re-dialing on disconnects + // TODO: add this step to the iteration above. + peers_to_dial.iter().for_each(|enr| { self.discovery_mut().remove_cached_enr(&enr.peer_id()); - self.peer_manager_mut().dial_peer(&enr.peer_id(), Some(enr)); - } + }); + + self.peer_manager_mut().dial_peers(peers_to_dial); } /* Sub-behaviour event handling functions */ @@ -1358,11 +1359,8 @@ impl Network { event: DiscoveredPeers, ) -> Option> { let to_dial_peers = self.peer_manager_mut().peers_discovered(event.peers); - for enr in to_dial_peers { - debug!(self.log, "Dialing discovered peer"; "peer_id" => %enr.peer_id()); - // For any dial event, inform the peer manager - self.peer_manager_mut().dial_peer(&enr.peer_id(), Some(enr)); - } + // For any dial event, inform the peer manager + self.peer_manager_mut().dial_peers(to_dial_peers); None } @@ -1588,4 +1586,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} +} \ No newline at end of file From d8bb593ad7974398de224358f0f4544cd8752d55 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 8 Aug 2023 14:14:49 +1000 Subject: [PATCH 09/70] Small logging and comment changes --- beacon_node/lighthouse_network/src/config.rs | 2 +- .../lighthouse_network/src/discovery/enr_ext.rs | 2 +- .../lighthouse_network/src/discovery/mod.rs | 2 +- beacon_node/lighthouse_network/src/lib.rs | 2 +- beacon_node/lighthouse_network/src/listen_addr.rs | 2 +- .../lighthouse_network/src/peer_manager/mod.rs | 13 ++++++------- .../src/peer_manager/network_behaviour.rs | 15 ++++++++++----- beacon_node/lighthouse_network/src/service/mod.rs | 13 +++++++------ .../lighthouse_network/src/service/utils.rs | 2 +- beacon_node/network/src/nat.rs | 2 +- beacon_node/network/src/service.rs | 2 +- beacon_node/src/config.rs | 2 +- lcli/src/generate_bootnode_enr.rs | 2 +- testing/simulator/src/local_network.rs | 2 +- 14 files changed, 34 insertions(+), 29 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index b0ed8b6dcf6..043b747a1b7 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -590,4 +590,4 @@ pub const fn is_global_ipv6(addr: &Ipv6Addr) -> bool { || is_documentation(addr) || is_unique_local(addr) || is_unicast_link_local(addr)) -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 96304c011c8..77c4f0e43a7 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -397,4 +397,4 @@ mod tests { assert_eq!(enr.node_id(), node_id); } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 51b3770f9a8..d9f79a537f7 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1242,4 +1242,4 @@ mod tests { // when a peer belongs to multiple subnet ids, we use the highest ttl. assert_eq!(results.get(&enr1.peer_id()).unwrap(), &instant1); } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/lib.rs b/beacon_node/lighthouse_network/src/lib.rs index 85b9f9126a0..3d539af3b28 100644 --- a/beacon_node/lighthouse_network/src/lib.rs +++ b/beacon_node/lighthouse_network/src/lib.rs @@ -88,4 +88,4 @@ pub use peer_manager::{ // pub use service::{load_private_key, Context, Libp2pEvent, Service, NETWORK_KEY_FILENAME}; pub use service::api_types::{PeerRequestId, Request, Response}; pub use service::utils::*; -pub use service::{Gossipsub, NetworkEvent}; \ No newline at end of file +pub use service::{Gossipsub, NetworkEvent}; diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 6b4b494a2ea..74c39a2ab81 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -123,4 +123,4 @@ impl slog::KV for ListenAddress { } slog::Result::Ok(()) } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 3f5a088f48a..af1a2e45e7d 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -312,12 +312,9 @@ impl PeerManager { /// Peers that have been returned by discovery requests that are suitable for dialing are /// returned here. /// - /// NOTE: By dialing `PeerId`s and not multiaddrs, libp2p requests the multiaddr associated - /// with a new `PeerId` which involves a discovery routing table lookup. We could dial the - /// multiaddr here, however this could relate to duplicate PeerId's etc. If the lookup - /// proves resource constraining, we should switch to multiaddr dialling here. + /// This function decides whether or not to dial these peers. #[allow(clippy::mutable_key_type)] - pub fn peers_discovered(&mut self, results: HashMap>) -> Vec { + pub fn peers_discovered(&mut self, results: HashMap>) { let mut to_dial_peers = Vec::with_capacity(4); let connected_or_dialing = self.network_globals.connected_or_dialing_peers(); @@ -347,6 +344,7 @@ impl PeerManager { .write() .update_min_ttl(&enr.peer_id(), min_ttl); } + debug!(self.log, "Dialing discovered peer"; "peer_id" => %enr.peer_id()); to_dial_peers.push(enr); } } @@ -354,7 +352,8 @@ impl PeerManager { // Queue another discovery if we need to self.maintain_peer_count(to_dial_peers.len()); - to_dial_peers + // Dial the required peers + self.dial_peers(to_dial_peers); } /// A STATUS message has been received from a peer. This resets the status timer. @@ -2335,4 +2334,4 @@ mod tests { }) } } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index eefb0af6b9d..999c14375d1 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -97,15 +97,20 @@ impl NetworkBehaviour for PeerManager { } if let Some(enr) = self.peers_to_dial.pop() { - self.inject_peer_connection(&enr.peer_id(), ConnectingType::Dialing, Some(enr.clone())); + let peer_id = enr.peer_id(); + self.inject_peer_connection(&peer_id, ConnectingType::Dialing, Some(enr.clone())); + let quic_multiaddrs = enr.multiaddr_quic(); + if !quic_multiaddrs.is_empty() { + debug!(self.log, "Dialing QUIC supported peer"; "peer_id"=> %peer_id, "quic_multiaddrs" => ?quic_multiaddrs); + } + // Prioritize Quic connections over Tcp ones. - let multiaddrs = enr - .multiaddr_quic() + let multiaddrs = quic_multiaddrs .into_iter() .chain(enr.multiaddr_tcp()) .collect(); return Poll::Ready(ToSwarm::Dial { - opts: DialOpts::peer_id(enr.peer_id()) + opts: DialOpts::peer_id(peer_id) .condition(PeerCondition::Disconnected) .addresses(multiaddrs) .build(), @@ -303,4 +308,4 @@ impl PeerManager { } } } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9de750ea8a5..0e0e8a1d11c 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1045,7 +1045,7 @@ impl Network { } } - /// Dial cached enrs in discovery service that are in the given `subnet_id` and aren't + /// Dial cached Enrs in discovery service that are in the given `subnet_id` and aren't /// in Connected, Dialing or Banned state. fn dial_cached_enrs_in_subnet(&mut self, subnet: Subnet) { let predicate = subnet_predicate::(vec![subnet], &self.log); @@ -1063,7 +1063,6 @@ impl Network { .collect(); // Remove the ENR from the cache to prevent continual re-dialing on disconnects - // TODO: add this step to the iteration above. peers_to_dial.iter().for_each(|enr| { self.discovery_mut().remove_cached_enr(&enr.peer_id()); }); @@ -1358,9 +1357,11 @@ impl Network { &mut self, event: DiscoveredPeers, ) -> Option> { - let to_dial_peers = self.peer_manager_mut().peers_discovered(event.peers); - // For any dial event, inform the peer manager - self.peer_manager_mut().dial_peers(to_dial_peers); + // Inform the peer manager about discovered peers. + // + // The peer manager will subsequently decide which peers need to be dialed and then dial + // them. + self.peer_manager_mut().peers_discovered(event.peers); None } @@ -1586,4 +1587,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 79c6afc83b1..c9dbd7ab8b2 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -279,4 +279,4 @@ pub(crate) fn save_metadata_to_disk( ); } } -} \ No newline at end of file +} diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index 99b43db0334..2e923420ea3 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -203,4 +203,4 @@ pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Log Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), } } -} \ No newline at end of file +} diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index aedcdb45918..987faeb922a 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -981,4 +981,4 @@ impl Drop for NetworkService { info!(self.log, "Network service shutdown"); } -} \ No newline at end of file +} diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 9689b2ab209..382d7ce3458 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1423,4 +1423,4 @@ where .into_iter() .next() .ok_or(format!("Must provide at least one value to {}", flag_name)) -} \ No newline at end of file +} diff --git a/lcli/src/generate_bootnode_enr.rs b/lcli/src/generate_bootnode_enr.rs index 54294f2b42c..84151b6d1e5 100644 --- a/lcli/src/generate_bootnode_enr.rs +++ b/lcli/src/generate_bootnode_enr.rs @@ -57,4 +57,4 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { .map_err(|e| format!("Unable to write key to {}: {:?}", NETWORK_KEY_FILENAME, e))?; Ok(()) -} \ No newline at end of file +} diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 9d03de14716..5c6ef30cd65 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -315,4 +315,4 @@ impl LocalNetwork { ); genesis_time - now } -} \ No newline at end of file +} From 4c9159cd338b87878b963dae4cc44dccc7527f04 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 8 Aug 2023 14:23:34 +1000 Subject: [PATCH 10/70] Correct line spacing in book docs --- book/src/developers.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/book/src/developers.md b/book/src/developers.md index 6ae7f91938d..2ba09bd3412 100644 --- a/book/src/developers.md +++ b/book/src/developers.md @@ -6,7 +6,6 @@ This section lists Lighthouse-specific decisions that are not strictly spec'd an other protocol developers wishing to interact with lighthouse. - ## Custom ENR Fields Lighthouse currently uses the following ENR fields: @@ -16,10 +15,8 @@ Lighthouse currently uses the following ENR fields: | Field | Description | | ---- | ---- | | `eth2` | The `ENRForkId` in SSZ bytes specifying which fork the node is on | -| `attnets` | An SSZ bitfield which indicates which of the 64 subnets the node is subscribed to for -an extended period of time | -| `syncnets` | An SSZ bitfield which indicates which of the sync committee subnets the node is -subscribed to | +| `attnets` | An SSZ bitfield which indicates which of the 64 subnets the node is subscribed to for an extended period of time | +| `syncnets` | An SSZ bitfield which indicates which of the sync committee subnets the node is subscribed to | ### Lighthouse Custom Fields @@ -40,10 +37,8 @@ custom RPC error messages. | Code | Message | Description | | ---- | ---- | ---- | -| 128 | Unable to Verify Network | Teku uses this, so we adopted it. It relates to having a fork -mismatch | -| 129 | Too Many Peers | Lighthouse can close a connection because it has reached its peer-limit and -pruned excess peers | +| 128 | Unable to Verify Network | Teku uses this, so we adopted it. It relates to having a fork mismatch | +| 129 | Too Many Peers | Lighthouse can close a connection because it has reached its peer-limit and pruned excess peers | | 250 | Bad Score | The node has been dropped due to having a bad peer score | | 251 | Banned | The peer has been banned and disconnected | | 252 | Banned IP | The IP the node is connected to us with has been banned | From 34926466109333de0b35a4defc6195d2ebc7ffdb Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 8 Aug 2023 14:53:29 +1000 Subject: [PATCH 11/70] Appease the all-wise all-knowing clippy --- beacon_node/lighthouse_network/src/config.rs | 5 ++--- .../lighthouse_network/src/discovery/enr_ext.rs | 6 +++--- .../lighthouse_network/src/discovery/mod.rs | 4 ++-- beacon_node/lighthouse_network/src/service/mod.rs | 6 ++---- beacon_node/network/src/nat.rs | 14 +++++--------- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 025dd236481..58e785c0e45 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -193,8 +193,7 @@ impl Config { /// Sets the listening address to use both an ipv4 and ipv6 address. The discv5 ip_mode and /// table filter is adjusted accordingly to ensure addresses that are present in the enr are /// globally reachable. - /// TODO: add Quic support IPV6 port when https://github.com/libp2p/rust-libp2p/issues/4165 - /// gets addressed. + #[allow(clippy::too_many_arguments)] pub fn set_ipv4_ipv6_listening_addresses( &mut self, v4_addr: Ipv4Addr, @@ -592,4 +591,4 @@ pub const fn is_global_ipv6(addr: &Ipv6Addr) -> bool { || is_documentation(addr) || is_unique_local(addr) || is_unicast_link_local(addr)) -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 77c4f0e43a7..3c558ae56d1 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -6,8 +6,8 @@ use libp2p::core::multiaddr::Protocol; use libp2p::identity::{ed25519, secp256k1, KeyType, Keypair, PublicKey}; use tiny_keccak::{Hasher, Keccak}; -const QUIC_ENR_KEY: &'static str = "quic"; -const QUIC6_ENR_KEY: &'static str = "quic6"; +const QUIC_ENR_KEY: &str = "quic"; +const QUIC6_ENR_KEY: &str = "quic6"; /// Extend ENR for libp2p types. pub trait EnrExt { @@ -397,4 +397,4 @@ mod tests { assert_eq!(enr.node_id(), node_id); } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index d9f79a537f7..79271abd053 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1240,6 +1240,6 @@ mod tests { assert_eq!(results.len(), 2); // when a peer belongs to multiple subnet ids, we use the highest ttl. - assert_eq!(results.get(&enr1.peer_id()).unwrap(), &instant1); + assert_eq!(results.get(&enr1).unwrap(), &instant1); } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 4f8727362db..35ba4ea99d7 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -416,10 +416,8 @@ impl Network { for listen_multiaddr in config.listen_addrs().listen_addresses() { // If QUIC is disabled, ignore listening on QUIC ports - if config.disable_quic_support { - if listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) { + if config.disable_quic_support && listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) { continue; - } } match self.swarm.listen_on(listen_multiaddr.clone()) { @@ -1601,4 +1599,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} +} \ No newline at end of file diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index 2e923420ea3..b109ae71709 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -110,17 +110,13 @@ pub fn construct_upnp_mappings( let mut udp_sockets = Vec::new(); // Set the discovery UDP port mapping - if !config.disable_discovery { - if set_udp_mapping(config.disc_port).is_ok() { - udp_sockets.push(config.disc_port); - } + if !config.disable_discovery && set_udp_mapping(config.disc_port).is_ok() { + udp_sockets.push(config.disc_port); } // Set the quic UDP port mapping - if !config.disable_quic_support { - if set_udp_mapping(config.quic_port).is_ok() { - udp_sockets.push(config.quic_port) - } + if !config.disable_quic_support && set_udp_mapping(config.quic_port).is_ok() { + udp_sockets.push(config.quic_port) } // report any updates to the network service. @@ -203,4 +199,4 @@ pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Log Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), } } -} +} \ No newline at end of file From c4f0a384c59a5f205c3b303764ea11b18694b788 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 8 Aug 2023 14:54:37 +1000 Subject: [PATCH 12/70] fmt --- beacon_node/lighthouse_network/src/config.rs | 2 +- beacon_node/lighthouse_network/src/discovery/enr_ext.rs | 2 +- beacon_node/lighthouse_network/src/discovery/mod.rs | 2 +- beacon_node/lighthouse_network/src/service/mod.rs | 8 +++++--- beacon_node/network/src/nat.rs | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 58e785c0e45..a0cf0bdbba9 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -591,4 +591,4 @@ pub const fn is_global_ipv6(addr: &Ipv6Addr) -> bool { || is_documentation(addr) || is_unique_local(addr) || is_unicast_link_local(addr)) -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 3c558ae56d1..f1a04c9d7c9 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -397,4 +397,4 @@ mod tests { assert_eq!(enr.node_id(), node_id); } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 79271abd053..cc239501309 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1242,4 +1242,4 @@ mod tests { // when a peer belongs to multiple subnet ids, we use the highest ttl. assert_eq!(results.get(&enr1).unwrap(), &instant1); } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 35ba4ea99d7..76c2baef5f0 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -416,8 +416,10 @@ impl Network { for listen_multiaddr in config.listen_addrs().listen_addresses() { // If QUIC is disabled, ignore listening on QUIC ports - if config.disable_quic_support && listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) { - continue; + if config.disable_quic_support + && listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) + { + continue; } match self.swarm.listen_on(listen_multiaddr.clone()) { @@ -1599,4 +1601,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} \ No newline at end of file +} diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index b109ae71709..4476ccd4425 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -199,4 +199,4 @@ pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Log Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), } } -} \ No newline at end of file +} From 583c23365c4163522b10de6e96906052122fc0ac Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 8 Aug 2023 16:39:58 +1000 Subject: [PATCH 13/70] Correct quic port and improve logs --- beacon_node/lighthouse_network/src/listen_addr.rs | 4 ++-- beacon_node/lighthouse_network/src/service/mod.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 74c39a2ab81..6960bdc11b5 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -66,7 +66,7 @@ impl ListenAddress { let v4_quic_multiaddrs = self.v4().map(|v4_addr| { Multiaddr::from(v4_addr.addr) - .with(Protocol::Udp(v4_addr.tcp_port)) + .with(Protocol::Udp(v4_addr.quic_port)) .with(Protocol::QuicV1) }); @@ -123,4 +123,4 @@ impl slog::KV for ListenAddress { } slog::Result::Ok(()) } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 76c2baef5f0..9bca2eaa9c7 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1565,7 +1565,6 @@ impl Network { peer_id, connection_id: _, } => { - debug!(self.log, "Swarm Dialing"; "peer_id" => ?peer_id); None } }; @@ -1601,4 +1600,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} +} \ No newline at end of file From 1821941249a769185adb584ad982a6256a5ff315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Tue, 8 Aug 2023 15:56:33 +0100 Subject: [PATCH 14/70] Network: cleanup remove no longer required fn it's only a one liner for PeerManager delegation of discovered peers. --- .../lighthouse_network/src/service/mod.rs | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9bca2eaa9c7..d01c62f5961 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1366,19 +1366,6 @@ impl Network { } } - /// Handle a discovery event. - fn inject_discovery_event( - &mut self, - event: DiscoveredPeers, - ) -> Option> { - // Inform the peer manager about discovered peers. - // - // The peer manager will subsequently decide which peers need to be dialed and then dial - // them. - self.peer_manager_mut().peers_discovered(event.peers); - None - } - /// Handle an identify event. fn inject_identify_event( &mut self, @@ -1479,7 +1466,14 @@ impl Network { BehaviourEvent::BannedPeers(void) => void::unreachable(void), BehaviourEvent::Gossipsub(ge) => self.inject_gs_event(ge), BehaviourEvent::Eth2Rpc(re) => self.inject_rpc_event(re), - BehaviourEvent::Discovery(de) => self.inject_discovery_event(de), + // Inform the peer manager about discovered peers. + // + // The peer manager will subsequently decide which peers need to be dialed and then dial + // them. + BehaviourEvent::Discovery(DiscoveredPeers { peers }) => { + self.peer_manager_mut().peers_discovered(peers); + None + } BehaviourEvent::Identify(ie) => self.inject_identify_event(ie), BehaviourEvent::PeerManager(pe) => self.inject_pm_event(pe), BehaviourEvent::ConnectionLimits(le) => void::unreachable(le), @@ -1561,12 +1555,7 @@ impl Network { None } } - SwarmEvent::Dialing { - peer_id, - connection_id: _, - } => { - None - } + SwarmEvent::Dialing { .. } => None, }; if let Some(ev) = maybe_event { @@ -1600,4 +1589,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} \ No newline at end of file +} From ae548091ebe8ebb22a9e123ddabcd5a958a29476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Tue, 8 Aug 2023 15:54:05 +0100 Subject: [PATCH 15/70] Discovery: - remove duplicated check for empty subnets - remove no longer used `enr_for_peer` --- .../lighthouse_network/src/discovery/mod.rs | 48 ++++--------------- 1 file changed, 10 insertions(+), 38 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index cc239501309..c0754a4776c 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -383,20 +383,6 @@ impl Discovery { self.discv5.table_entries_enr() } - /// Returns the ENR of a known peer if it exists. - pub fn enr_of_peer(&mut self, peer_id: &PeerId) -> Option { - // first search the local cache - if let Some(enr) = self.cached_enrs.get(peer_id) { - return Some(enr.clone()); - } - // not in the local cache, look in the routing table - if let Ok(node_id) = enr_ext::peer_id_to_node_id(peer_id) { - self.discv5.find_enr(&node_id) - } else { - None - } - } - /// Updates the local ENR TCP port. /// There currently isn't a case to update the address here. We opt for discovery to /// automatically update the external address. @@ -733,23 +719,6 @@ impl Discovery { target_peers: usize, additional_predicate: impl Fn(&Enr) -> bool + Send + 'static, ) { - // Make sure there are subnet queries included - let contains_queries = match &query { - QueryType::Subnet(queries) => !queries.is_empty(), - QueryType::FindPeers => true, - }; - - if !contains_queries { - debug!( - self.log, - "No subnets included in this request. Skipping discovery request." - ); - return; - } - - // Generate a random target node id. - let random_node = NodeId::random(); - let enr_fork_id = match self.local_enr().eth2() { Ok(v) => v, Err(e) => { @@ -773,7 +742,8 @@ impl Discovery { // Build the future let query_future = self .discv5 - .find_node_predicate(random_node, predicate, target_peers) + // Generate a random target node id. + .find_node_predicate(NodeId::random(), predicate, target_peers) .map(|v| QueryResult { query_type: query, result: v, @@ -797,12 +767,14 @@ impl Discovery { } Ok(r) => { debug!(self.log, "Discovery query completed"; "peers_found" => r.len()); - let mut results: HashMap<_, Option> = HashMap::new(); - r.into_iter().for_each(|enr| { - // cache the found ENR's - self.cached_enrs.put(enr.peer_id(), enr.clone()); - results.insert(enr, None); - }); + let results = r + .into_iter() + .map(|enr| { + // cache the found ENR's + self.cached_enrs.put(enr.peer_id(), enr.clone()); + (enr, None) + }) + .collect(); return Some(results); } Err(e) => { From 191e0ae9c7a91ee5c89d4e774fcee542d8458789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Tue, 8 Aug 2023 21:41:28 +0100 Subject: [PATCH 16/70] PeerManager: move should_dial() logic to dial_peer() as we seem to be calling everytime we call dial_peer(). --- .../src/peer_manager/mod.rs | 36 +++++++++---------- .../lighthouse_network/src/service/mod.rs | 8 ++--- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index af1a2e45e7d..c2889c04a1f 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -315,8 +315,7 @@ impl PeerManager { /// This function decides whether or not to dial these peers. #[allow(clippy::mutable_key_type)] pub fn peers_discovered(&mut self, results: HashMap>) { - let mut to_dial_peers = Vec::with_capacity(4); - + let mut to_dial_peers = 0; let connected_or_dialing = self.network_globals.connected_or_dialing_peers(); for (enr, min_ttl) in results { // There are two conditions in deciding whether to dial this peer. @@ -327,14 +326,8 @@ impl PeerManager { // considered a priority. We have pre-allocated some extra priority slots for these // peers as specified by PRIORITY_PEER_EXCESS. Therefore we dial these peers, even // if we are already at our max_peer limit. - if (min_ttl.is_some() - && connected_or_dialing + to_dial_peers.len() < self.max_priority_peers() - || connected_or_dialing + to_dial_peers.len() < self.max_peers()) - && self - .network_globals - .peers - .read() - .should_dial(&enr.peer_id()) + if min_ttl.is_some() && connected_or_dialing + to_dial_peers < self.max_priority_peers() + || connected_or_dialing + to_dial_peers < self.max_peers() { // This should be updated with the peer dialing. In fact created once the peer is // dialed @@ -345,15 +338,13 @@ impl PeerManager { .update_min_ttl(&enr.peer_id(), min_ttl); } debug!(self.log, "Dialing discovered peer"; "peer_id" => %enr.peer_id()); - to_dial_peers.push(enr); + self.dial_peer(enr); + to_dial_peers += 1; } } // Queue another discovery if we need to - self.maintain_peer_count(to_dial_peers.len()); - - // Dial the required peers - self.dial_peers(to_dial_peers); + self.maintain_peer_count(to_dial_peers); } /// A STATUS message has been received from a peer. This resets the status timer. @@ -409,9 +400,16 @@ impl PeerManager { /* Notifications from the Swarm */ - // A peer is being dialed. - pub fn dial_peers(&mut self, mut peers: Vec) { - self.peers_to_dial.append(&mut peers); + /// A peer is being dialed. + pub fn dial_peer(&mut self, peer: Enr) { + if self + .network_globals + .peers + .read() + .should_dial(&peer.peer_id()) + { + self.peers_to_dial.push(peer); + } } /// Reports if a peer is banned or not. @@ -2334,4 +2332,4 @@ mod tests { }) } } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index d01c62f5961..4c4c371653c 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1066,9 +1066,8 @@ impl Network { let peers_to_dial: Vec = self .discovery() .cached_enrs() - .filter_map(|(peer_id, enr)| { - let peers = self.network_globals.peers.read(); - if predicate(enr) && peers.should_dial(peer_id) { + .filter_map(|(_peer_id, enr)| { + if predicate(enr) { Some(enr.clone()) } else { None @@ -1078,10 +1077,9 @@ impl Network { // Remove the ENR from the cache to prevent continual re-dialing on disconnects peers_to_dial.iter().for_each(|enr| { + self.peer_manager_mut().dial_peer(enr.clone()); self.discovery_mut().remove_cached_enr(&enr.peer_id()); }); - - self.peer_manager_mut().dial_peers(peers_to_dial); } /* Sub-behaviour event handling functions */ From 31f45b713a80dc9d12e49e66562e25b467b75557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Tue, 8 Aug 2023 23:50:47 +0100 Subject: [PATCH 17/70] cargo fmt --- beacon_node/lighthouse_network/src/listen_addr.rs | 2 +- beacon_node/lighthouse_network/src/peer_manager/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 6960bdc11b5..e55df7d4814 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -123,4 +123,4 @@ impl slog::KV for ListenAddress { } slog::Result::Ok(()) } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index c2889c04a1f..87137168d7d 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -2332,4 +2332,4 @@ mod tests { }) } } -} \ No newline at end of file +} From d3953e0bc60a2d613adc8851af4aa3286a8f3ba3 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 9 Aug 2023 12:09:34 +1000 Subject: [PATCH 18/70] fmt --- beacon_node/lighthouse_network/src/listen_addr.rs | 2 +- beacon_node/lighthouse_network/src/service/mod.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 6960bdc11b5..e55df7d4814 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -123,4 +123,4 @@ impl slog::KV for ListenAddress { } slog::Result::Ok(()) } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9bca2eaa9c7..b203e0aac93 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1564,9 +1564,7 @@ impl Network { SwarmEvent::Dialing { peer_id, connection_id: _, - } => { - None - } + } => None, }; if let Some(ev) = maybe_event { @@ -1600,4 +1598,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} \ No newline at end of file +} From dcc78ccbc76d3236a396a273bfe37da5663c5d3b Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 9 Aug 2023 16:45:16 +1000 Subject: [PATCH 19/70] Remove unused var --- beacon_node/lighthouse_network/src/service/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index b203e0aac93..77c71398ef6 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1562,7 +1562,7 @@ impl Network { } } SwarmEvent::Dialing { - peer_id, + peer_id: _, connection_id: _, } => None, }; @@ -1598,4 +1598,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} +} \ No newline at end of file From c7c1749c1d6b1e7f10cc84d0067201d8fd12f80e Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 9 Aug 2023 16:46:35 +1000 Subject: [PATCH 20/70] Can't change a single line without fmt complaining --- beacon_node/lighthouse_network/src/service/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 77c71398ef6..972b05f57f2 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1598,4 +1598,4 @@ impl Network { pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } -} \ No newline at end of file +} From 14b42c136d87819de20e9e774dc0ad02e6b6be4f Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 10 Aug 2023 15:56:11 +1000 Subject: [PATCH 21/70] Add QUIC to default ENR builder and fight tests --- beacon_node/lighthouse_network/src/config.rs | 23 ++-- .../lighthouse_network/src/discovery/enr.rs | 35 ++++- .../src/discovery/enr_ext.rs | 6 +- .../lighthouse_network/src/listen_addr.rs | 23 ++-- beacon_node/src/cli.rs | 26 +++- beacon_node/src/config.rs | 41 ++++-- boot_node/src/config.rs | 10 +- lighthouse/tests/beacon_node.rs | 130 +++++++++++++----- testing/simulator/src/local_network.rs | 4 +- 9 files changed, 214 insertions(+), 84 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index a0cf0bdbba9..293afc2db51 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -58,18 +58,24 @@ pub struct Config { /// that no discovery address has been set in the CLI args. pub enr_address: (Option, Option), - /// The disc4 port to broadcast to peers in order to reach back for discovery. - pub enr_disc4_port: Option, + /// The udp port to broadcast to peers in order to reach back for discovery. + pub enr_udp4_port: Option, + + /// The UDP port to broadcast to peers in order to reach back for quic libp2p services. + pub enr_quic4_port: Option, /// The tcp4 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp4_port: Option, - /// The disc6 port to broadcast to peers in order to reach back for discovery. - pub enr_disc6_port: Option, + /// The udp6 port to broadcast to peers in order to reach back for discovery. + pub enr_udp6_port: Option, /// The tcp6 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp6_port: Option, + /// The UDP port to broadcast to peers in order to reach back for quic libp2p services. + pub enr_quic6_port: Option, + /// Target number of connected peers. pub target_peers: usize, @@ -333,10 +339,11 @@ impl Default for Config { network_dir, listen_addresses, enr_address: (None, None), - - enr_disc4_port: None, + enr_udp4_port: None, + enr_quic4_port: None, enr_tcp4_port: None, - enr_disc6_port: None, + enr_udp6_port: None, + enr_quic6_port: None, enr_tcp6_port: None, target_peers: 50, gs_config, @@ -591,4 +598,4 @@ pub const fn is_global_ipv6(addr: &Ipv6Addr) -> bool { || is_documentation(addr) || is_unique_local(addr) || is_unicast_link_local(addr)) -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index 0ec061eac7a..b07d55776bd 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -17,6 +17,8 @@ use std::path::Path; use std::str::FromStr; use types::{EnrForkId, EthSpec}; +use super::enr_ext::{EnrExt, QUIC6_ENR_KEY, QUIC_ENR_KEY}; + /// The ENR field specifying the fork id. pub const ETH2_ENR_KEY: &str = "eth2"; /// The ENR field specifying the attestation subnet bitfield. @@ -142,7 +144,7 @@ pub fn build_or_load_enr( pub fn create_enr_builder_from_config( config: &NetworkConfig, - enable_tcp: bool, + enable_libp2p: bool, ) -> EnrBuilder { let mut builder = EnrBuilder::new("v4"); let (maybe_ipv4_address, maybe_ipv6_address) = &config.enr_address; @@ -155,15 +157,35 @@ pub fn create_enr_builder_from_config( builder.ip6(*ip); } - if let Some(udp4_port) = config.enr_disc4_port { + if let Some(udp4_port) = config.enr_udp4_port { builder.udp4(udp4_port); } - if let Some(udp6_port) = config.enr_disc6_port { + if let Some(udp6_port) = config.enr_udp6_port { builder.udp6(udp6_port); } - if enable_tcp { + // Add QUIC fields to the ENR. + // If `enable_libp2p` is disabled, then we should not support QUIC in the ENR either. + if enable_libp2p && !config.disable_quic_support { + // If we are listening on ipv4, add the quic ipv4 port + if let Some(quic4_port) = config + .enr_quic4_port + .or_else(|| config.listen_addrs().v4().map(|v4_addr| v4_addr.quic_port)) + { + builder.add_value(QUIC_ENR_KEY, &quic4_port); + } + + // If we are listening on ipv6, add the quic ipv6 port + if let Some(quic6_port) = config + .enr_quic6_port + .or_else(|| config.listen_addrs().v6().map(|v6_addr| v6_addr.quic_port)) + { + builder.add_value(QUIC6_ENR_KEY, &quic6_port); + } + } + + if enable_libp2p { // If the ENR port is not set, and we are listening over that ip version, use the listening port instead. let tcp4_port = config .enr_tcp4_port @@ -218,6 +240,9 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool { // tcp ports must match && local_enr.tcp4() == disk_enr.tcp4() && local_enr.tcp6() == disk_enr.tcp6() + // quic ports must match + && local_enr.quic() == disk_enr.quic() + && local_enr.quic6() == disk_enr.quic6() // must match on the same fork && local_enr.get(ETH2_ENR_KEY) == disk_enr.get(ETH2_ENR_KEY) // take preference over disk udp port if one is not specified @@ -258,4 +283,4 @@ pub fn save_enr_to_disk(dir: &Path, enr: &Enr, log: &slog::Logger) { ); } } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index f1a04c9d7c9..8ab882728f2 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -6,8 +6,8 @@ use libp2p::core::multiaddr::Protocol; use libp2p::identity::{ed25519, secp256k1, KeyType, Keypair, PublicKey}; use tiny_keccak::{Hasher, Keccak}; -const QUIC_ENR_KEY: &str = "quic"; -const QUIC6_ENR_KEY: &str = "quic6"; +pub const QUIC_ENR_KEY: &str = "quic"; +pub const QUIC6_ENR_KEY: &str = "quic6"; /// Extend ENR for libp2p types. pub trait EnrExt { @@ -397,4 +397,4 @@ mod tests { assert_eq!(enr.node_id(), node_id); } -} +} \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index e55df7d4814..145854a4845 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -60,26 +60,31 @@ impl ListenAddress { /// Returns the addresses the Swarm will listen on, given the setup. pub fn listen_addresses(&self) -> impl Iterator { - let v4_tcp_multiaddrs = self + let v4_tcp_multiaddr = self .v4() .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); - let v4_quic_multiaddrs = self.v4().map(|v4_addr| { + let v4_quic_multiaddr = self.v4().map(|v4_addr| { Multiaddr::from(v4_addr.addr) .with(Protocol::Udp(v4_addr.quic_port)) .with(Protocol::QuicV1) }); - let v6_tcp_multiaddrs = self + let v6_quic_multiaddr = self.v6().map(|v6_addr| { + Multiaddr::from(v6_addr.addr) + .with(Protocol::Udp(v6_addr.quic_port)) + .with(Protocol::QuicV1) + }); + + let v6_tcp_multiaddr = self .v6() .map(|v6_addr| Multiaddr::from(v6_addr.addr).with(Protocol::Tcp(v6_addr.tcp_port))); - // TODO: Add QUIC IPv6 multiaddr once it is supported - - v4_tcp_multiaddrs + v4_tcp_multiaddr .into_iter() - .chain(v4_quic_multiaddrs) - .chain(v6_tcp_multiaddrs) + .chain(v4_quic_multiaddr) + .chain(v6_quic_multiaddr) + .chain(v6_tcp_multiaddr) } #[cfg(test)] @@ -123,4 +128,4 @@ impl slog::KV for ListenAddress { } slog::Result::Ok(()) } -} +} \ No newline at end of file diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 3c126bc727e..6ad6943b0d9 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -175,7 +175,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-udp-port") .value_name("PORT") .help("The UDP4 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV4.") + can connect to your local node on this port over IPv4.") + .takes_value(true), + ) + .arg( + Arg::with_name("enr-quic-port") + .long("enr-quic-port") + .value_name("PORT") + .help("The quic UDP4 port that will be set on the local ENR. Set this only if you are sure other nodes \ + can connect to your local node on this port over IPv4.") .takes_value(true), ) .arg( @@ -183,7 +191,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-udp6-port") .value_name("PORT") .help("The UDP6 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV6.") + can connect to your local node on this port over IPv6.") + .takes_value(true), + ) + .arg( + Arg::with_name("enr-quic6-port") + .long("enr-quic6-port") + .value_name("PORT") + .help("The quic UDP6 port that will be set on the local ENR. Set this only if you are sure other nodes \ + can connect to your local node on this port over IPv6.") .takes_value(true), ) .arg( @@ -191,7 +207,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-tcp-port") .value_name("PORT") .help("The TCP4 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV4. The --port flag is \ + can connect to your local node on this port over IPv4. The --port flag is \ used if this is not set.") .takes_value(true), ) @@ -200,7 +216,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-tcp6-port") .value_name("PORT") .help("The TCP6 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV6. The --port6 flag is \ + can connect to your local node on this port over IPv6. The --port6 flag is \ used if this is not set.") .takes_value(true), ) @@ -1166,4 +1182,4 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .possible_values(ProgressiveBalancesMode::VARIANTS) ) -} +} \ No newline at end of file diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 6923e12341b..39a79f1945a 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -77,6 +77,7 @@ pub fn get_config( let data_dir_ref = client_config.data_dir().clone(); + println!("SET NETWORK CONFIG"); set_network_config(&mut client_config.network, cli_args, &data_dir_ref, log)?; /* @@ -916,13 +917,21 @@ pub fn set_network_config( } if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp-port") { - config.enr_disc4_port = Some( + config.enr_udp4_port = Some( enr_udp_port_str .parse::() .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?, ); } + if let Some(enr_quic_port_str) = cli_args.value_of("enr-quic-port") { + config.enr_quic4_port = Some( + enr_quic_port_str + .parse::() + .map_err(|_| format!("Invalid quic port: {}", enr_quic_port_str))?, + ); + } + if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp-port") { config.enr_tcp4_port = Some( enr_tcp_port_str @@ -932,13 +941,21 @@ pub fn set_network_config( } if let Some(enr_udp_port_str) = cli_args.value_of("enr-udp6-port") { - config.enr_disc6_port = Some( + config.enr_udp6_port = Some( enr_udp_port_str .parse::() .map_err(|_| format!("Invalid discovery port: {}", enr_udp_port_str))?, ); } + if let Some(enr_quic_port_str) = cli_args.value_of("enr-quic6-port") { + config.enr_quic6_port = Some( + enr_quic_port_str + .parse::() + .map_err(|_| format!("Invalid quic port: {}", enr_quic_port_str))?, + ); + } + if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp6-port") { config.enr_tcp6_port = Some( enr_tcp_port_str @@ -958,7 +975,7 @@ pub fn set_network_config( ipv4_addr.addr }; config.enr_address.0 = Some(ipv4_enr_addr); - config.enr_disc4_port = Some(ipv4_addr.disc_port); + config.enr_udp4_port = Some(ipv4_addr.disc_port); } if let Some(ipv6_addr) = config.listen_addrs().v6().cloned() { @@ -968,7 +985,7 @@ pub fn set_network_config( ipv6_addr.addr }; config.enr_address.1 = Some(ipv6_enr_addr); - config.enr_disc6_port = Some(ipv6_addr.disc_port); + config.enr_udp6_port = Some(ipv6_addr.disc_port); } } @@ -1132,6 +1149,7 @@ pub fn parse_listening_addresses( cli_args: &ArgMatches, log: &Logger, ) -> Result { + dbg!("PARSE"); let listen_addresses_str = cli_args .values_of("listen-address") .expect("--listen_addresses has a default value"); @@ -1215,6 +1233,8 @@ pub fn parse_listening_addresses( format!("Failed to parse --quic6-port as an integer: {parse_error}") })?; + println!("{:?}", maybe_disc6_port); + println!("{:?}", (maybe_ipv4, maybe_ipv6)); // Now put everything together let listening_addresses = match (maybe_ipv4, maybe_ipv6) { (None, None) => { @@ -1237,18 +1257,9 @@ pub fn parse_listening_addresses( warn!(log, "When listening only over IPv6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") } - // TODO: Remove this warning once https://github.com/libp2p/rust-libp2p/issues/4165 - // is resolved. - if maybe_quic_port.is_some() || maybe_quic6_port.is_some() { - warn!(log, "QUIC is currently disabled over Ipv6.") - } - - // TODO: Once QUIC is supported over IPv6, uncomment this. - /* if maybe_quic6_port.is_some() { warn!(log, "When listening only over IPv6, use the --quic-port flag. The value of --quic-port6 will be ignored.") } - */ // use zero ports if required. If not, use the specific udp port. If none given, use // the tcp port. @@ -1322,11 +1333,13 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp6_port) .transpose()? .unwrap_or(port6); + println!("{:?}", maybe_disc6_port); let ipv6_disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? .or(maybe_disc6_port) .unwrap_or(ipv6_tcp_port); + println!("{:?}", ipv6_disc_port); let ipv6_quic_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? @@ -1425,4 +1438,4 @@ where .into_iter() .next() .ok_or(format!("Must provide at least one value to {}", flag_name)) -} +} \ No newline at end of file diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index 7c5f2dd0f52..cef57e4ed41 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -59,17 +59,17 @@ impl BootNodeConfig { // Set the Enr Discovery ports to the listening ports if not present. if let Some(listening_addr_v4) = network_config.listen_addrs().v4() { - network_config.enr_disc4_port = Some( + network_config.enr_udp4_port = Some( network_config - .enr_disc4_port + .enr_udp4_port .unwrap_or(listening_addr_v4.disc_port), ) }; if let Some(listening_addr_v6) = network_config.listen_addrs().v6() { - network_config.enr_disc6_port = Some( + network_config.enr_udp6_port = Some( network_config - .enr_disc6_port + .enr_udp6_port .unwrap_or(listening_addr_v6.disc_port), ) }; @@ -190,4 +190,4 @@ impl BootNodeConfigSerialization { enable_enr_auto_update: discv5_config.enr_update, } } -} +} \ No newline at end of file diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index bc5f610881b..937da98bbb8 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -991,12 +991,12 @@ fn network_port_flag_over_ipv4() { .run() .with_config(|config| { assert_eq!( - config - .network - .listen_addrs() - .v4() - .map(|listen_addr| (listen_addr.udp_port, listen_addr.tcp_port)), - Some((port, port)) + config.network.listen_addrs().v4().map(|listen_addr| ( + listen_addr.disc_port, + listen_addr.quic_port, + listen_addr.tcp_port + )), + Some((port, port + 1, port)) ); }); } @@ -1009,22 +1009,22 @@ fn network_port_flag_over_ipv6() { .run() .with_config(|config| { assert_eq!( - config - .network - .listen_addrs() - .v6() - .map(|listen_addr| (listen_addr.udp_port, listen_addr.tcp_port)), - Some((port, port)) + config.network.listen_addrs().v6().map(|listen_addr| ( + listen_addr.disc_port, + listen_addr.quic_port, + listen_addr.tcp_port + )), + Some((port, port + 1, port)) ); }); } #[test] fn network_port_and_discovery_port_flags_over_ipv4() { let tcp4_port = unused_tcp4_port().expect("Unable to find unused port."); - let udp4_port = unused_udp4_port().expect("Unable to find unused port."); + let disc4_port = unused_udp4_port().expect("Unable to find unused port."); CommandLineTest::new() .flag("port", Some(tcp4_port.to_string().as_str())) - .flag("discovery-port", Some(udp4_port.to_string().as_str())) + .flag("discovery-port", Some(disc4_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( @@ -1032,19 +1032,19 @@ fn network_port_and_discovery_port_flags_over_ipv4() { .network .listen_addrs() .v4() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp4_port, udp4_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp4_port, disc4_port)) ); }); } #[test] fn network_port_and_discovery_port_flags_over_ipv6() { let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); - let udp6_port = unused_udp6_port().expect("Unable to find unused port."); + let disc6_port = unused_udp6_port().expect("Unable to find unused port."); CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("port", Some(tcp6_port.to_string().as_str())) - .flag("discovery-port", Some(udp6_port.to_string().as_str())) + .flag("discovery-port", Some(disc6_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( @@ -1052,24 +1052,25 @@ fn network_port_and_discovery_port_flags_over_ipv6() { .network .listen_addrs() .v6() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp6_port, udp6_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp6_port, disc6_port)) ); }); } #[test] fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { let tcp4_port = unused_tcp4_port().expect("Unable to find unused port."); - let udp4_port = unused_udp4_port().expect("Unable to find unused port."); + let disc4_port = unused_udp4_port().expect("Unable to find unused port."); let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); - let udp6_port = unused_udp6_port().expect("Unable to find unused port."); + let disc6_port = unused_udp6_port().expect("Unable to find unused port."); + println!("Thing"); CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("listen-address", Some("127.0.0.1")) .flag("port", Some(tcp4_port.to_string().as_str())) - .flag("discovery-port", Some(udp4_port.to_string().as_str())) + .flag("discovery-port", Some(disc4_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) - .flag("discovery-port6", Some(udp6_port.to_string().as_str())) + .flag("discovery-port6", Some(disc6_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( @@ -1077,8 +1078,8 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { .network .listen_addrs() .v4() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp4_port, udp4_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp4_port, disc4_port)) ); assert_eq!( @@ -1086,8 +1087,47 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { .network .listen_addrs() .v6() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp6_port, udp6_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp6_port, disc6_port)) + ); + }); +} + +#[test] +fn network_port_discovery_quic_port_flags_over_ipv4_and_ipv6() { + let tcp4_port = unused_tcp4_port().expect("Unable to find unused port."); + let disc4_port = unused_udp4_port().expect("Unable to find unused port."); + let quic4_port = unused_udp4_port().expect("Unable to find unused port."); + let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); + let disc6_port = unused_udp6_port().expect("Unable to find unused port."); + let quic6_port = unused_udp6_port().expect("Unable to find unused port."); + CommandLineTest::new() + .flag("listen-address", Some("::1")) + .flag("listen-address", Some("127.0.0.1")) + .flag("port", Some(tcp4_port.to_string().as_str())) + .flag("discovery-port", Some(disc4_port.to_string().as_str())) + .flag("quic-port", Some(quic4_port.to_string().as_str())) + .flag("port6", Some(tcp6_port.to_string().as_str())) + .flag("discovery-port6", Some(disc6_port.to_string().as_str())) + .flag("quic-port6", Some(disc6_port.to_string().as_str())) + .run() + .with_config(|config| { + assert_eq!( + config.network.listen_addrs().v4().map(|listen_addr| ( + listen_addr.tcp_port, + listen_addr.disc_port, + listen_addr.quic_port + )), + Some((tcp4_port, disc4_port, quic4_port)) + ); + + assert_eq!( + config.network.listen_addrs().v6().map(|listen_addr| ( + listen_addr.tcp_port, + listen_addr.disc_port, + listen_addr.quic_port + )), + Some((tcp6_port, disc6_port, quic6_port)) ); }); } @@ -1099,6 +1139,14 @@ fn disable_discovery_flag() { .run_with_zero_port() .with_config(|config| assert!(config.network.disable_discovery)); } + +#[test] +fn disable_quic_flag() { + CommandLineTest::new() + .flag("disable-quic", None) + .run_with_zero_port() + .with_config(|config| assert!(config.network.disable_quic_support)); +} #[test] fn disable_peer_scoring_flag() { CommandLineTest::new() @@ -1205,6 +1253,14 @@ fn enr_udp_port_flag() { .with_config(|config| assert_eq!(config.network.enr_udp4_port, Some(port))); } #[test] +fn enr_quic_port_flag() { + let port = unused_udp4_port().expect("Unable to find unused port."); + CommandLineTest::new() + .flag("enr-quic-port", Some(port.to_string().as_str())) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.network.enr_quic4_port, Some(port))); +} +#[test] fn enr_tcp_port_flag() { let port = unused_tcp4_port().expect("Unable to find unused port."); CommandLineTest::new() @@ -1221,6 +1277,14 @@ fn enr_udp6_port_flag() { .with_config(|config| assert_eq!(config.network.enr_udp6_port, Some(port))); } #[test] +fn enr_quic6_port_flag() { + let port = unused_udp6_port().expect("Unable to find unused port."); + CommandLineTest::new() + .flag("enr-quic-port", Some(port.to_string().as_str())) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.network.enr_quic6_port, Some(port))); +} +#[test] fn enr_tcp6_port_flag() { let port = unused_tcp6_port().expect("Unable to find unused port."); CommandLineTest::new() @@ -1243,7 +1307,7 @@ fn enr_match_flag_over_ipv4() { assert_eq!( config.network.listen_addrs().v4().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((addr, udp4_port, tcp4_port)) @@ -1268,7 +1332,7 @@ fn enr_match_flag_over_ipv6() { assert_eq!( config.network.listen_addrs().v6().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((addr, udp6_port, tcp6_port)) @@ -1300,7 +1364,7 @@ fn enr_match_flag_over_ipv4_and_ipv6() { assert_eq!( config.network.listen_addrs().v6().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((ipv6_addr, udp6_port, tcp6_port)) @@ -1308,7 +1372,7 @@ fn enr_match_flag_over_ipv4_and_ipv6() { assert_eq!( config.network.listen_addrs().v4().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((ipv4_addr, udp4_port, tcp4_port)) @@ -2294,4 +2358,4 @@ fn progressive_balances_fast() { ProgressiveBalancesMode::Fast ) }); -} +} \ No newline at end of file diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 5c6ef30cd65..26e91065654 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -156,7 +156,7 @@ impl LocalNetwork { std::net::Ipv4Addr::UNSPECIFIED, BOOTNODE_PORT + count, BOOTNODE_PORT + count, - BOOTNODE_PORT + count + 1, + BOOTNODE_PORT + 1000 + count, // Offset the quick ports by 1000 ); beacon_config.network.enr_disc4_port = Some(BOOTNODE_PORT + count); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT + count); @@ -315,4 +315,4 @@ impl LocalNetwork { ); genesis_time - now } -} +} \ No newline at end of file From 632a57b017148c527001d328a2dd5f60ea45cc6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 10 Aug 2023 11:42:34 +0100 Subject: [PATCH 22/70] cargo fmt --- beacon_node/lighthouse_network/src/config.rs | 2 +- beacon_node/lighthouse_network/src/discovery/enr.rs | 2 +- beacon_node/lighthouse_network/src/discovery/enr_ext.rs | 2 +- beacon_node/src/cli.rs | 2 +- beacon_node/src/config.rs | 2 +- boot_node/src/config.rs | 2 +- lighthouse/tests/beacon_node.rs | 2 +- testing/simulator/src/local_network.rs | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 293afc2db51..508e9339f43 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -598,4 +598,4 @@ pub const fn is_global_ipv6(addr: &Ipv6Addr) -> bool { || is_documentation(addr) || is_unique_local(addr) || is_unicast_link_local(addr)) -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index b07d55776bd..6921b3aeec9 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -283,4 +283,4 @@ pub fn save_enr_to_disk(dir: &Path, enr: &Enr, log: &slog::Logger) { ); } } -} \ No newline at end of file +} diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 8ab882728f2..feff96fe775 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -397,4 +397,4 @@ mod tests { assert_eq!(enr.node_id(), node_id); } -} \ No newline at end of file +} diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 6ad6943b0d9..c1452a92d9c 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1182,4 +1182,4 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .possible_values(ProgressiveBalancesMode::VARIANTS) ) -} \ No newline at end of file +} diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 39a79f1945a..22ef5ba904a 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1438,4 +1438,4 @@ where .into_iter() .next() .ok_or(format!("Must provide at least one value to {}", flag_name)) -} \ No newline at end of file +} diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index cef57e4ed41..9c7c2973abf 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -190,4 +190,4 @@ impl BootNodeConfigSerialization { enable_enr_auto_update: discv5_config.enr_update, } } -} \ No newline at end of file +} diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 937da98bbb8..4a07df3ea33 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2358,4 +2358,4 @@ fn progressive_balances_fast() { ProgressiveBalancesMode::Fast ) }); -} \ No newline at end of file +} diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 26e91065654..02d844bf3d0 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -315,4 +315,4 @@ impl LocalNetwork { ); genesis_time - now } -} \ No newline at end of file +} From 2def400ed09b9eecf275f384d1edc4d10457e56b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 10 Aug 2023 12:30:48 +0100 Subject: [PATCH 23/70] remove leftover println's --- beacon_node/src/config.rs | 7 +------ lighthouse/tests/beacon_node.rs | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 22ef5ba904a..2663eb907af 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -77,7 +77,6 @@ pub fn get_config( let data_dir_ref = client_config.data_dir().clone(); - println!("SET NETWORK CONFIG"); set_network_config(&mut client_config.network, cli_args, &data_dir_ref, log)?; /* @@ -1233,8 +1232,6 @@ pub fn parse_listening_addresses( format!("Failed to parse --quic6-port as an integer: {parse_error}") })?; - println!("{:?}", maybe_disc6_port); - println!("{:?}", (maybe_ipv4, maybe_ipv6)); // Now put everything together let listening_addresses = match (maybe_ipv4, maybe_ipv6) { (None, None) => { @@ -1333,13 +1330,11 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp6_port) .transpose()? .unwrap_or(port6); - println!("{:?}", maybe_disc6_port); let ipv6_disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? .or(maybe_disc6_port) .unwrap_or(ipv6_tcp_port); - println!("{:?}", ipv6_disc_port); let ipv6_quic_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? @@ -1438,4 +1433,4 @@ where .into_iter() .next() .ok_or(format!("Must provide at least one value to {}", flag_name)) -} +} \ No newline at end of file diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 4a07df3ea33..817885f5d35 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1063,7 +1063,6 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { let disc4_port = unused_udp4_port().expect("Unable to find unused port."); let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); let disc6_port = unused_udp6_port().expect("Unable to find unused port."); - println!("Thing"); CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("listen-address", Some("127.0.0.1")) @@ -2358,4 +2357,4 @@ fn progressive_balances_fast() { ProgressiveBalancesMode::Fast ) }); -} +} \ No newline at end of file From 7de61b58b8ded5e8fa5beba5e46e38fc92474e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 10 Aug 2023 12:31:03 +0100 Subject: [PATCH 24/70] fix benchmarks --- beacon_node/lighthouse_network/tests/common.rs | 2 +- beacon_node/src/config.rs | 2 +- lcli/src/generate_bootnode_enr.rs | 2 +- lighthouse/tests/beacon_node.rs | 2 +- testing/simulator/src/local_network.rs | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 9ff10f278d0..c190ec0143c 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -76,7 +76,7 @@ pub fn build_config(port: u16, mut boot_nodes: Vec) -> NetworkConfig { .unwrap(); config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port, port + 1); - config.enr_disc4_port = Some(port); + config.enr_udp4_port = Some(port); config.enr_address = (Some(std::net::Ipv4Addr::LOCALHOST), None); config.boot_nodes_enr.append(&mut boot_nodes); config.network_dir = path.into_path(); diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 2663eb907af..7ef97f6168f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1433,4 +1433,4 @@ where .into_iter() .next() .ok_or(format!("Must provide at least one value to {}", flag_name)) -} \ No newline at end of file +} diff --git a/lcli/src/generate_bootnode_enr.rs b/lcli/src/generate_bootnode_enr.rs index 84151b6d1e5..0584cd65496 100644 --- a/lcli/src/generate_bootnode_enr.rs +++ b/lcli/src/generate_bootnode_enr.rs @@ -27,7 +27,7 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { let mut config = NetworkConfig::default(); config.enr_address = (Some(ip), None); - config.enr_disc4_port = Some(udp_port); + config.enr_udp4_port = Some(udp_port); config.enr_tcp6_port = Some(tcp_port); let secp256k1_keypair = secp256k1::Keypair::generate(); diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 817885f5d35..55a28ed927a 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2357,4 +2357,4 @@ fn progressive_balances_fast() { ProgressiveBalancesMode::Fast ) }); -} \ No newline at end of file +} diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 02d844bf3d0..70ba1f4759d 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -66,7 +66,7 @@ impl LocalNetwork { BOOTNODE_PORT, QUIC_PORT, ); - beacon_config.network.enr_disc4_port = Some(BOOTNODE_PORT); + beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT); beacon_config.network.discv5_config.table_filter = |_| true; @@ -158,7 +158,7 @@ impl LocalNetwork { BOOTNODE_PORT + count, BOOTNODE_PORT + 1000 + count, // Offset the quick ports by 1000 ); - beacon_config.network.enr_disc4_port = Some(BOOTNODE_PORT + count); + beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT + count); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT + count); beacon_config.network.discv5_config.table_filter = |_| true; beacon_config.network.proposer_only = is_proposer; From 477163410414b493304208b7cfb0844e054d6546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 10 Aug 2023 16:33:04 +0100 Subject: [PATCH 25/70] fix debug tests --- beacon_node/network/src/service/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 9943da15a0d..e29d079eb3a 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -62,7 +62,7 @@ mod tests { ); let mut config = NetworkConfig::default(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 21212, 21212); + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 21212, 21212, 21212); config.discv5_config.table_filter = |_| true; // Do not ignore local IPs config.upnp_enabled = false; config.boot_nodes_enr = enrs.clone(); From 8858f0e11c866b0c4fba2a136f532812ed63b102 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 11 Aug 2023 20:30:15 +1000 Subject: [PATCH 26/70] Correct port mapping bug --- beacon_node/lighthouse_network/src/config.rs | 2 +- beacon_node/lighthouse_network/src/listen_addr.rs | 3 --- beacon_node/src/config.rs | 10 ++-------- lighthouse/tests/beacon_node.rs | 1 - 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 293afc2db51..d4533f1e088 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -208,8 +208,8 @@ impl Config { quic4_port: u16, v6_addr: Ipv6Addr, tcp6_port: u16, - quic6_port: u16, disc6_port: u16, + quic6_port: u16, ) { self.listen_addresses = ListenAddress::DualStack( ListenAddr { diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 145854a4845..86415d9500c 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -11,9 +11,6 @@ pub struct ListenAddr { /// The UDP port that discovery will listen on. pub disc_port: u16, /// The UDP port that QUIC will listen on. - /// NB: Quic port is not yet supported with IPV6 - /// See https://github.com/libp2p/rust-libp2p/issues/4165 - /// It will therefore be disabled at the swarm level. pub quic_port: u16, /// The TCP port that libp2p will listen on. pub tcp_port: u16, diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 39a79f1945a..b13b0e32d49 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -77,7 +77,6 @@ pub fn get_config( let data_dir_ref = client_config.data_dir().clone(); - println!("SET NETWORK CONFIG"); set_network_config(&mut client_config.network, cli_args, &data_dir_ref, log)?; /* @@ -965,9 +964,9 @@ pub fn set_network_config( } if cli_args.is_present("enr-match") { - // Match the Ip and UDP port in the enr. + // Match the IP and UDP port in the ENR. - // set the enr address to localhost if the address is unspecified + // Set the ENR address to localhost if the address is unspecified if let Some(ipv4_addr) = config.listen_addrs().v4().cloned() { let ipv4_enr_addr = if ipv4_addr.addr == Ipv4Addr::UNSPECIFIED { Ipv4Addr::LOCALHOST @@ -1149,7 +1148,6 @@ pub fn parse_listening_addresses( cli_args: &ArgMatches, log: &Logger, ) -> Result { - dbg!("PARSE"); let listen_addresses_str = cli_args .values_of("listen-address") .expect("--listen_addresses has a default value"); @@ -1233,8 +1231,6 @@ pub fn parse_listening_addresses( format!("Failed to parse --quic6-port as an integer: {parse_error}") })?; - println!("{:?}", maybe_disc6_port); - println!("{:?}", (maybe_ipv4, maybe_ipv6)); // Now put everything together let listening_addresses = match (maybe_ipv4, maybe_ipv6) { (None, None) => { @@ -1333,13 +1329,11 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp6_port) .transpose()? .unwrap_or(port6); - println!("{:?}", maybe_disc6_port); let ipv6_disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? .or(maybe_disc6_port) .unwrap_or(ipv6_tcp_port); - println!("{:?}", ipv6_disc_port); let ipv6_quic_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 937da98bbb8..817885f5d35 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1063,7 +1063,6 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { let disc4_port = unused_udp4_port().expect("Unable to find unused port."); let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); let disc6_port = unused_udp6_port().expect("Unable to find unused port."); - println!("Thing"); CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("listen-address", Some("127.0.0.1")) From ee673510092e9e37fb99b6353574c84a1c74374d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 11 Aug 2023 14:45:20 +0100 Subject: [PATCH 27/70] fix enr-quic6-port test --- lighthouse/tests/beacon_node.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 20d24708144..36be85ba278 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1280,7 +1280,7 @@ fn enr_udp6_port_flag() { fn enr_quic6_port_flag() { let port = unused_udp6_port().expect("Unable to find unused port."); CommandLineTest::new() - .flag("enr-quic-port", Some(port.to_string().as_str())) + .flag("enr-quic6-port", Some(port.to_string().as_str())) .run_with_zero_port() .with_config(|config| assert_eq!(config.network.enr_quic6_port, Some(port))); } From 4c5cffc2c9e20655576973c24934cacde06dbf6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Sat, 12 Aug 2023 09:59:22 +0100 Subject: [PATCH 28/70] update libp2p-quic to 0.9.2 --- Cargo.lock | 5 +++-- beacon_node/lighthouse_network/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e240f3dfaf3..679f36ef894 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4227,9 +4227,9 @@ dependencies = [ [[package]] name = "libp2p-quic" -version = "0.9.0-alpha" +version = "0.9.2-alpha" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af35e582535de00bbac3d62412d53c1f07c80f1b6ea1451716b9e4ed5002498d" +checksum = "efb614a60b5cef499aec647b5514efbd89df7740b0e27e90577c95e6b556f7c6" dependencies = [ "bytes", "futures", @@ -4243,6 +4243,7 @@ dependencies = [ "quinn", "rand 0.8.5", "rustls 0.21.5", + "socket2 0.5.3", "thiserror", "tokio", ] diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index b86749913fc..843db6b104f 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -44,7 +44,7 @@ prometheus-client = "0.21.0" unused_port = { path = "../../common/unused_port" } delay_map = "0.3.0" void = "1" -libp2p-quic= { version = "0.9.0-alpha", features=["tokio"]} +libp2p-quic= { version = "0.9.2-alpha", features=["tokio"]} [dependencies.libp2p] version = "0.52" From 68d3a84ee78e0b0c06fe81406c6d0fe827aee8d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Sat, 12 Aug 2023 10:28:53 +0100 Subject: [PATCH 29/70] cargo clippy --- .../lighthouse_network/src/discovery/enr.rs | 36 +++++++++---------- .../lighthouse_network/src/service/utils.rs | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index 6921b3aeec9..859c3a89430 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -165,27 +165,27 @@ pub fn create_enr_builder_from_config( builder.udp6(udp6_port); } - // Add QUIC fields to the ENR. - // If `enable_libp2p` is disabled, then we should not support QUIC in the ENR either. - if enable_libp2p && !config.disable_quic_support { - // If we are listening on ipv4, add the quic ipv4 port - if let Some(quic4_port) = config - .enr_quic4_port - .or_else(|| config.listen_addrs().v4().map(|v4_addr| v4_addr.quic_port)) - { - builder.add_value(QUIC_ENR_KEY, &quic4_port); - } + if enable_libp2p { + // Add QUIC fields to the ENR. + // If `enable_libp2p` is disabled, then we should not support QUIC in the ENR either. + if !config.disable_quic_support { + // If we are listening on ipv4, add the quic ipv4 port + if let Some(quic4_port) = config + .enr_quic4_port + .or_else(|| config.listen_addrs().v4().map(|v4_addr| v4_addr.quic_port)) + { + builder.add_value(QUIC_ENR_KEY, &quic4_port); + } - // If we are listening on ipv6, add the quic ipv6 port - if let Some(quic6_port) = config - .enr_quic6_port - .or_else(|| config.listen_addrs().v6().map(|v6_addr| v6_addr.quic_port)) - { - builder.add_value(QUIC6_ENR_KEY, &quic6_port); + // If we are listening on ipv6, add the quic ipv6 port + if let Some(quic6_port) = config + .enr_quic6_port + .or_else(|| config.listen_addrs().v6().map(|v6_addr| v6_addr.quic_port)) + { + builder.add_value(QUIC6_ENR_KEY, &quic6_port); + } } - } - if enable_libp2p { // If the ENR port is not set, and we are listening over that ip version, use the listening port instead. let tcp4_port = config .enr_tcp4_port diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index c9dbd7ab8b2..628301fb236 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -74,7 +74,7 @@ pub fn build_transport( transport.with_bandwidth_logging() }; - Ok((transport.boxed(), bandwidth)) + Ok((transport, bandwidth)) } // Useful helper functions for debugging. Currently not used in the client. From fbdd01523fe140edab616382d9feaf6c8a5b5b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Sat, 12 Aug 2023 11:20:49 +0100 Subject: [PATCH 30/70] EnrExt: rename quic to quic4 for consistency --- beacon_node/lighthouse_network/src/discovery/enr.rs | 2 +- .../lighthouse_network/src/discovery/enr_ext.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index 859c3a89430..9c0b3fe99b2 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -241,7 +241,7 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool { && local_enr.tcp4() == disk_enr.tcp4() && local_enr.tcp6() == disk_enr.tcp6() // quic ports must match - && local_enr.quic() == disk_enr.quic() + && local_enr.quic4() == disk_enr.quic4() && local_enr.quic6() == disk_enr.quic6() // must match on the same fork && local_enr.get(ETH2_ENR_KEY) == disk_enr.get(ETH2_ENR_KEY) diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index feff96fe775..10ccfb2678d 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -34,7 +34,7 @@ pub trait EnrExt { fn multiaddr_quic(&self) -> Vec; /// Returns the quic port if one is set. - fn quic(&self) -> Option; + fn quic4(&self) -> Option; /// Returns the quic6 port if one is set. fn quic6(&self) -> Option; @@ -62,7 +62,7 @@ impl EnrExt for Enr { } /// Returns the quic port if one is set. - fn quic(&self) -> Option { + fn quic4(&self) -> Option { self.get_decodable(QUIC_ENR_KEY).and_then(Result::ok) } @@ -71,7 +71,7 @@ impl EnrExt for Enr { self.get_decodable(QUIC6_ENR_KEY).and_then(Result::ok) } - /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. + /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp`, `quic` or `udp` key **or** an `ip6` and either a `tcp6` `quic6` or `udp6`. /// The vector remains empty if these fields are not defined. fn multiaddr(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); @@ -81,7 +81,7 @@ impl EnrExt for Enr { multiaddr.push(Protocol::Udp(udp)); multiaddrs.push(multiaddr); } - if let Some(quic) = self.quic() { + if let Some(quic) = self.quic4() { let mut multiaddr: Multiaddr = ip.into(); multiaddr.push(Protocol::Udp(quic)); multiaddr.push(Protocol::QuicV1); @@ -214,7 +214,7 @@ impl EnrExt for Enr { fn multiaddr_quic(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); // Check for quic first as it is less likely - if let Some(quic_port) = self.quic() { + if let Some(quic_port) = self.quic4() { if let Some(ip) = self.ip4() { let mut multiaddr: Multiaddr = ip.into(); multiaddr.push(Protocol::Udp(quic_port)); From a10019a2770c772e12d391a9cf39c7d6e8daa307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Sat, 12 Aug 2023 12:20:04 +0100 Subject: [PATCH 31/70] increase quic port on dht persistance test --- beacon_node/network/src/service/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 0c51be38a8f..23bcf456dee 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -60,7 +60,7 @@ mod tests { ); let mut config = NetworkConfig::default(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 21212, 21212, 21212); + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 21212, 21212, 21213); config.discv5_config.table_filter = |_| true; // Do not ignore local IPs config.upnp_enabled = false; config.boot_nodes_enr = enrs.clone(); From 649890844484538029f22e59eca865f002efa9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 14 Aug 2023 11:35:10 +0100 Subject: [PATCH 32/70] fix quic port on unused_v6_ports test --- beacon_node/lighthouse_network/src/listen_addr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 6b47990b821..74c4ba110b9 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -99,7 +99,7 @@ impl ListenAddress { ListenAddress::V6(ListenAddr { addr: Ipv6Addr::UNSPECIFIED, disc_port: unused_port::unused_udp6_port().unwrap(), - quic_port: unused_port::unused_tcp6_port().unwrap(), + quic_port: unused_port::unused_udp6_port().unwrap(), tcp_port: unused_port::unused_tcp6_port().unwrap(), }) } From 91c4fadc5fd3f999089ccf772591d8588e244c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 14 Aug 2023 13:25:41 +0100 Subject: [PATCH 33/70] make rpc tests use tcp multiaddr, and rename them accordingly --- beacon_node/lighthouse_network/tests/common.rs | 2 +- .../lighthouse_network/tests/rpc_tests.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index c190ec0143c..7915a915d8d 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -139,7 +139,7 @@ pub async fn build_node_pair( let mut sender = build_libp2p_instance(rt.clone(), vec![], sender_log, fork_name, spec).await; let mut receiver = build_libp2p_instance(rt, vec![], receiver_log, fork_name, spec).await; - let receiver_multiaddr = receiver.local_enr().multiaddr()[1].clone(); + let receiver_multiaddr = receiver.local_enr().multiaddr()[2].clone(); // let the two nodes set up listeners let sender_fut = async { diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 05fa5ab8542..2d9eea0e8c2 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -49,7 +49,7 @@ fn merge_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBloc // Tests the STATUS RPC message #[test] #[allow(clippy::single_match)] -fn test_status_rpc() { +fn test_tcp_status_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -141,7 +141,7 @@ fn test_status_rpc() { // Tests a streamed BlocksByRange RPC Message #[test] #[allow(clippy::single_match)] -fn test_blocks_by_range_chunked_rpc() { +fn test_tcp_blocks_by_range_chunked_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -267,7 +267,7 @@ fn test_blocks_by_range_chunked_rpc() { // Tests rejection of blocks over `MAX_RPC_SIZE`. #[test] #[allow(clippy::single_match)] -fn test_blocks_by_range_over_limit() { +fn test_tcp_blocks_by_range_over_limit() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -350,7 +350,7 @@ fn test_blocks_by_range_over_limit() { // Tests that a streamed BlocksByRange RPC Message terminates when all expected chunks were received #[test] -fn test_blocks_by_range_chunked_rpc_terminates_correctly() { +fn test_tcp_blocks_by_range_chunked_rpc_terminates_correctly() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -476,7 +476,7 @@ fn test_blocks_by_range_chunked_rpc_terminates_correctly() { // Tests an empty response to a BlocksByRange RPC Message #[test] #[allow(clippy::single_match)] -fn test_blocks_by_range_single_empty_rpc() { +fn test_tcp_blocks_by_range_single_empty_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Trace; let enable_logging = false; @@ -576,7 +576,7 @@ fn test_blocks_by_range_single_empty_rpc() { // serves to test the snappy framing format as well. #[test] #[allow(clippy::single_match)] -fn test_blocks_by_root_chunked_rpc() { +fn test_tcp_blocks_by_root_chunked_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -702,7 +702,7 @@ fn test_blocks_by_root_chunked_rpc() { // Tests a streamed, chunked BlocksByRoot RPC Message terminates when all expected reponses have been received #[test] -fn test_blocks_by_root_chunked_rpc_terminates_correctly() { +fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -836,7 +836,7 @@ fn test_blocks_by_root_chunked_rpc_terminates_correctly() { // Tests a Goodbye RPC message #[test] #[allow(clippy::single_match)] -fn test_goodbye_rpc() { +fn tcp_test_goodbye_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Trace; let enable_logging = false; From ce6746715a337bb97c294337690c2abc315c2abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 14 Aug 2023 19:20:39 +0100 Subject: [PATCH 34/70] fix network_port_discovery_quic_port_flags_over_ipv4_and_ipv6 test --- beacon_node/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b3ec84fd43e..de16b58e614 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1250,7 +1250,7 @@ pub fn parse_listening_addresses( // parse the possible quic port. let maybe_quic6_port = cli_args - .value_of("quic6-port") + .value_of("quic-port6") .map(str::parse::) .transpose() .map_err(|parse_error| { From 3527d913cb8a9909a063ffc890d2fe3287fdaf13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 16 Aug 2023 14:51:50 +0100 Subject: [PATCH 35/70] fix test network_port_discovery_quic_port_flags_over_ipv4_and_ipv6 --- beacon_node/src/config.rs | 2 +- lighthouse/tests/beacon_node.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index de16b58e614..a5213921449 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1364,7 +1364,7 @@ pub fn parse_listening_addresses( .then(unused_port::unused_udp6_port) .transpose()? .or(maybe_quic6_port) - .unwrap_or(ipv6_disc_port + 1); + .unwrap_or(ipv6_tcp_port + 1); ListenAddress::DualStack( lighthouse_network::ListenAddr { diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 36be85ba278..ea268f867f3 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1109,7 +1109,7 @@ fn network_port_discovery_quic_port_flags_over_ipv4_and_ipv6() { .flag("quic-port", Some(quic4_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) .flag("discovery-port6", Some(disc6_port.to_string().as_str())) - .flag("quic-port6", Some(disc6_port.to_string().as_str())) + .flag("quic-port6", Some(quic6_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( From 137bb89c1610297171dd0ce6262224866f6e508b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 16 Aug 2023 16:02:49 +0100 Subject: [PATCH 36/70] add quic to initialized ENR log --- beacon_node/lighthouse_network/src/discovery/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 73819c1a3f2..b3a5518eec2 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -208,7 +208,8 @@ impl Discovery { let local_node_id = local_enr.node_id(); info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq(), "id"=> %local_enr.node_id(), - "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6() + "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6(), + "quic4" => ?local_enr.quic4(), "quic6" => ?local_enr.quic6() ); // convert the keypair into an ENR key From 99f72d04cbb01944bcf9f4c5f6f2e2effe886c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 16 Aug 2023 16:02:49 +0100 Subject: [PATCH 37/70] add quic to ENR echo logs --- beacon_node/lighthouse_network/src/discovery/mod.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 73819c1a3f2..d3bf169d7cd 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -208,7 +208,8 @@ impl Discovery { let local_node_id = local_enr.node_id(); info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq(), "id"=> %local_enr.node_id(), - "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6() + "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6(), + "quic4" => ?local_enr.quic4(), "quic6" => ?local_enr.quic6() ); // convert the keypair into an ENR key @@ -230,7 +231,8 @@ impl Discovery { "peer_id" => %bootnode_enr.peer_id(), "ip" => ?bootnode_enr.ip4(), "udp" => ?bootnode_enr.udp4(), - "tcp" => ?bootnode_enr.tcp4() + "tcp" => ?bootnode_enr.tcp4(), + "quic" => ?bootnode_enr.quic4() ); let repr = bootnode_enr.to_string(); let _ = discv5.add_enr(bootnode_enr).map_err(|e| { @@ -281,7 +283,8 @@ impl Discovery { "peer_id" => %enr.peer_id(), "ip" => ?enr.ip4(), "udp" => ?enr.udp4(), - "tcp" => ?enr.tcp4() + "tcp" => ?enr.tcp4(), + "quic" => ?enr.quic4() ); let _ = discv5.add_enr(enr).map_err(|e| { error!( From adfb7db69a577c1f607d97b41e08f436a0069d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 17 Aug 2023 15:37:09 +0100 Subject: [PATCH 38/70] fix simulations by incrementing quic port by 1000 --- beacon_node/network/src/service.rs | 2 +- testing/simulator/src/local_network.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 987faeb922a..96ab3cd3d16 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -977,7 +977,7 @@ impl Drop for NetworkService { } // attempt to remove port mappings - crate::nat::remove_mappings(self.upnp_mappings.0, &(self.upnp_mappings.1), &self.log); + crate::nat::remove_mappings(self.upnp_mappings.0, &self.upnp_mappings.1, &self.log); info!(self.log, "Network service shutdown"); } diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 70ba1f4759d..69fa8ded023 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -14,7 +14,7 @@ use std::{sync::Arc, time::Duration}; use types::{Epoch, EthSpec}; const BOOTNODE_PORT: u16 = 42424; -const QUIC_PORT: u16 = BOOTNODE_PORT + 1; +const QUIC_PORT: u16 = 43424; pub const INVALID_ADDRESS: &str = "http://127.0.0.1:42423"; pub const EXECUTION_PORT: u16 = 4000; @@ -156,7 +156,7 @@ impl LocalNetwork { std::net::Ipv4Addr::UNSPECIFIED, BOOTNODE_PORT + count, BOOTNODE_PORT + count, - BOOTNODE_PORT + 1000 + count, // Offset the quick ports by 1000 + QUIC_PORT + count, ); beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT + count); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT + count); From 3d3d8c8a37582316f12987572021a67df761466b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Sun, 20 Aug 2023 23:47:24 +0100 Subject: [PATCH 39/70] add quic rpc tests. --- .../lighthouse_network/tests/common.rs | 22 ++- .../lighthouse_network/tests/rpc_tests.rs | 160 +++++++++++++++--- 2 files changed, 161 insertions(+), 21 deletions(-) diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 7915a915d8d..e1d745fa000 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -124,6 +124,12 @@ pub fn get_enr(node: &LibP2PService) -> Enr { node.local_enr() } +// Protocol for the node pair connection. +pub enum Protocol { + Tcp, + Quic, +} + // Constructs a pair of nodes with separate loggers. The sender dials the receiver. // This returns a (sender, receiver) pair. #[allow(dead_code)] @@ -132,6 +138,7 @@ pub async fn build_node_pair( log: &slog::Logger, fork_name: ForkName, spec: &ChainSpec, + protocol: Protocol, ) -> (Libp2pInstance, Libp2pInstance) { let sender_log = log.new(o!("who" => "sender")); let receiver_log = log.new(o!("who" => "receiver")); @@ -139,7 +146,20 @@ pub async fn build_node_pair( let mut sender = build_libp2p_instance(rt.clone(), vec![], sender_log, fork_name, spec).await; let mut receiver = build_libp2p_instance(rt, vec![], receiver_log, fork_name, spec).await; - let receiver_multiaddr = receiver.local_enr().multiaddr()[2].clone(); + let receiver_multiaddr = match protocol { + Protocol::Tcp => receiver + .local_enr() + .multiaddr_tcp() + .first() + .unwrap() + .clone(), + Protocol::Quic => receiver + .local_enr() + .multiaddr_quic() + .first() + .unwrap() + .clone(), + }; // let the two nodes set up listeners let sender_fut = async { diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 2d9eea0e8c2..b2fc70a1f11 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -1,4 +1,8 @@ #![cfg(test)] + +mod common; + +use common::Protocol; use lighthouse_network::rpc::methods::*; use lighthouse_network::{rpc::max_rpc_size, NetworkEvent, ReportSource, Request, Response}; use slog::{debug, warn, Level}; @@ -14,8 +18,6 @@ use types::{ Slot, }; -mod common; - type E = MinimalEthSpec; /// Merge block with length < max_rpc_size. @@ -62,8 +64,14 @@ fn test_tcp_status_rpc() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // Dummy STATUS RPC message let rpc_request = Request::Status(StatusMessage { @@ -156,8 +164,14 @@ fn test_tcp_blocks_by_range_chunked_rpc() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Merge, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Merge, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); @@ -282,8 +296,14 @@ fn test_tcp_blocks_by_range_over_limit() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Merge, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Merge, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); @@ -366,8 +386,14 @@ fn test_tcp_blocks_by_range_chunked_rpc_terminates_correctly() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); @@ -488,8 +514,14 @@ fn test_tcp_blocks_by_range_single_empty_rpc() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, 10)); @@ -589,8 +621,14 @@ fn test_tcp_blocks_by_root_chunked_rpc() { let rt = Arc::new(Runtime::new().unwrap()); // get sender/receiver rt.block_on(async { - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Merge, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Merge, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRoot Request let rpc_request = @@ -716,8 +754,14 @@ fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { let rt = Arc::new(Runtime::new().unwrap()); // get sender/receiver rt.block_on(async { - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRoot Request let rpc_request = @@ -838,8 +882,78 @@ fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { #[allow(clippy::single_match)] fn tcp_test_goodbye_rpc() { // set up the logging. The level and enabled logging or not - let log_level = Level::Trace; - let enable_logging = false; + let log_level = Level::Debug; + let enable_logging = true; + + let log = common::build_log(log_level, enable_logging); + + let rt = Arc::new(Runtime::new().unwrap()); + + let spec = E::default_spec(); + + // get sender/receiver + rt.block_on(async { + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; + + // build the sender future + let sender_future = async { + loop { + match sender.next_event().await { + NetworkEvent::PeerConnectedOutgoing(peer_id) => { + // Send a goodbye and disconnect + debug!(log, "Sending RPC"); + sender.goodbye_peer( + &peer_id, + GoodbyeReason::IrrelevantNetwork, + ReportSource::SyncService, + ); + } + NetworkEvent::PeerDisconnected(_) => { + return; + } + _ => {} // Ignore other RPC messages + } + } + }; + + // build the receiver future + let receiver_future = async { + loop { + match receiver.next_event().await { + NetworkEvent::PeerDisconnected(_) => { + // Should receive sent RPC request + return; + } + _ => {} // Ignore other events + } + } + }; + + let total_future = futures::future::join(sender_future, receiver_future); + + tokio::select! { + _ = total_future => {} + _ = sleep(Duration::from_secs(30)) => { + panic!("Future timed out"); + } + } + }) +} + +// Tests a Goodbye RPC message +#[test] +#[allow(clippy::single_match)] +fn quic_test_goodbye_rpc() { + // set up the logging. The level and enabled logging or not + let log_level = Level::Debug; + let enable_logging = true; let log = common::build_log(log_level, enable_logging); @@ -849,8 +963,14 @@ fn tcp_test_goodbye_rpc() { // get sender/receiver rt.block_on(async { - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Quic, + ) + .await; // build the sender future let sender_future = async { From 1bb942bff9c57ada8f45c0990084798e14457dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 23 Aug 2023 14:05:56 +0100 Subject: [PATCH 40/70] put quic transport in the dns transport --- .../lighthouse_network/src/service/utils.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 628301fb236..0996df71640 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -45,15 +45,12 @@ pub fn build_transport( local_private_key: Keypair, quic_support: bool, ) -> std::io::Result<(BoxedTransport, Arc)> { - // Creates the TCP transport layer - let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)); - // Enables DNS over the TCP transport. - let transport = libp2p::dns::TokioDnsConfig::system(tcp)?; - // yamux config let mut yamux_config = yamux::Config::default(); yamux_config.set_window_update_mode(yamux::WindowUpdateMode::on_read()); - let transport = transport + + // Creates the TCP transport layer + let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)) .upgrade(core::upgrade::Version::V1) .authenticate(generate_noise_config(&local_private_key)) .multiplex(yamux_config) @@ -63,17 +60,19 @@ pub fn build_transport( // Enables Quic // The default quic configuration suits us for now. let quic_config = libp2p_quic::Config::new(&local_private_key); - transport - .or_transport(libp2p_quic::tokio::Transport::new(quic_config)) + tcp.or_transport(libp2p_quic::tokio::Transport::new(quic_config)) .map(|either_output, _| match either_output { Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), }) .with_bandwidth_logging() } else { - transport.with_bandwidth_logging() + tcp.with_bandwidth_logging() }; + // // Enables DNS over the transport. + let transport = libp2p::dns::TokioDnsConfig::system(transport)?.boxed(); + Ok((transport, bandwidth)) } From 5c29f8337b7ddceafcaa31d97e71570ef515a069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 23 Aug 2023 14:12:20 +0100 Subject: [PATCH 41/70] add multiaddress to connection established logging --- .../src/peer_manager/network_behaviour.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 5bd3155d73b..c64221f1d20 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -197,7 +197,11 @@ impl PeerManager { endpoint: &ConnectedPoint, other_established: usize, ) { - debug!(self.log, "Connection established"; "peer_id" => %peer_id, "connection" => ?endpoint.to_endpoint()); + debug!(self.log, "Connection established"; "peer_id" => %peer_id, + "multiaddr" => %endpoint.get_remote_address(), + "connection" => ?endpoint.to_endpoint() + ); + if other_established == 0 { self.events.push(PeerManagerEvent::MetaData(peer_id)); } From 262b1e590322b60dca95b60b42af7b008c30b38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 24 Aug 2023 18:02:43 +0100 Subject: [PATCH 42/70] update local test net scripts for quic --- scripts/local_testnet/beacon_node.sh | 17 ++++++++++------- scripts/local_testnet/start_local_testnet.sh | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index 1a04d12d4a0..70a36614c7a 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -39,10 +39,11 @@ done # Get positional arguments data_dir=${@:$OPTIND+0:1} -network_port=${@:$OPTIND+1:1} -http_port=${@:$OPTIND+2:1} -execution_endpoint=${@:$OPTIND+3:1} -execution_jwt=${@:$OPTIND+4:1} +tcp_port=${@:$OPTIND+1:1} +quic_port=${@:$OPTIND+2:1} +http_port=${@:$OPTIND+3:1} +execution_endpoint=${@:$OPTIND+4:1} +execution_jwt=${@:$OPTIND+5:1} lighthouse_binary=lighthouse @@ -56,9 +57,11 @@ exec $lighthouse_binary \ --disable-peer-scoring \ --staking \ --enr-address 127.0.0.1 \ - --enr-udp-port $network_port \ - --enr-tcp-port $network_port \ - --port $network_port \ + --enr-udp-port $tcp_port \ + --enr-tcp-port $tcp_port \ + --enr-quic-port $quic_port \ + --port $tcp_port \ + --quic-port $quic_port \ --http-port $http_port \ --disable-packet-filter \ --target-peers $((BN_COUNT - 1)) \ diff --git a/scripts/local_testnet/start_local_testnet.sh b/scripts/local_testnet/start_local_testnet.sh index 4b8357b993e..a9b12a252da 100755 --- a/scripts/local_testnet/start_local_testnet.sh +++ b/scripts/local_testnet/start_local_testnet.sh @@ -139,7 +139,7 @@ sed -i 's/"shanghaiTime".*$/"shanghaiTime": 0,/g' $genesis_file for (( bn=1; bn<=$BN_COUNT; bn++ )); do secret=$DATADIR/geth_datadir$bn/geth/jwtsecret echo $secret - execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_http_port_base + $bn)) http://localhost:$((EL_base_auth_http + $bn)) $secret + execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_http_port_base + $bn)) $((BN_udp_tcp_base + $bn + 100)) http://localhost:$((EL_base_auth_http + $bn)) $secret done # Start requested number of validator clients From ac838251f3c29db5d832f766be5867dca866891f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 25 Aug 2023 13:47:49 +0100 Subject: [PATCH 43/70] update libp2p-quic to stable --- beacon_node/lighthouse_network/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index e61de4eb27a..fa4b47935b9 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -44,7 +44,7 @@ prometheus-client = "0.21.0" unused_port = { path = "../../common/unused_port" } delay_map = "0.3.0" void = "1" -libp2p-quic= { version = "0.9.2-alpha", features=["tokio"]} +libp2p-quic= { version = "0.9.2", features=["tokio"]} libp2p-mplex = "0.40.0" [dependencies.libp2p] From 9247496d340c856c2baf11747ec234e57cdccdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 25 Aug 2023 17:27:24 +0100 Subject: [PATCH 44/70] add connected TCP and QUIC peers to prometheus metrics --- beacon_node/lighthouse_network/src/metrics.rs | 10 ++++ .../src/peer_manager/network_behaviour.rs | 59 +++++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index 58cc9920126..880a8763690 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -14,6 +14,16 @@ lazy_static! { "Count of libp2p peers currently connected" ); + pub static ref TCP_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_peers", + "Count of libp2p peers currently connected via TCP" + ); + + pub static ref QUIC_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_peers", + "Count of libp2p peers currently connected via TCP" + ); + pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( "libp2p_peer_connect_event_total", "Count of libp2p peer connect events (not the current number of connected peers)" diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index c64221f1d20..f399a80742c 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -3,7 +3,7 @@ use std::task::{Context, Poll}; use futures::StreamExt; -use libp2p::core::ConnectedPoint; +use libp2p::core::{multiaddr, ConnectedPoint}; use libp2p::identity::PeerId; use libp2p::swarm::behaviour::{ConnectionClosed, ConnectionEstablished, DialFailure, FromSwarm}; use libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; @@ -137,9 +137,11 @@ impl NetworkBehaviour for PeerManager { } FromSwarm::ConnectionClosed(ConnectionClosed { peer_id, + endpoint, + remaining_established, .. - }) => self.on_connection_closed(peer_id, remaining_established), + }) => self.on_connection_closed(peer_id, endpoint, remaining_established), FromSwarm::DialFailure(DialFailure { peer_id, error, @@ -251,25 +253,48 @@ impl PeerManager { // NOTE: We don't register peers that we are disconnecting immediately. The network service // does not need to know about these peers. - match endpoint { + let remote_addr = match endpoint { ConnectedPoint::Listener { send_back_addr, .. } => { self.inject_connect_ingoing(&peer_id, send_back_addr.clone(), None); self.events .push(PeerManagerEvent::PeerConnectedIncoming(peer_id)); + send_back_addr } ConnectedPoint::Dialer { address, .. } => { self.inject_connect_outgoing(&peer_id, address.clone(), None); self.events .push(PeerManagerEvent::PeerConnectedOutgoing(peer_id)); + address } - } + }; // increment prometheus metrics + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::Quic) => { + metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); + } + Some(_) => unreachable!(), + None => error!(self.log, "Connected via unknown transport"; "addr" => %remote_addr), + }; + self.update_connected_peer_metrics(); metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); } - fn on_connection_closed(&mut self, peer_id: PeerId, remaining_established: usize) { + fn on_connection_closed( + &mut self, + peer_id: PeerId, + endpoint: &ConnectedPoint, + remaining_established: usize, + ) { if remaining_established > 0 { return; } @@ -295,7 +320,31 @@ impl PeerManager { // reference so that peer manager can track this peer. self.inject_disconnect(&peer_id); + let remote_addr = match endpoint { + ConnectedPoint::Listener { send_back_addr, .. } => { + send_back_addr + } + ConnectedPoint::Dialer { address, .. } => { + address + } + }; + // Update the prometheus metrics + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::Quic) => { + metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); + } + // If it's an unknown protocol we already logged when connection was established. + _ => {} + }; self.update_connected_peer_metrics(); metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); } From 62363913366d06e212553da1c5f50efd1269829a Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 28 Aug 2023 16:08:24 +1000 Subject: [PATCH 45/70] Differentiate the metrics in prometheus --- beacon_node/lighthouse_network/src/metrics.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index 880a8763690..ae02b689d81 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -15,13 +15,13 @@ lazy_static! { ); pub static ref TCP_PEERS_CONNECTED: Result = try_create_int_gauge( - "libp2p_peers", + "libp2p_tcp_peers", "Count of libp2p peers currently connected via TCP" ); pub static ref QUIC_PEERS_CONNECTED: Result = try_create_int_gauge( - "libp2p_peers", - "Count of libp2p peers currently connected via TCP" + "libp2p_quic_peers", + "Count of libp2p peers currently connected via QUIC" ); pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( From e6294c591067fdda246fc90b14f8c44eeabdbb8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 28 Aug 2023 09:35:16 +0100 Subject: [PATCH 46/70] cargo fmt --- .../src/peer_manager/network_behaviour.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index f399a80742c..b3ab4fc06bb 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -137,7 +137,7 @@ impl NetworkBehaviour for PeerManager { } FromSwarm::ConnectionClosed(ConnectionClosed { peer_id, - endpoint, + endpoint, remaining_established, .. @@ -321,12 +321,8 @@ impl PeerManager { self.inject_disconnect(&peer_id); let remote_addr = match endpoint { - ConnectedPoint::Listener { send_back_addr, .. } => { - send_back_addr - } - ConnectedPoint::Dialer { address, .. } => { - address - } + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + ConnectedPoint::Dialer { address, .. } => address, }; // Update the prometheus metrics From 96a381004ef9adf46bb3a01929be58485ea54192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 28 Aug 2023 13:06:15 +0100 Subject: [PATCH 47/70] remove unrequired features section --- beacon_node/lighthouse_network/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index fa4b47935b9..3e399f74800 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -60,6 +60,3 @@ exit-future = "0.2.0" void = "1" quickcheck = "0.9.2" quickcheck_macros = "0.9.1" - -[features] -libp2p-websocket = [] From a8ab8c7e0682b495a39013c9f4b146ba064814cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 28 Aug 2023 15:44:03 +0100 Subject: [PATCH 48/70] move parse_listening_addresses for better reviewing --- beacon_node/src/config.rs | 434 +++++++++++++++++++------------------- 1 file changed, 217 insertions(+), 217 deletions(-) diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 994ce336d04..ff2f4f4821b 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -860,6 +860,223 @@ pub fn get_config( Ok(client_config) } +/// Gets the listening_addresses for lighthouse based on the cli options. +pub fn parse_listening_addresses( + cli_args: &ArgMatches, + log: &Logger, +) -> Result { + let listen_addresses_str = cli_args + .values_of("listen-address") + .expect("--listen_addresses has a default value"); + + let use_zero_ports = cli_args.is_present("zero-ports"); + + // parse the possible ips + let mut maybe_ipv4 = None; + let mut maybe_ipv6 = None; + for addr_str in listen_addresses_str { + let addr = addr_str.parse::().map_err(|parse_error| { + format!("Failed to parse listen-address ({addr_str}) as an Ip address: {parse_error}") + })?; + + match addr { + IpAddr::V4(v4_addr) => match &maybe_ipv4 { + Some(first_ipv4_addr) => { + return Err(format!( + "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ + Got two IpV4 addresses {first_ipv4_addr} and {v4_addr}" + )); + } + None => maybe_ipv4 = Some(v4_addr), + }, + IpAddr::V6(v6_addr) => match &maybe_ipv6 { + Some(first_ipv6_addr) => { + return Err(format!( + "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ + Got two IpV6 addresses {first_ipv6_addr} and {v6_addr}" + )); + } + None => maybe_ipv6 = Some(v6_addr), + }, + } + } + + // parse the possible tcp ports + let port = cli_args + .value_of("port") + .expect("--port has a default value") + .parse::() + .map_err(|parse_error| format!("Failed to parse --port as an integer: {parse_error}"))?; + let port6 = cli_args + .value_of("port6") + .map(str::parse::) + .transpose() + .map_err(|parse_error| format!("Failed to parse --port6 as an integer: {parse_error}"))? + .unwrap_or(9090); + + // parse the possible discovery ports. + let maybe_disc_port = cli_args + .value_of("discovery-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --discovery-port as an integer: {parse_error}") + })?; + let maybe_disc6_port = cli_args + .value_of("discovery-port6") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --discovery-port6 as an integer: {parse_error}") + })?; + + // parse the possible quic port. + let maybe_quic_port = cli_args + .value_of("quic-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic-port as an integer: {parse_error}") + })?; + + // parse the possible quic port. + let maybe_quic6_port = cli_args + .value_of("quic-port6") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic6-port as an integer: {parse_error}") + })?; + + // Now put everything together + let listening_addresses = match (maybe_ipv4, maybe_ipv6) { + (None, None) => { + // This should never happen unless clap is broken + return Err("No listening addresses provided".into()); + } + (None, Some(ipv6)) => { + // A single ipv6 address was provided. Set the ports + + if cli_args.is_present("port6") { + warn!(log, "When listening only over IPv6, use the --port flag. The value of --port6 will be ignored.") + } + // use zero ports if required. If not, use the given port. + let tcp_port = use_zero_ports + .then(unused_port::unused_tcp6_port) + .transpose()? + .unwrap_or(port); + + if maybe_disc6_port.is_some() { + warn!(log, "When listening only over IPv6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") + } + + if maybe_quic6_port.is_some() { + warn!(log, "When listening only over IPv6, use the --quic-port flag. The value of --quic-port6 will be ignored.") + } + + // use zero ports if required. If not, use the specific udp port. If none given, use + // the tcp port. + let disc_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_disc_port) + .unwrap_or(port); + + let quic_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + + ListenAddress::V6(lighthouse_network::ListenAddr { + addr: ipv6, + quic_port, + disc_port, + tcp_port, + }) + } + (Some(ipv4), None) => { + // A single ipv4 address was provided. Set the ports + + // use zero ports if required. If not, use the given port. + let tcp_port = use_zero_ports + .then(unused_port::unused_tcp4_port) + .transpose()? + .unwrap_or(port); + // use zero ports if required. If not, use the specific discovery port. If none given, use + // the tcp port. + let disc_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_disc_port) + .unwrap_or(port); + // use zero ports if required. If not, use the specific quic port. If none given, use + // the tcp port + 1. + let quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + + ListenAddress::V4(lighthouse_network::ListenAddr { + addr: ipv4, + disc_port, + quic_port, + tcp_port, + }) + } + (Some(ipv4), Some(ipv6)) => { + let ipv4_tcp_port = use_zero_ports + .then(unused_port::unused_tcp4_port) + .transpose()? + .unwrap_or(port); + let ipv4_disc_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_disc_port) + .unwrap_or(ipv4_tcp_port); + let ipv4_quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + + // Defaults to 9090 when required + let ipv6_tcp_port = use_zero_ports + .then(unused_port::unused_tcp6_port) + .transpose()? + .unwrap_or(port6); + let ipv6_disc_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_disc6_port) + .unwrap_or(ipv6_tcp_port); + let ipv6_quic_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_quic6_port) + .unwrap_or(ipv6_tcp_port + 1); + + ListenAddress::DualStack( + lighthouse_network::ListenAddr { + addr: ipv4, + disc_port: ipv4_disc_port, + quic_port: ipv4_quic_port, + tcp_port: ipv4_tcp_port, + }, + lighthouse_network::ListenAddr { + addr: ipv6, + disc_port: ipv6_disc_port, + quic_port: ipv6_quic_port, + tcp_port: ipv6_tcp_port, + }, + ) + } + }; + + Ok(listening_addresses) +} + /// Sets the network config from the command line arguments. pub fn set_network_config( config: &mut NetworkConfig, @@ -1186,223 +1403,6 @@ pub fn set_network_config( Ok(()) } -/// Gets the listening_addresses for lighthouse based on the cli options. -pub fn parse_listening_addresses( - cli_args: &ArgMatches, - log: &Logger, -) -> Result { - let listen_addresses_str = cli_args - .values_of("listen-address") - .expect("--listen_addresses has a default value"); - - let use_zero_ports = cli_args.is_present("zero-ports"); - - // parse the possible ips - let mut maybe_ipv4 = None; - let mut maybe_ipv6 = None; - for addr_str in listen_addresses_str { - let addr = addr_str.parse::().map_err(|parse_error| { - format!("Failed to parse listen-address ({addr_str}) as an Ip address: {parse_error}") - })?; - - match addr { - IpAddr::V4(v4_addr) => match &maybe_ipv4 { - Some(first_ipv4_addr) => { - return Err(format!( - "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ - Got two IpV4 addresses {first_ipv4_addr} and {v4_addr}" - )); - } - None => maybe_ipv4 = Some(v4_addr), - }, - IpAddr::V6(v6_addr) => match &maybe_ipv6 { - Some(first_ipv6_addr) => { - return Err(format!( - "When setting the --listen-address option twice, use an IpV4 address and an Ipv6 address. \ - Got two IpV6 addresses {first_ipv6_addr} and {v6_addr}" - )); - } - None => maybe_ipv6 = Some(v6_addr), - }, - } - } - - // parse the possible tcp ports - let port = cli_args - .value_of("port") - .expect("--port has a default value") - .parse::() - .map_err(|parse_error| format!("Failed to parse --port as an integer: {parse_error}"))?; - let port6 = cli_args - .value_of("port6") - .map(str::parse::) - .transpose() - .map_err(|parse_error| format!("Failed to parse --port6 as an integer: {parse_error}"))? - .unwrap_or(9090); - - // parse the possible discovery ports. - let maybe_disc_port = cli_args - .value_of("discovery-port") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --discovery-port as an integer: {parse_error}") - })?; - let maybe_disc6_port = cli_args - .value_of("discovery-port6") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --discovery-port6 as an integer: {parse_error}") - })?; - - // parse the possible quic port. - let maybe_quic_port = cli_args - .value_of("quic-port") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --quic-port as an integer: {parse_error}") - })?; - - // parse the possible quic port. - let maybe_quic6_port = cli_args - .value_of("quic-port6") - .map(str::parse::) - .transpose() - .map_err(|parse_error| { - format!("Failed to parse --quic6-port as an integer: {parse_error}") - })?; - - // Now put everything together - let listening_addresses = match (maybe_ipv4, maybe_ipv6) { - (None, None) => { - // This should never happen unless clap is broken - return Err("No listening addresses provided".into()); - } - (None, Some(ipv6)) => { - // A single ipv6 address was provided. Set the ports - - if cli_args.is_present("port6") { - warn!(log, "When listening only over IPv6, use the --port flag. The value of --port6 will be ignored.") - } - // use zero ports if required. If not, use the given port. - let tcp_port = use_zero_ports - .then(unused_port::unused_tcp6_port) - .transpose()? - .unwrap_or(port); - - if maybe_disc6_port.is_some() { - warn!(log, "When listening only over IPv6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") - } - - if maybe_quic6_port.is_some() { - warn!(log, "When listening only over IPv6, use the --quic-port flag. The value of --quic-port6 will be ignored.") - } - - // use zero ports if required. If not, use the specific udp port. If none given, use - // the tcp port. - let disc_port = use_zero_ports - .then(unused_port::unused_udp6_port) - .transpose()? - .or(maybe_disc_port) - .unwrap_or(port); - - let quic_port = use_zero_ports - .then(unused_port::unused_udp6_port) - .transpose()? - .or(maybe_quic_port) - .unwrap_or(port + 1); - - ListenAddress::V6(lighthouse_network::ListenAddr { - addr: ipv6, - quic_port, - disc_port, - tcp_port, - }) - } - (Some(ipv4), None) => { - // A single ipv4 address was provided. Set the ports - - // use zero ports if required. If not, use the given port. - let tcp_port = use_zero_ports - .then(unused_port::unused_tcp4_port) - .transpose()? - .unwrap_or(port); - // use zero ports if required. If not, use the specific discovery port. If none given, use - // the tcp port. - let disc_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_disc_port) - .unwrap_or(port); - // use zero ports if required. If not, use the specific quic port. If none given, use - // the tcp port + 1. - let quic_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_quic_port) - .unwrap_or(port + 1); - - ListenAddress::V4(lighthouse_network::ListenAddr { - addr: ipv4, - disc_port, - quic_port, - tcp_port, - }) - } - (Some(ipv4), Some(ipv6)) => { - let ipv4_tcp_port = use_zero_ports - .then(unused_port::unused_tcp4_port) - .transpose()? - .unwrap_or(port); - let ipv4_disc_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_disc_port) - .unwrap_or(ipv4_tcp_port); - let ipv4_quic_port = use_zero_ports - .then(unused_port::unused_udp4_port) - .transpose()? - .or(maybe_quic_port) - .unwrap_or(port + 1); - - // Defaults to 9090 when required - let ipv6_tcp_port = use_zero_ports - .then(unused_port::unused_tcp6_port) - .transpose()? - .unwrap_or(port6); - let ipv6_disc_port = use_zero_ports - .then(unused_port::unused_udp6_port) - .transpose()? - .or(maybe_disc6_port) - .unwrap_or(ipv6_tcp_port); - let ipv6_quic_port = use_zero_ports - .then(unused_port::unused_udp6_port) - .transpose()? - .or(maybe_quic6_port) - .unwrap_or(ipv6_tcp_port + 1); - - ListenAddress::DualStack( - lighthouse_network::ListenAddr { - addr: ipv4, - disc_port: ipv4_disc_port, - quic_port: ipv4_quic_port, - tcp_port: ipv4_tcp_port, - }, - lighthouse_network::ListenAddr { - addr: ipv6, - disc_port: ipv6_disc_port, - quic_port: ipv6_quic_port, - tcp_port: ipv6_tcp_port, - }, - ) - } - }; - - Ok(listening_addresses) -} - /// Gets the datadir which should be used. pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf { // Read the `--datadir` flag. From 82f13bc940b4416ee469a9dccb4a39edea32fa2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 28 Aug 2023 15:49:18 +0100 Subject: [PATCH 49/70] improve doc wording --- beacon_node/lighthouse_network/src/config.rs | 12 ++++++------ beacon_node/lighthouse_network/src/discovery/enr.rs | 4 ++-- .../lighthouse_network/src/discovery/enr_ext.rs | 2 +- beacon_node/src/config.rs | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 3212dc144c0..d8c3f619dcb 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -58,22 +58,22 @@ pub struct Config { /// that no discovery address has been set in the CLI args. pub enr_address: (Option, Option), - /// The udp port to broadcast to peers in order to reach back for discovery. + /// The udp ipv4 port to broadcast to peers in order to reach back for discovery. pub enr_udp4_port: Option, - /// The UDP port to broadcast to peers in order to reach back for quic libp2p services. + /// The quic ipv4 port to broadcast to peers in order to reach back for libp2p services. pub enr_quic4_port: Option, - /// The tcp4 port to broadcast to peers in order to reach back for libp2p services. + /// The tcp ipv4 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp4_port: Option, - /// The udp6 port to broadcast to peers in order to reach back for discovery. + /// The udp ipv6 port to broadcast to peers in order to reach back for discovery. pub enr_udp6_port: Option, - /// The tcp6 port to broadcast to peers in order to reach back for libp2p services. + /// The tcp ipv6 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp6_port: Option, - /// The UDP port to broadcast to peers in order to reach back for quic libp2p services. + /// The quic ipv6 port to broadcast to peers in order to reach back for libp2p services. pub enr_quic6_port: Option, /// Target number of connected peers. diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index 9c0b3fe99b2..cbd17a3c350 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -169,7 +169,7 @@ pub fn create_enr_builder_from_config( // Add QUIC fields to the ENR. // If `enable_libp2p` is disabled, then we should not support QUIC in the ENR either. if !config.disable_quic_support { - // If we are listening on ipv4, add the quic ipv4 port + // If we are listening on ipv4, add the quic ipv4 port. if let Some(quic4_port) = config .enr_quic4_port .or_else(|| config.listen_addrs().v4().map(|v4_addr| v4_addr.quic_port)) @@ -177,7 +177,7 @@ pub fn create_enr_builder_from_config( builder.add_value(QUIC_ENR_KEY, &quic4_port); } - // If we are listening on ipv6, add the quic ipv6 port + // If we are listening on ipv6, add the quic ipv6 port. if let Some(quic6_port) = config .enr_quic6_port .or_else(|| config.listen_addrs().v6().map(|v6_addr| v6_addr.quic_port)) diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 10ccfb2678d..cb321a78d02 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -213,7 +213,7 @@ impl EnrExt for Enr { /// The vector remains empty if these fields are not defined. fn multiaddr_quic(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); - // Check for quic first as it is less likely + // Check for quic first as it is less likely. if let Some(quic_port) = self.quic4() { if let Some(ip) = self.ip4() { let mut multiaddr: Multiaddr = ip.into(); diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index ff2f4f4821b..b6d0b75d9ff 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1226,7 +1226,7 @@ pub fn set_network_config( if cli_args.is_present("enr-match") { // Match the IP and UDP port in the ENR. - // Set the ENR address to localhost if the address is unspecified + // Set the ENR address to localhost if the address is unspecified. if let Some(ipv4_addr) = config.listen_addrs().v4().cloned() { let ipv4_enr_addr = if ipv4_addr.addr == Ipv4Addr::UNSPECIFIED { Ipv4Addr::LOCALHOST From 36a044ac34d957758de5ea8329e25f338985ce95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 28 Aug 2023 16:47:24 +0100 Subject: [PATCH 50/70] fix metrics protocol identification --- .../lighthouse_network/src/peer_manager/network_behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index b3ab4fc06bb..49f271698b0 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -275,7 +275,7 @@ impl PeerManager { multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) ) }) { - Some(multiaddr::Protocol::Quic) => { + Some(multiaddr::Protocol::QuicV1) => { metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); } Some(multiaddr::Protocol::Tcp(_)) => { From 9f74dd9938e1da77df51459cfcc50cd12ce1967d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 30 Aug 2023 18:19:22 +0100 Subject: [PATCH 51/70] update socket address for quic connections, on update_connection_state() --- beacon_node/lighthouse_network/src/peer_manager/peerdb.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index 52f0bbd9dfc..a8ad1bfa537 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -771,12 +771,14 @@ impl PeerDB { Protocol::Ip4(ip) => (Some(ip.into()), found_port), Protocol::Ip6(ip) => (Some(ip.into()), found_port), Protocol::Tcp(port) => (found_ip, Some(port)), + // Quic + Protocol::Udp(port) => (found_ip, Some(port)), _ => (found_ip, found_port), }, ) { (Some(ip), Some(port)) => Some(SocketAddr::new(ip, port)), (Some(_ip), None) => { - crit!(self.log, "Connected peer has an IP but no TCP port"; "peer_id" => %peer_id); + crit!(self.log, "Connected peer has an IP but no Transport port"; "peer_id" => %peer_id); None } _ => None, From c14b8456b22f350df3d5cede452804c3e9f0064a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 31 Aug 2023 16:06:07 +0100 Subject: [PATCH 52/70] address review --- .../lighthouse_network/src/discovery/enr.rs | 3 +- .../src/discovery/enr_ext.rs | 1 - .../src/peer_manager/network_behaviour.rs | 72 ++++++++++--------- .../lighthouse_network/src/service/mod.rs | 8 +-- .../lighthouse_network/tests/common.rs | 14 +--- beacon_node/src/cli.rs | 2 - 6 files changed, 46 insertions(+), 54 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index cbd17a3c350..3f46285a807 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -167,7 +167,8 @@ pub fn create_enr_builder_from_config( if enable_libp2p { // Add QUIC fields to the ENR. - // If `enable_libp2p` is disabled, then we should not support QUIC in the ENR either. + // Since QUIC is used as an alternative transport for the libp2p protocols, + // the related fields should only be added when both QUIC and libp2p are enabled if !config.disable_quic_support { // If we are listening on ipv4, add the quic ipv4 port. if let Some(quic4_port) = config diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index cb321a78d02..e68265fdff6 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -213,7 +213,6 @@ impl EnrExt for Enr { /// The vector remains empty if these fields are not defined. fn multiaddr_quic(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); - // Check for quic first as it is less likely. if let Some(quic_port) = self.quic4() { if let Some(ip) = self.ip4() { let mut multiaddr: Multiaddr = ip.into(); diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 49f271698b0..cf9937d46df 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -269,24 +269,26 @@ impl PeerManager { }; // increment prometheus metrics - match remote_addr.iter().find(|proto| { - matches!( - proto, - multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) - ) - }) { - Some(multiaddr::Protocol::QuicV1) => { - metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); - } - Some(multiaddr::Protocol::Tcp(_)) => { - metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); - } - Some(_) => unreachable!(), - None => error!(self.log, "Connected via unknown transport"; "addr" => %remote_addr), - }; + if self.metrics_enabled { + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::QuicV1) => { + metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); + } + Some(_) => unreachable!(), + None => error!(self.log, "Connected via unknown transport"; "addr" => %remote_addr), + }; - self.update_connected_peer_metrics(); - metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + self.update_connected_peer_metrics(); + metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + } } fn on_connection_closed( @@ -326,23 +328,25 @@ impl PeerManager { }; // Update the prometheus metrics - match remote_addr.iter().find(|proto| { - matches!( - proto, - multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) - ) - }) { - Some(multiaddr::Protocol::Quic) => { - metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); - } - Some(multiaddr::Protocol::Tcp(_)) => { - metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); - } - // If it's an unknown protocol we already logged when connection was established. - _ => {} - }; - self.update_connected_peer_metrics(); - metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); + if self.metrics_enabled { + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::Quic) => { + metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); + } + // If it's an unknown protocol we already logged when connection was established. + _ => {} + }; + self.update_connected_peer_metrics(); + metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); + } } /// A dial attempt has failed. diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 4c4c371653c..bcc14b19dd7 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -476,7 +476,6 @@ impl Network { } } - // for multiaddr in &bootnode_enr.multiaddr() { // ignore udp multiaddr if it exists let components = multiaddr.iter().collect::>(); @@ -1076,10 +1075,11 @@ impl Network { .collect(); // Remove the ENR from the cache to prevent continual re-dialing on disconnects - peers_to_dial.iter().for_each(|enr| { - self.peer_manager_mut().dial_peer(enr.clone()); + for enr in peers_to_dial { + debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %enr.peer_id()); self.discovery_mut().remove_cached_enr(&enr.peer_id()); - }); + self.peer_manager_mut().dial_peer(enr); + } } /* Sub-behaviour event handling functions */ diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index e1d745fa000..e72883a1699 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -147,18 +147,8 @@ pub async fn build_node_pair( let mut receiver = build_libp2p_instance(rt, vec![], receiver_log, fork_name, spec).await; let receiver_multiaddr = match protocol { - Protocol::Tcp => receiver - .local_enr() - .multiaddr_tcp() - .first() - .unwrap() - .clone(), - Protocol::Quic => receiver - .local_enr() - .multiaddr_quic() - .first() - .unwrap() - .clone(), + Protocol::Tcp => receiver.local_enr().multiaddr_tcp().pop().unwrap(), + Protocol::Quic => receiver.local_enr().multiaddr_quic().pop().unwrap(), }; // let the two nodes set up listeners diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 898dbd43235..ffc29365cb9 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -263,14 +263,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("disable-discovery") .long("disable-discovery") .help("Disables the discv5 discovery protocol. The node will not search for new peers or participate in the discovery protocol.") - .takes_value(false) .hidden(true) ) .arg( Arg::with_name("disable-quic") .long("disable-quic") .help("Disables the quic transport. The node will rely solely on the TCP transport for libp2p connections.") - .takes_value(false), ) .arg( Arg::with_name("disable-peer-scoring") From b90e969db5de3c240e8f9381f3a5176a58c6fa7e Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 4 Sep 2023 18:39:56 +1000 Subject: [PATCH 53/70] Remove references and assumptions to TCP based libp2p port --- beacon_node/client/src/lib.rs | 14 ------- beacon_node/http_api/src/lib.rs | 17 ++------- beacon_node/http_api/src/test_utils.rs | 2 - .../lighthouse_network/src/discovery/mod.rs | 24 +----------- .../src/peer_manager/peerdb.rs | 36 +++--------------- .../src/peer_manager/peerdb/peer_info.rs | 38 +++++++++++-------- .../lighthouse_network/src/service/mod.rs | 2 - .../lighthouse_network/src/types/globals.rs | 20 ---------- .../src/network_beacon_processor/tests.rs | 2 - 9 files changed, 34 insertions(+), 121 deletions(-) diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 584a0d736de..399aa06511e 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -46,20 +46,6 @@ impl Client { self.http_metrics_listen_addr } - /// Returns the ipv4 port of the client's libp2p stack, if it was started. - pub fn libp2p_listen_ipv4_port(&self) -> Option { - self.network_globals - .as_ref() - .and_then(|n| n.listen_port_tcp4()) - } - - /// Returns the ipv6 port of the client's libp2p stack, if it was started. - pub fn libp2p_listen_ipv6_port(&self) -> Option { - self.network_globals - .as_ref() - .and_then(|n| n.listen_port_tcp6()) - } - /// Returns the list of libp2p addresses the client is listening to. pub fn libp2p_listen_addresses(&self) -> Option> { self.network_globals.as_ref().map(|n| n.listen_multiaddrs()) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 19d62c060d9..08b18e86d00 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -2825,12 +2825,8 @@ pub fn serve( })?; if let Some(peer_info) = network_globals.peers.read().peer_info(&peer_id) { - let address = if let Some(socket_addr) = peer_info.seen_addresses().next() { - let mut addr = lighthouse_network::Multiaddr::from(socket_addr.ip()); - addr.push(lighthouse_network::multiaddr::Protocol::Tcp( - socket_addr.port(), - )); - addr.to_string() + let address = if let Some(multiaddr) = peer_info.seen_multiaddrs().next() { + multiaddr.to_string() } else if let Some(addr) = peer_info.listening_addresses().first() { addr.to_string() } else { @@ -2878,13 +2874,8 @@ pub fn serve( .peers() .for_each(|(peer_id, peer_info)| { let address = - if let Some(socket_addr) = peer_info.seen_addresses().next() { - let mut addr = - lighthouse_network::Multiaddr::from(socket_addr.ip()); - addr.push(lighthouse_network::multiaddr::Protocol::Tcp( - socket_addr.port(), - )); - addr.to_string() + if let Some(multiaddr) = peer_info.seen_multiaddrs().next() { + multiaddr.to_string() } else if let Some(addr) = peer_info.listening_addresses().first() { addr.to_string() } else { diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 33834d58ca0..fb3497ff8ba 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -152,8 +152,6 @@ pub async fn create_api_server_on_port( let enr = EnrBuilder::new("v4").build(&enr_key).unwrap(); let network_globals = Arc::new(NetworkGlobals::new( enr.clone(), - Some(TCP_PORT), - None, meta_data, vec![], false, diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 270a2a2dd27..05ac7101b39 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -21,7 +21,6 @@ pub use libp2p::identity::{Keypair, PublicKey}; use enr::{ATTESTATION_BITFIELD_ENR_KEY, ETH2_ENR_KEY, SYNC_COMMITTEE_BITFIELD_ENR_KEY}; use futures::prelude::*; use futures::stream::FuturesUnordered; -use libp2p::multiaddr::Protocol; use libp2p::swarm::behaviour::{DialFailure, FromSwarm}; use libp2p::swarm::THandlerInEvent; pub use libp2p::{ @@ -1001,25 +1000,8 @@ impl NetworkBehaviour for Discovery { // update network globals *self.network_globals.local_enr.write() = enr; // A new UDP socket has been detected. - // Build a multiaddr to report to libp2p - let addr = match socket_addr.ip() { - IpAddr::V4(v4_addr) => { - self.network_globals.listen_port_tcp4().map(|tcp4_port| { - Multiaddr::from(v4_addr).with(Protocol::Tcp(tcp4_port)) - }) - } - IpAddr::V6(v6_addr) => { - self.network_globals.listen_port_tcp6().map(|tcp6_port| { - Multiaddr::from(v6_addr).with(Protocol::Tcp(tcp6_port)) - }) - } - }; - - if let Some(address) = addr { - // NOTE: This doesn't actually track the external TCP port. More sophisticated NAT handling - // should handle this. - return Poll::Ready(ToSwarm::NewExternalAddrCandidate(address)); - } + // NOTE: We assume libp2p itself can keep track of IP changes and we do + // not inform it about IP changes found via discovery. } Discv5Event::EnrAdded { .. } | Discv5Event::TalkRequest(_) @@ -1106,8 +1088,6 @@ mod tests { let log = build_log(slog::Level::Debug, false); let globals = NetworkGlobals::new( enr, - Some(9000), - None, MetaData::V2(MetaDataV2 { seq_number: 0, attnets: Default::default(), diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index a8ad1bfa537..c781e83eb9c 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -1,16 +1,11 @@ -use crate::{ - metrics, - multiaddr::{Multiaddr, Protocol}, - types::Subnet, - Enr, Gossipsub, PeerId, -}; +use crate::{metrics, multiaddr::Multiaddr, types::Subnet, Enr, Gossipsub, PeerId}; use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerInfo}; use rand::seq::SliceRandom; use score::{PeerAction, ReportSource, Score, ScoreState}; use slog::{crit, debug, error, trace, warn}; use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::time::Instant; use sync_status::SyncStatus; use types::EthSpec; @@ -166,7 +161,7 @@ impl PeerDB { /// Checks if the peer's known addresses are currently banned. fn ip_is_banned(&self, peer: &PeerInfo) -> Option { peer.seen_ip_addresses() - .find(|ip| self.banned_peers_count.ip_is_banned(ip)) + .find(|ip| self.banned_peers_count.ip_is_banned(&ip)) } /// Returns true if the IP is banned. @@ -764,30 +759,10 @@ impl PeerDB { | PeerConnectionStatus::Dialing { .. } => {} } - // Add the seen ip address and port to the peer's info - let socket_addr = match seen_address.iter().fold( - (None, None), - |(found_ip, found_port), protocol| match protocol { - Protocol::Ip4(ip) => (Some(ip.into()), found_port), - Protocol::Ip6(ip) => (Some(ip.into()), found_port), - Protocol::Tcp(port) => (found_ip, Some(port)), - // Quic - Protocol::Udp(port) => (found_ip, Some(port)), - _ => (found_ip, found_port), - }, - ) { - (Some(ip), Some(port)) => Some(SocketAddr::new(ip, port)), - (Some(_ip), None) => { - crit!(self.log, "Connected peer has an IP but no Transport port"; "peer_id" => %peer_id); - None - } - _ => None, - }; - // Update the connection state match direction { - ConnectionDirection::Incoming => info.connect_ingoing(socket_addr), - ConnectionDirection::Outgoing => info.connect_outgoing(socket_addr), + ConnectionDirection::Incoming => info.connect_ingoing(Some(seen_address)), + ConnectionDirection::Outgoing => info.connect_outgoing(Some(seen_address)), } } @@ -1276,6 +1251,7 @@ impl BannedPeersCount { #[cfg(test)] mod tests { use super::*; + use libp2p::core::multiaddr::Protocol; use libp2p::core::Multiaddr; use slog::{o, Drain}; use std::net::{Ipv4Addr, Ipv6Addr}; diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs index 555266d0e2e..44c54511ddc 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs @@ -2,15 +2,15 @@ use super::client::Client; use super::score::{PeerAction, Score, ScoreState}; use super::sync_status::SyncStatus; use crate::discovery::Eth2Enr; -use crate::Multiaddr; use crate::{rpc::MetaData, types::Subnet}; use discv5::Enr; +use libp2p::core::multiaddr::{Multiaddr, Protocol}; use serde::{ ser::{SerializeStruct, Serializer}, Serialize, }; use std::collections::HashSet; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::time::Instant; use strum::AsRefStr; use types::EthSpec; @@ -29,9 +29,9 @@ pub struct PeerInfo { /// The known listening addresses of this peer. This is given by identify and can be arbitrary /// (including local IPs). listening_addresses: Vec, - /// This is addresses we have physically seen and this is what we use for banning/un-banning + /// These are the multiaddrs we have physically seen and is what we use for banning/un-banning /// peers. - seen_addresses: HashSet, + seen_multiaddrs: HashSet, /// The current syncing state of the peer. The state may be determined after it's initial /// connection. sync_status: SyncStatus, @@ -60,7 +60,7 @@ impl Default for PeerInfo { client: Client::default(), connection_status: Default::default(), listening_addresses: Vec::new(), - seen_addresses: HashSet::new(), + seen_multiaddrs: HashSet::new(), subnets: HashSet::new(), sync_status: SyncStatus::Unknown, meta_data: None, @@ -227,15 +227,21 @@ impl PeerInfo { } /// Returns the seen addresses of the peer. - pub fn seen_addresses(&self) -> impl Iterator + '_ { - self.seen_addresses.iter() + pub fn seen_multiaddrs(&self) -> impl Iterator + '_ { + self.seen_multiaddrs.iter() } /// Returns a list of seen IP addresses for the peer. pub fn seen_ip_addresses(&self) -> impl Iterator + '_ { - self.seen_addresses - .iter() - .map(|socket_addr| socket_addr.ip()) + self.seen_multiaddrs.iter().filter_map(|multiaddr| { + multiaddr.iter().find_map(|protocol| { + match protocol { + Protocol::Ip4(ip) => Some(ip.into()), + Protocol::Ip6(ip) => Some(ip.into()), + _ => None, // Only care for IP addresses + } + }) + }) } /// Returns the connection status of the peer. @@ -415,7 +421,7 @@ impl PeerInfo { /// Modifies the status to Connected and increases the number of ingoing /// connections by one - pub(super) fn connect_ingoing(&mut self, seen_address: Option) { + pub(super) fn connect_ingoing(&mut self, seen_multiaddr: Option) { match &mut self.connection_status { Connected { n_in, .. } => *n_in += 1, Disconnected { .. } @@ -428,14 +434,14 @@ impl PeerInfo { } } - if let Some(socket_addr) = seen_address { - self.seen_addresses.insert(socket_addr); + if let Some(multiaddr) = seen_multiaddr { + self.seen_multiaddrs.insert(multiaddr); } } /// Modifies the status to Connected and increases the number of outgoing /// connections by one - pub(super) fn connect_outgoing(&mut self, seen_address: Option) { + pub(super) fn connect_outgoing(&mut self, seen_multiaddr: Option) { match &mut self.connection_status { Connected { n_out, .. } => *n_out += 1, Disconnected { .. } @@ -447,8 +453,8 @@ impl PeerInfo { self.connection_direction = Some(ConnectionDirection::Outgoing); } } - if let Some(ip_addr) = seen_address { - self.seen_addresses.insert(ip_addr); + if let Some(multiaddr) = seen_multiaddr { + self.seen_multiaddrs.insert(multiaddr); } } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 4c4c371653c..44abe72b6f2 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -156,8 +156,6 @@ impl Network { let meta_data = utils::load_or_build_metadata(&config.network_dir, &log); let globals = NetworkGlobals::new( enr, - config.listen_addrs().v4().map(|v4_addr| v4_addr.tcp_port), - config.listen_addrs().v6().map(|v6_addr| v6_addr.tcp_port), meta_data, config .trusted_peers diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index 97eaaa0051b..b2b605e8aec 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -16,10 +16,6 @@ pub struct NetworkGlobals { pub peer_id: RwLock, /// Listening multiaddrs. pub listen_multiaddrs: RwLock>, - /// The TCP port that the libp2p service is listening on over Ipv4. - listen_port_tcp4: Option, - /// The TCP port that the libp2p service is listening on over Ipv6. - listen_port_tcp6: Option, /// The collection of known peers. pub peers: RwLock>, // The local meta data of our node. @@ -35,8 +31,6 @@ pub struct NetworkGlobals { impl NetworkGlobals { pub fn new( enr: Enr, - listen_port_tcp4: Option, - listen_port_tcp6: Option, local_metadata: MetaData, trusted_peers: Vec, disable_peer_scoring: bool, @@ -46,8 +40,6 @@ impl NetworkGlobals { local_enr: RwLock::new(enr.clone()), peer_id: RwLock::new(enr.peer_id()), listen_multiaddrs: RwLock::new(Vec::new()), - listen_port_tcp4, - listen_port_tcp6, local_metadata: RwLock::new(local_metadata), peers: RwLock::new(PeerDB::new(trusted_peers, disable_peer_scoring, log)), gossipsub_subscriptions: RwLock::new(HashSet::new()), @@ -72,16 +64,6 @@ impl NetworkGlobals { self.listen_multiaddrs.read().clone() } - /// Returns the libp2p TCP port that this node has been configured to listen on. - pub fn listen_port_tcp4(&self) -> Option { - self.listen_port_tcp4 - } - - /// Returns the UDP discovery port that this node has been configured to listen on. - pub fn listen_port_tcp6(&self) -> Option { - self.listen_port_tcp6 - } - /// Returns the number of libp2p connected peers. pub fn connected_peers(&self) -> usize { self.peers.read().connected_peer_ids().count() @@ -139,8 +121,6 @@ impl NetworkGlobals { let enr = discv5::enr::EnrBuilder::new("v4").build(&enr_key).unwrap(); NetworkGlobals::new( enr, - Some(9000), - None, MetaData::V2(MetaDataV2 { seq_number: 0, attnets: Default::default(), diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index a678edbf1ff..a92fe642805 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -197,8 +197,6 @@ impl TestRig { let enr = EnrBuilder::new("v4").build(&enr_key).unwrap(); let network_globals = Arc::new(NetworkGlobals::new( enr, - Some(TCP_PORT), - None, meta_data, vec![], false, From aaced27e2ac4ba51a64c333d96f45bdc5c6364a1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 4 Sep 2023 18:59:49 +1000 Subject: [PATCH 54/70] fmt --- beacon_node/network/src/network_beacon_processor/tests.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index a92fe642805..08d7f2debf1 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -195,13 +195,7 @@ impl TestRig { }); let enr_key = CombinedKey::generate_secp256k1(); let enr = EnrBuilder::new("v4").build(&enr_key).unwrap(); - let network_globals = Arc::new(NetworkGlobals::new( - enr, - meta_data, - vec![], - false, - &log, - )); + let network_globals = Arc::new(NetworkGlobals::new(enr, meta_data, vec![], false, &log)); let executor = harness.runtime.task_executor.clone(); From 5d0dee5379e1b9ababa1e4f6485a76e0694918a7 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 5 Sep 2023 15:11:05 +1000 Subject: [PATCH 55/70] Clippy lints --- beacon_node/lighthouse_network/src/peer_manager/peerdb.rs | 2 +- beacon_node/network/src/network_beacon_processor/tests.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index c781e83eb9c..4a1efe8f2e9 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -161,7 +161,7 @@ impl PeerDB { /// Checks if the peer's known addresses are currently banned. fn ip_is_banned(&self, peer: &PeerInfo) -> Option { peer.seen_ip_addresses() - .find(|ip| self.banned_peers_count.ip_is_banned(&ip)) + .find(|ip| self.banned_peers_count.ip_is_banned(ip)) } /// Returns true if the IP is banned. diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 08d7f2debf1..ac5722a565b 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -37,7 +37,6 @@ const VALIDATOR_COUNT: usize = SLOTS_PER_EPOCH as usize; const SMALL_CHAIN: u64 = 2; const LONG_CHAIN: u64 = SLOTS_PER_EPOCH * 2; -const TCP_PORT: u16 = 42; const SEQ_NUMBER: u64 = 0; /// The default time to wait for `BeaconProcessor` events. From 655a7575f2a2841d16423066288a8718da654414 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 5 Sep 2023 17:03:20 +1000 Subject: [PATCH 56/70] Improve inbound connection log --- beacon_node/lighthouse_network/src/service/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 166457589c2..722d81b70db 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1501,7 +1501,7 @@ impl Network { format!("Dialing local peer id {endpoint:?}") } libp2p::swarm::ListenError::Denied { cause } => { - format!("Connection was denied with cause {cause}") + format!("Connection was denied with cause: {cause:?}") } libp2p::swarm::ListenError::Transport(t) => match t { libp2p::TransportError::MultiaddrNotSupported(m) => { From ed3dfb6ac452889ec0946b253214a015dc524abb Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 6 Sep 2023 17:08:17 +1000 Subject: [PATCH 57/70] Update QUIC ports on UPnP mapping changes --- .../src/discovery/enr_ext.rs | 2 - .../lighthouse_network/src/discovery/mod.rs | 17 +++ .../lighthouse_network/src/listen_addr.rs | 2 +- .../lighthouse_network/src/service/mod.rs | 2 +- .../lighthouse_network/tests/rpc_tests.rs | 103 ++++-------------- beacon_node/network/src/nat.rs | 51 ++++++--- beacon_node/network/src/service.rs | 38 +++---- 7 files changed, 95 insertions(+), 120 deletions(-) diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index e68265fdff6..2efaa76ac31 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -210,7 +210,6 @@ impl EnrExt for Enr { } /// Returns a list of multiaddrs if the ENR has an `ip` and a `quic` key **or** an `ip6` and a `quic6`. - /// The vector remains empty if these fields are not defined. fn multiaddr_quic(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); if let Some(quic_port) = self.quic4() { @@ -234,7 +233,6 @@ impl EnrExt for Enr { } /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. - /// The vector remains empty if these fields are not defined. fn multiaddr_tcp(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); if let Some(ip) = self.ip4() { diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 05ac7101b39..4d8807336bf 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -402,6 +402,23 @@ impl Discovery { Ok(()) } + // TODO: Group these functions here once the ENR is shared across discv5 and lighthouse and + // Lighthouse can modify the ENR directly. + // This currently doesn't support ipv6. All of these functions should be removed and + // addressed properly in the following issue. + // https://github.com/sigp/lighthouse/issues/4706 + pub fn update_enr_quic_port(&mut self, port: u16) -> Result<(), String> { + self.discv5 + .enr_insert("quic", &port) + .map_err(|e| format!("{:?}", e))?; + + // replace the global version + *self.network_globals.local_enr.write() = self.discv5.local_enr(); + // persist modified enr to disk + enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); + Ok(()) + } + /// Updates the local ENR UDP socket. /// /// This is with caution. Discovery should automatically maintain this. This should only be diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 74c4ba110b9..4b43ca5af14 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -56,7 +56,7 @@ impl ListenAddress { } /// Returns the addresses the Swarm will listen on, given the setup. - pub fn listen_addresses(&self) -> impl Iterator { + pub fn libp2p_addresses(&self) -> impl Iterator { let v4_tcp_multiaddr = self .v4() .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 722d81b70db..e4e11f29c55 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -412,7 +412,7 @@ impl Network { info!(self.log, "Libp2p Starting"; "peer_id" => %enr.peer_id(), "bandwidth_config" => format!("{}-{}", config.network_load, NetworkLoad::from(config.network_load).name)); debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery, "quic_enabled" => !config.disable_quic_support); - for listen_multiaddr in config.listen_addrs().listen_addresses() { + for listen_multiaddr in config.listen_addrs().libp2p_addresses() { // If QUIC is disabled, ignore listening on QUIC ports if config.disable_quic_support && listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index b2fc70a1f11..85f52140e37 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -877,14 +877,9 @@ fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { }) } -// Tests a Goodbye RPC message -#[test] -#[allow(clippy::single_match)] -fn tcp_test_goodbye_rpc() { - // set up the logging. The level and enabled logging or not - let log_level = Level::Debug; - let enable_logging = true; - +/// Established a pair of nodes and disconnects the pair based on the selected protocol via an RPC +/// Goodbye message. +fn goodbye_test(log_level: Level, enable_logging: bool, protocol: Protocol) { let log = common::build_log(log_level, enable_logging); let rt = Arc::new(Runtime::new().unwrap()); @@ -893,14 +888,9 @@ fn tcp_test_goodbye_rpc() { // get sender/receiver rt.block_on(async { - let (mut sender, mut receiver) = common::build_node_pair( - Arc::downgrade(&rt), - &log, - ForkName::Base, - &spec, - Protocol::Tcp, - ) - .await; + let (mut sender, mut receiver) = + common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec, protocol) + .await; // build the sender future let sender_future = async { @@ -926,12 +916,9 @@ fn tcp_test_goodbye_rpc() { // build the receiver future let receiver_future = async { loop { - match receiver.next_event().await { - NetworkEvent::PeerDisconnected(_) => { - // Should receive sent RPC request - return; - } - _ => {} // Ignore other events + if let NetworkEvent::PeerDisconnected(_) = receiver.next_event().await { + // Should receive sent RPC request + return; } } }; @@ -950,69 +937,19 @@ fn tcp_test_goodbye_rpc() { // Tests a Goodbye RPC message #[test] #[allow(clippy::single_match)] -fn quic_test_goodbye_rpc() { +fn tcp_test_goodbye_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = true; + goodbye_test(log_level, enable_logging, Protocol::Tcp); +} - let log = common::build_log(log_level, enable_logging); - - let rt = Arc::new(Runtime::new().unwrap()); - - let spec = E::default_spec(); - - // get sender/receiver - rt.block_on(async { - let (mut sender, mut receiver) = common::build_node_pair( - Arc::downgrade(&rt), - &log, - ForkName::Base, - &spec, - Protocol::Quic, - ) - .await; - - // build the sender future - let sender_future = async { - loop { - match sender.next_event().await { - NetworkEvent::PeerConnectedOutgoing(peer_id) => { - // Send a goodbye and disconnect - debug!(log, "Sending RPC"); - sender.goodbye_peer( - &peer_id, - GoodbyeReason::IrrelevantNetwork, - ReportSource::SyncService, - ); - } - NetworkEvent::PeerDisconnected(_) => { - return; - } - _ => {} // Ignore other RPC messages - } - } - }; - - // build the receiver future - let receiver_future = async { - loop { - match receiver.next_event().await { - NetworkEvent::PeerDisconnected(_) => { - // Should receive sent RPC request - return; - } - _ => {} // Ignore other events - } - } - }; - - let total_future = futures::future::join(sender_future, receiver_future); - - tokio::select! { - _ = total_future => {} - _ = sleep(Duration::from_secs(30)) => { - panic!("Future timed out"); - } - } - }) +// Tests a Goodbye RPC message +#[test] +#[allow(clippy::single_match)] +fn quic_test_goodbye_rpc() { + // set up the logging. The level and enabled logging or not + let log_level = Level::Debug; + let enable_logging = true; + goodbye_test(log_level, enable_logging, Protocol::Quic); } diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index 4476ccd4425..d011ac42e84 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -24,6 +24,29 @@ pub struct UPnPConfig { disable_quic_support: bool, } +/// Contains mappings that managed to be established. +#[derive(Default, Debug)] +pub struct EstablishedUPnPMappings { + /// A TCP port mapping for libp2p. + pub tcp_port: Option, + /// A UDP port for the QUIC libp2p transport. + pub udp_quic_port: Option, + /// A UDP port for discv5. + pub udp_disc_port: Option, +} + +impl EstablishedUPnPMappings { + /// Returns true if at least one value is set. + pub fn is_some(&self) -> bool { + self.tcp_port.is_some() || self.udp_quic_port.is_some() || self.udp_disc_port.is_some() + } + + // Iterator over the UDP ports + pub fn udp_ports(&self) -> impl Iterator { + self.udp_quic_port.iter().chain(self.udp_disc_port.iter()) + } +} + impl UPnPConfig { pub fn from_config(config: &NetworkConfig) -> Option { config.listen_addrs().v4().map(|v4_addr| UPnPConfig { @@ -74,6 +97,8 @@ pub fn construct_upnp_mappings( debug!(log, "UPnP Local IP Discovered"; "ip" => ?local_ip); + let mut mappings = EstablishedUPnPMappings::default(); + match local_ip { IpAddr::V4(address) => { let libp2p_socket = SocketAddrV4::new(address, config.tcp_port); @@ -82,16 +107,16 @@ pub fn construct_upnp_mappings( // one. // I've found this to be more reliable. If multiple users are behind a single // router, they should ideally try to set different port numbers. - let tcp_socket = add_port_mapping( + mappings.tcp_port = add_port_mapping( &gateway, igd::PortMappingProtocol::TCP, libp2p_socket, "tcp", &log, - ).and_then(|_| { + ).map(|_| { let external_socket = external_ip.as_ref().map(|ip| SocketAddr::new((*ip).into(), config.tcp_port)).map_err(|_| ()); info!(log, "UPnP TCP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.tcp_port)); - external_socket + config.tcp_port }).ok(); let set_udp_mapping = |udp_port| { @@ -107,21 +132,21 @@ pub fn construct_upnp_mappings( }) }; - let mut udp_sockets = Vec::new(); - // Set the discovery UDP port mapping if !config.disable_discovery && set_udp_mapping(config.disc_port).is_ok() { - udp_sockets.push(config.disc_port); + mappings.udp_disc_port = Some(config.disc_port); } // Set the quic UDP port mapping if !config.disable_quic_support && set_udp_mapping(config.quic_port).is_ok() { - udp_sockets.push(config.quic_port) + mappings.udp_quic_port = Some(config.quic_port); } // report any updates to the network service. - network_send.send(NetworkMessage::UPnPMappingEstablished{ tcp_socket, udp_sockets }) - .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e)); + if mappings.is_some() { + network_send.send(NetworkMessage::UPnPMappingEstablished{ mappings }) + .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e)); + } } _ => debug!(log, "UPnP no routes constructed. IPv6 not supported"), } @@ -174,12 +199,12 @@ fn add_port_mapping( } /// Removes the specified TCP and UDP port mappings. -pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Logger) { - if tcp_port.is_some() || !udp_ports.is_empty() { +pub fn remove_mappings(mappings: &EstablishedUPnPMappings, log: &slog::Logger) { + if mappings.is_some() { debug!(log, "Removing UPnP port mappings"); match igd::search_gateway(Default::default()) { Ok(gateway) => { - if let Some(tcp_port) = tcp_port { + if let Some(tcp_port) = mappings.tcp_port { match gateway.remove_port(igd::PortMappingProtocol::TCP, tcp_port) { Ok(()) => debug!(log, "UPnP Removed TCP port mapping"; "port" => tcp_port), Err(e) => { @@ -187,7 +212,7 @@ pub fn remove_mappings(tcp_port: Option, udp_ports: &[u16], log: &slog::Log } } } - for udp_port in udp_ports { + for udp_port in mappings.udp_ports() { match gateway.remove_port(igd::PortMappingProtocol::UDP, *udp_port) { Ok(()) => debug!(log, "UPnP Removed UDP port mapping"; "port" => udp_port), Err(e) => { diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 96ab3cd3d16..174a0ec14c6 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,4 +1,5 @@ use super::sync::manager::RequestId as SyncId; +use crate::nat::EstablishedUPnPMappings; use crate::network_beacon_processor::InvalidBlockStorage; use crate::persisted_dht::{clear_dht, load_dht, persist_dht}; use crate::router::{Router, RouterMessage}; @@ -26,7 +27,7 @@ use lighthouse_network::{ MessageId, NetworkEvent, NetworkGlobals, PeerId, }; use slog::{crit, debug, error, info, o, trace, warn}; -use std::{collections::HashSet, net::SocketAddr, pin::Pin, sync::Arc, time::Duration}; +use std::{collections::HashSet, pin::Pin, sync::Arc, time::Duration}; use store::HotColdDB; use strum::IntoStaticStr; use task_executor::ShutdownReason; @@ -93,12 +94,10 @@ pub enum NetworkMessage { /// The result of the validation validation_result: MessageAcceptance, }, - /// Called if a known external TCP socket address has been updated. + /// Called if UPnP managed to establish an external port mapping. UPnPMappingEstablished { - /// The external TCP address has been updated. - tcp_socket: Option, - /// The external UDP sockets have been established. - udp_sockets: Vec, + /// The mappings that were established. + mappings: EstablishedUPnPMappings, }, /// Reports a peer to the peer manager for performing an action. ReportPeer { @@ -191,7 +190,7 @@ pub struct NetworkService { network_globals: Arc>, /// Stores potentially created UPnP mappings to be removed on shutdown. (TCP port and UDP /// ports). - upnp_mappings: (Option, Vec), + upnp_mappings: EstablishedUPnPMappings, /// A delay that expires when a new fork takes place. next_fork_update: Pin>>, /// A delay that expires when we need to subscribe to a new fork's topics. @@ -356,7 +355,7 @@ impl NetworkService { router_send, store, network_globals: network_globals.clone(), - upnp_mappings: (None, Vec::new()), + upnp_mappings: EstablishedUPnPMappings::default(), next_fork_update, next_fork_subscriptions, next_unsubscribe, @@ -612,18 +611,17 @@ impl NetworkService { } => { self.libp2p.send_error_reponse(peer_id, id, error, reason); } - NetworkMessage::UPnPMappingEstablished { - tcp_socket, - udp_sockets, - } => { - self.upnp_mappings = (tcp_socket.map(|s| s.port()), udp_sockets); + NetworkMessage::UPnPMappingEstablished { mappings } => { + self.upnp_mappings = mappings; // If there is an external TCP port update, modify our local ENR. - if let Some(tcp_socket) = tcp_socket { - if let Err(e) = self - .libp2p - .discovery_mut() - .update_enr_tcp_port(tcp_socket.port()) - { + if let Some(tcp_port) = self.upnp_mappings.tcp_port { + if let Err(e) = self.libp2p.discovery_mut().update_enr_tcp_port(tcp_port) { + warn!(self.log, "Failed to update ENR"; "error" => e); + } + } + // If there is an external QUIC port update, modify our local ENR. + if let Some(quic_port) = self.upnp_mappings.udp_quic_port { + if let Err(e) = self.libp2p.discovery_mut().update_enr_quic_port(quic_port) { warn!(self.log, "Failed to update ENR"; "error" => e); } } @@ -977,7 +975,7 @@ impl Drop for NetworkService { } // attempt to remove port mappings - crate::nat::remove_mappings(self.upnp_mappings.0, &self.upnp_mappings.1, &self.log); + crate::nat::remove_mappings(&self.upnp_mappings, &self.log); info!(self.log, "Network service shutdown"); } From 19f3c463e1643860413566fb2c1703b85eed177f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 7 Sep 2023 12:29:53 +0100 Subject: [PATCH 58/70] fix quic metric protocol on connection close --- .../lighthouse_network/src/peer_manager/network_behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index cf9937d46df..8eb45b216e6 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -335,7 +335,7 @@ impl PeerManager { multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) ) }) { - Some(multiaddr::Protocol::Quic) => { + Some(multiaddr::Protocol::QuicV1) => { metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); } Some(multiaddr::Protocol::Tcp(_)) => { From 28d81d65c6b48f1f0219aac30596a90ac5275481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 7 Sep 2023 14:14:00 +0100 Subject: [PATCH 59/70] fix doppelganger_protection script --- scripts/tests/doppelganger_protection.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/tests/doppelganger_protection.sh b/scripts/tests/doppelganger_protection.sh index e9d3e39ce50..04b2eb96d22 100755 --- a/scripts/tests/doppelganger_protection.sh +++ b/scripts/tests/doppelganger_protection.sh @@ -51,15 +51,15 @@ sleep 20 echo "Starting local beacon nodes" -exit_if_fails ../local_testnet/beacon_node.sh -d debug $HOME/.lighthouse/local-testnet/node_1 9000 8000 http://localhost:5000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> beacon1.log & -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 http://localhost:5100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> /dev/null & -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 http://localhost:5200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh -d debug $HOME/.lighthouse/local-testnet/node_1 9000 8000 10000 http://localhost:5000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 10100 http://localhost:5100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 10200 http://localhost:5200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & echo "Starting local validator clients" -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:8000 &> /dev/null & -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:8100 &> /dev/null & -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:8200 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:10000 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:10100 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:10200 &> /dev/null & echo "Waiting an epoch before starting the next validator client" sleep $(( $SECONDS_PER_SLOT * 32 )) @@ -70,7 +70,7 @@ if [[ "$BEHAVIOR" == "failure" ]]; then # Use same keys as keys from VC1 and connect to BN2 # This process should not last longer than 2 epochs - timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:8100 + timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:10100 DOPPELGANGER_EXIT=$? echo "Shutting down" @@ -96,7 +96,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then echo "Starting the last validator client" - ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:8100 & + ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:101100 & DOPPELGANGER_FAILURE=0 # Sleep three epochs, then make sure all validators were active in epoch 2. Use @@ -154,4 +154,4 @@ if [[ "$BEHAVIOR" == "success" ]]; then fi fi -exit 0 \ No newline at end of file +exit 0 From e230a57e9e5d4a6e5252f05a35952c2cb9e35313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 7 Sep 2023 22:52:11 +0100 Subject: [PATCH 60/70] fix ports for doppelganger protection test on ci --- scripts/tests/doppelganger_protection.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/tests/doppelganger_protection.sh b/scripts/tests/doppelganger_protection.sh index 04b2eb96d22..7e54fe18cdb 100755 --- a/scripts/tests/doppelganger_protection.sh +++ b/scripts/tests/doppelganger_protection.sh @@ -43,23 +43,23 @@ sleep 10 echo "Starting local execution nodes" -exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir1 7000 6000 5000 $genesis_file &> geth.log & -exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir2 7100 6100 5100 $genesis_file &> /dev/null & -exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir3 7200 6200 5200 $genesis_file &> /dev/null & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir1 6000 5000 4000 $genesis_file &> geth.log & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir2 6100 5100 4100 $genesis_file &> /dev/null & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir3 6200 5200 4200 $genesis_file &> /dev/null & sleep 20 echo "Starting local beacon nodes" -exit_if_fails ../local_testnet/beacon_node.sh -d debug $HOME/.lighthouse/local-testnet/node_1 9000 8000 10000 http://localhost:5000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> /dev/null & -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 10100 http://localhost:5100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> /dev/null & -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 10200 http://localhost:5200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh -d debug $HOME/.lighthouse/local-testnet/node_1 8000 7000 9000 http://localhost:4000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 8100 7100 9100 http://localhost:4100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 8200 7200 9200 http://localhost:4200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & echo "Starting local validator clients" -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:10000 &> /dev/null & -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:10100 &> /dev/null & -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:10200 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:9000 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:9100 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:9200 &> /dev/null & echo "Waiting an epoch before starting the next validator client" sleep $(( $SECONDS_PER_SLOT * 32 )) @@ -70,7 +70,7 @@ if [[ "$BEHAVIOR" == "failure" ]]; then # Use same keys as keys from VC1 and connect to BN2 # This process should not last longer than 2 epochs - timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:10100 + timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:9100 DOPPELGANGER_EXIT=$? echo "Shutting down" From dd444767c66645f77a10132f97e0d73ae0d77f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 8 Sep 2023 00:02:12 +0100 Subject: [PATCH 61/70] fix missing port for last validator client --- scripts/tests/doppelganger_protection.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/doppelganger_protection.sh b/scripts/tests/doppelganger_protection.sh index 7e54fe18cdb..78b83a11527 100755 --- a/scripts/tests/doppelganger_protection.sh +++ b/scripts/tests/doppelganger_protection.sh @@ -96,7 +96,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then echo "Starting the last validator client" - ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:101100 & + ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:9100 & DOPPELGANGER_FAILURE=0 # Sleep three epochs, then make sure all validators were active in epoch 2. Use From bba822062e21c8d648ea23c7483a0044c9344909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 8 Sep 2023 09:00:44 +0100 Subject: [PATCH 62/70] fix last two ports in doppelganger protection test --- scripts/tests/doppelganger_protection.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/doppelganger_protection.sh b/scripts/tests/doppelganger_protection.sh index 78b83a11527..1eefa7cf522 100755 --- a/scripts/tests/doppelganger_protection.sh +++ b/scripts/tests/doppelganger_protection.sh @@ -110,7 +110,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then cd $HOME/.lighthouse/local-testnet/node_4/validators for val in 0x*; do [[ -e $val ]] || continue - curl -s localhost:8100/lighthouse/validator_inclusion/3/$val | jq | grep -q '"is_previous_epoch_target_attester": false' + curl -s localhost:9100/lighthouse/validator_inclusion/3/$val | jq | grep -q '"is_previous_epoch_target_attester": false' IS_ATTESTER=$? if [[ $IS_ATTESTER -eq 0 ]]; then echo "$val did not attest in epoch 2." @@ -128,7 +128,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then sleep $(( $SECONDS_PER_SLOT * 32 * 2 )) for val in 0x*; do [[ -e $val ]] || continue - curl -s localhost:8100/lighthouse/validator_inclusion/5/$val | jq | grep -q '"is_previous_epoch_target_attester": true' + curl -s localhost:9100/lighthouse/validator_inclusion/5/$val | jq | grep -q '"is_previous_epoch_target_attester": true' IS_ATTESTER=$? if [[ $IS_ATTESTER -eq 0 ]]; then echo "$val attested in epoch 4." From df64302fb163dd57721326f440d4fabf35461657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 8 Sep 2023 09:19:20 +0100 Subject: [PATCH 63/70] fix port order on start_local_testnet.sh --- scripts/local_testnet/start_local_testnet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local_testnet/start_local_testnet.sh b/scripts/local_testnet/start_local_testnet.sh index a9b12a252da..c641871ad91 100755 --- a/scripts/local_testnet/start_local_testnet.sh +++ b/scripts/local_testnet/start_local_testnet.sh @@ -139,7 +139,7 @@ sed -i 's/"shanghaiTime".*$/"shanghaiTime": 0,/g' $genesis_file for (( bn=1; bn<=$BN_COUNT; bn++ )); do secret=$DATADIR/geth_datadir$bn/geth/jwtsecret echo $secret - execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_http_port_base + $bn)) $((BN_udp_tcp_base + $bn + 100)) http://localhost:$((EL_base_auth_http + $bn)) $secret + execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_udp_tcp_base + $bn + 100)) $((BN_http_port_base + $bn)) http://localhost:$((EL_base_auth_http + $bn)) $secret done # Start requested number of validator clients From 2be25d44aeaacab2d059eaea957badf40d29e5ec Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 11 Sep 2023 14:27:04 +1000 Subject: [PATCH 64/70] Hopefully improve port management in rpc tests --- .../lighthouse_network/tests/common.rs | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index e72883a1699..41233dca525 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -13,7 +13,7 @@ use tokio::runtime::Runtime; use types::{ ChainSpec, EnrForkId, Epoch, EthSpec, ForkContext, ForkName, Hash256, MinimalEthSpec, Slot, }; -use unused_port::unused_tcp4_port; +use unused_port::{unused_tcp4_port, unused_udp4_port}; type E = MinimalEthSpec; type ReqId = usize; @@ -68,15 +68,22 @@ pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger { } } -pub fn build_config(port: u16, mut boot_nodes: Vec) -> NetworkConfig { +pub fn build_config(mut boot_nodes: Vec) -> NetworkConfig { let mut config = NetworkConfig::default(); + + // Find unused ports + let tcp_port = unused_tcp4_port().unwrap(); + let disc_port = unused_udp4_port().unwrap(); + let quic_port = unused_udp4_port().unwrap(); + let path = TempBuilder::new() - .prefix(&format!("libp2p_test{}", port)) + .prefix(&format!("libp2p_test{}_{}_{}", tcp_port, disc_port, quic_port)) .tempdir() .unwrap(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port, port + 1); - config.enr_udp4_port = Some(port); + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, tcp_port, disc_port, quic_port); + config.enr_udp4_port = Some(disc_port); + config.enr_quic4_port = Some(quic_port); config.enr_address = (Some(std::net::Ipv4Addr::LOCALHOST), None); config.boot_nodes_enr.append(&mut boot_nodes); config.network_dir = path.into_path(); @@ -96,8 +103,7 @@ pub async fn build_libp2p_instance( fork_name: ForkName, spec: &ChainSpec, ) -> Libp2pInstance { - let port = unused_tcp4_port().unwrap(); - let config = build_config(port, boot_nodes); + let config = build_config(boot_nodes); // launch libp2p service let (signal, exit) = exit_future::signal(); From d01b2e198005662563ce5f6b0dddb9c8d9e0b029 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 11 Sep 2023 14:44:59 +1000 Subject: [PATCH 65/70] fmt --- beacon_node/lighthouse_network/tests/common.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 41233dca525..60822f8730d 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -77,11 +77,19 @@ pub fn build_config(mut boot_nodes: Vec) -> NetworkConfig { let quic_port = unused_udp4_port().unwrap(); let path = TempBuilder::new() - .prefix(&format!("libp2p_test{}_{}_{}", tcp_port, disc_port, quic_port)) + .prefix(&format!( + "libp2p_test{}_{}_{}", + tcp_port, disc_port, quic_port + )) .tempdir() .unwrap(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, tcp_port, disc_port, quic_port); + config.set_ipv4_listening_address( + std::net::Ipv4Addr::UNSPECIFIED, + tcp_port, + disc_port, + quic_port, + ); config.enr_udp4_port = Some(disc_port); config.enr_quic4_port = Some(quic_port); config.enr_address = (Some(std::net::Ipv4Addr::LOCALHOST), None); From 74f0375db3934f1a28da250e97a969cb6edacfe5 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 11 Sep 2023 19:32:37 +1000 Subject: [PATCH 66/70] Update tests due to unstable merge --- .../lighthouse_network/tests/common.rs | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 2be4294510c..969bdb76ca8 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -164,23 +164,49 @@ pub async fn build_node_pair( let mut sender = build_libp2p_instance(rt.clone(), vec![], sender_log, fork_name, spec).await; let mut receiver = build_libp2p_instance(rt, vec![], receiver_log, fork_name, spec).await; - let receiver_multiaddr = match protocol { - Protocol::Tcp => receiver.local_enr().multiaddr_tcp().pop().unwrap(), - Protocol::Quic => receiver.local_enr().multiaddr_quic().pop().unwrap(), - }; - // let the two nodes set up listeners let sender_fut = async { loop { if let NetworkEvent::NewListenAddr(addr) = sender.next_event().await { - return addr; + // Only end once we've listened on the protocol we care about + match protocol { + Protocol::Tcp => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::Tcp(_)) + }) { + return addr; + } + } + Protocol::Quic => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::QuicV1) + }) { + return addr; + } + } + } } } }; let receiver_fut = async { loop { if let NetworkEvent::NewListenAddr(addr) = receiver.next_event().await { - return addr; + match protocol { + Protocol::Tcp => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::Tcp(_)) + }) { + return addr; + } + } + Protocol::Quic => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::QuicV1) + }) { + return addr; + } + } + } } } }; From fa11ff902a74ed33adcbdcc0f33eb0a4fe52cde7 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 12 Sep 2023 11:24:09 +1000 Subject: [PATCH 67/70] Small test changes --- .../lighthouse_network/src/listen_addr.rs | 2 +- .../lighthouse_network/tests/common.rs | 26 ++++--------------- .../lighthouse_network/tests/rpc_tests.rs | 2 +- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 4b43ca5af14..53f7d9dacae 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -25,7 +25,7 @@ impl + Clone> ListenAddr { (self.addr.clone().into(), self.quic_port).into() } - pub fn libp2p_socket_addr(&self) -> SocketAddr { + pub fn tcp_socket_addr(&self) -> SocketAddr { (self.addr.clone().into(), self.tcp_port).into() } } diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 969bdb76ca8..7b437fe7a68 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -13,7 +13,6 @@ use tokio::runtime::Runtime; use types::{ ChainSpec, EnrForkId, Epoch, EthSpec, ForkContext, ForkName, Hash256, MinimalEthSpec, Slot, }; -use unused_port::{unused_tcp4_port, unused_udp4_port}; type E = MinimalEthSpec; type ReqId = usize; @@ -71,31 +70,16 @@ pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger { pub fn build_config(mut boot_nodes: Vec) -> NetworkConfig { let mut config = NetworkConfig::default(); - // Find unused ports - let tcp_port = unused_tcp4_port().unwrap(); - let disc_port = unused_udp4_port().unwrap(); - let quic_port = unused_udp4_port().unwrap(); + // Find unused ports by using the 0 port. + let port = 0; + let random_path: u16 = rand::random(); let path = TempBuilder::new() - .prefix(&format!( - "libp2p_test{}_{}_{}", - tcp_port, disc_port, quic_port - )) + .prefix(&format!("libp2p_test_{}", random_path)) .tempdir() .unwrap(); - config.set_ipv4_listening_address( - std::net::Ipv4Addr::UNSPECIFIED, - tcp_port, - disc_port, - quic_port, - ); - config.enr_udp4_port = if disc_port == 0 { - None - } else { - Some(disc_port) - }; - config.enr_quic4_port = Some(quic_port); + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port, port); config.enr_address = (Some(std::net::Ipv4Addr::LOCALHOST), None); config.boot_nodes_enr.append(&mut boot_nodes); config.network_dir = path.into_path(); diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 85f52140e37..795afd06b9e 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -877,7 +877,7 @@ fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { }) } -/// Established a pair of nodes and disconnects the pair based on the selected protocol via an RPC +/// Establishes a pair of nodes and disconnects the pair based on the selected protocol via an RPC /// Goodbye message. fn goodbye_test(log_level: Level, enable_logging: bool, protocol: Protocol) { let log = common::build_log(log_level, enable_logging); From 231e7dbf6e26eb113713b4a371224c107eec6035 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 13 Sep 2023 11:18:13 +1000 Subject: [PATCH 68/70] Nuke windows self-hosted runner --- .github/workflows/test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 1d9856b93cf..9b591fb270d 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -73,7 +73,7 @@ jobs: run: make test-release release-tests-windows: name: release-tests-windows - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019' }} + runs-on: 'windows-2019' # temp nuke self-hosted ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 From a4faad52429fcbd62c53211a3f3278c375da728e Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 13 Sep 2023 15:55:27 +1000 Subject: [PATCH 69/70] Revert "Nuke windows self-hosted runner" This reverts commit 231e7dbf6e26eb113713b4a371224c107eec6035. --- .github/workflows/test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 9b591fb270d..1d9856b93cf 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -73,7 +73,7 @@ jobs: run: make test-release release-tests-windows: name: release-tests-windows - runs-on: 'windows-2019' # temp nuke self-hosted ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019' }} + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 From 41c8207a3953f9ff12d0357245c90ce13b881712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Wed, 13 Sep 2023 14:36:27 +0100 Subject: [PATCH 70/70] beacon node: move connected peer metrics, put them before the ban status checks to avoid negative counts as each FromSwarm::ConnectionClosed event is always paired with an earlier FromSwarm::ConnectionEstablished, and when we ban before incrementing metrics we return earlier. --- .../src/peer_manager/network_behaviour.rs | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 8eb45b216e6..fedb876bb23 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -213,6 +213,34 @@ impl PeerManager { metrics::check_nat(); } + // increment prometheus metrics + if self.metrics_enabled { + let remote_addr = match endpoint { + ConnectedPoint::Dialer { address, .. } => address, + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + }; + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::QuicV1) => { + metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); + } + Some(_) => unreachable!(), + None => { + error!(self.log, "Connection established via unknown transport"; "addr" => %remote_addr) + } + }; + + self.update_connected_peer_metrics(); + metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + } + // Check to make sure the peer is not supposed to be banned match self.ban_status(&peer_id) { // TODO: directly emit the ban event? @@ -253,42 +281,18 @@ impl PeerManager { // NOTE: We don't register peers that we are disconnecting immediately. The network service // does not need to know about these peers. - let remote_addr = match endpoint { + match endpoint { ConnectedPoint::Listener { send_back_addr, .. } => { self.inject_connect_ingoing(&peer_id, send_back_addr.clone(), None); self.events .push(PeerManagerEvent::PeerConnectedIncoming(peer_id)); - send_back_addr } ConnectedPoint::Dialer { address, .. } => { self.inject_connect_outgoing(&peer_id, address.clone(), None); self.events .push(PeerManagerEvent::PeerConnectedOutgoing(peer_id)); - address } }; - - // increment prometheus metrics - if self.metrics_enabled { - match remote_addr.iter().find(|proto| { - matches!( - proto, - multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) - ) - }) { - Some(multiaddr::Protocol::QuicV1) => { - metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); - } - Some(multiaddr::Protocol::Tcp(_)) => { - metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); - } - Some(_) => unreachable!(), - None => error!(self.log, "Connected via unknown transport"; "addr" => %remote_addr), - }; - - self.update_connected_peer_metrics(); - metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); - } } fn on_connection_closed(