rpc: upgrade jsonrpsee v0.23

bridges/relays/client-substrate/Cargo.toml

@@ -15,10 +15,11 @@ async-std = { version = "1.9.0", features = ["attributes"] }
 async-trait = "0.1.79"
 codec = { package = "parity-scale-codec", version = "3.6.12" }
 futures = "0.3.30"
-jsonrpsee = { version = "0.22", features = ["macros", "ws-client"] }
+jsonrpsee = { version = "0.23.1", features = ["macros", "ws-client"] }
 log = { workspace = true }
 num-traits = "0.2"
 rand = "0.8.5"
+serde_json = { workspace = true }
 scale-info = { version = "2.11.1", features = ["derive"] }
 tokio = { version = "1.37", features = ["rt-multi-thread"] }
 thiserror = { workspace = true }

bridges/relays/client-substrate/src/client/subscription.rs

@@ -21,7 +21,6 @@ use async_std::{
 	stream::StreamExt,
 };
 use futures::{FutureExt, Stream};
-use jsonrpsee::core::ClientError;
 use sp_runtime::DeserializeOwned;
 use std::{
 	fmt::Debug,
@@ -143,7 +142,7 @@ impl<T: 'static + Clone + DeserializeOwned + Send> Subscription<T> {
 	/// Create new forwarded subscription.
 	pub fn new_forwarded(
 		desc: StreamDescription,
-		subscription: impl Stream<Item = StdResult<T, ClientError>> + Unpin + Send + 'static,
+		subscription: impl Stream<Item = StdResult<T, serde_json::Error>> + Unpin + Send + 'static,
 	) -> Self {
 		Self {
 			desc: desc.clone(),

cumulus/client/relay-chain-interface/Cargo.toml

@@ -23,5 +23,5 @@ sp-version = { path = "../../../substrate/primitives/version", default-features
 futures = "0.3.28"
 async-trait = "0.1.79"
 thiserror = { workspace = true }
-jsonrpsee-core = "0.22"
+jsonrpsee-core = "0.23.1"
 codec = { package = "parity-scale-codec", version = "3.6.12" }

cumulus/client/relay-chain-rpc-interface/Cargo.toml

@@ -33,7 +33,7 @@ tokio-util = { version = "0.7.8", features = ["compat"] }
 futures = "0.3.28"
 futures-timer = "3.0.2"
 codec = { package = "parity-scale-codec", version = "3.6.12" }
-jsonrpsee = { version = "0.22", features = ["ws-client"] }
+jsonrpsee = { version = "0.23.1", features = ["ws-client"] }
 tracing = "0.1.37"
 async-trait = "0.1.79"
 url = "2.4.0"

cumulus/client/relay-chain-rpc-interface/src/light_client_worker.rs

@@ -20,7 +20,7 @@
 
 use futures::{channel::mpsc::Sender, prelude::*, stream::FuturesUnordered};
 use jsonrpsee::core::client::{
-	Client as JsonRpseeClient, ClientBuilder, ClientT, Error, ReceivedMessage, TransportReceiverT,
+	Client as JsonRpseeClient, ClientBuilder, ClientT, ReceivedMessage, TransportReceiverT,
 	TransportSenderT,
 };
 use smoldot_light::{ChainId, Client as SmoldotClient, JsonRpcResponses};
@@ -124,7 +124,7 @@ pub struct LightClientRpcWorker {
 }
 
 fn handle_notification(
-	maybe_header: Option<Result<RelayHeader, Error>>,
+	maybe_header: Option<Result<RelayHeader, serde_json::Error>>,
 	senders: &mut Vec<Sender<RelayHeader>>,
 ) -> Result<(), ()> {
 	match maybe_header {

cumulus/polkadot-parachain/Cargo.toml

@@ -39,7 +39,7 @@ coretime-rococo-runtime = { path = "../parachains/runtimes/coretime/coretime-roc
 coretime-westend-runtime = { path = "../parachains/runtimes/coretime/coretime-westend" }
 bridge-hub-westend-runtime = { path = "../parachains/runtimes/bridge-hubs/bridge-hub-westend" }
 penpal-runtime = { path = "../parachains/runtimes/testing/penpal" }
-jsonrpsee = { version = "0.22", features = ["server"] }
+jsonrpsee = { version = "0.23.1", features = ["server"] }
 people-rococo-runtime = { path = "../parachains/runtimes/people/people-rococo" }
 people-westend-runtime = { path = "../parachains/runtimes/people/people-westend" }
 parachains-common = { path = "../parachains/common" }

cumulus/test/service/Cargo.toml

@@ -17,7 +17,7 @@ async-trait = "0.1.79"
 clap = { version = "4.5.3", features = ["derive"] }
 codec = { package = "parity-scale-codec", version = "3.6.12" }
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-jsonrpsee = { version = "0.22", features = ["server"] }
+jsonrpsee = { version = "0.23.1", features = ["server"] }
 rand = "0.8.5"
 serde = { features = ["derive"], workspace = true, default-features = true }
 serde_json = { workspace = true, default-features = true }

polkadot/rpc/Cargo.toml

@@ -10,7 +10,7 @@ description = "Polkadot specific RPC functionality."
 workspace = true
 
 [dependencies]
-jsonrpsee = { version = "0.22", features = ["server"] }
+jsonrpsee = { version = "0.23.1", features = ["server"] }
 polkadot-primitives = { path = "../primitives" }
 sc-client-api = { path = "../../substrate/client/api" }
 sp-blockchain = { path = "../../substrate/primitives/blockchain" }

prdoc/pr_4730.prdoc

@@ -0,0 +1,25 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: rpc upgrade jsonrpsee to v0.23.1
+
+doc:
+  - audience: Node Dev
+    description: |
+      Upgrade the rpc library jsonrpsee to v0.23.1 to utilize:
+
+      - Add Extensions which we now is using to get the connection id (used by the rpc spec v2)
+      - Update hyper to v1.0, http v1.0, soketto and related crates (hyper::service::make_service_fn is removed)
+      - The subscription API for the client is modified to know why a subscription was closed.
+
+crates:
+  - name: sc-rpc-spec-v2
+    bump: patch
+  - name: sc-rpc
+    bump: patch
+  - name: sc-rpc-server
+    bump: patch
+  - name: cumulus-relay-chain-rpc-interface
+    bump: patch
+  - name: frame-remote-externalities
+    bump: patch

substrate/bin/node/cli/Cargo.toml

@@ -41,7 +41,7 @@ array-bytes = "6.1"
 clap = { version = "4.5.3", features = ["derive"], optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.12" }
 serde = { features = ["derive"], workspace = true, default-features = true }
-jsonrpsee = { version = "0.22", features = ["server"] }
+jsonrpsee = { version = "0.23.1", features = ["server"] }
 futures = "0.3.30"
 log = { workspace = true, default-features = true }
 rand = "0.8"

substrate/bin/node/rpc/Cargo.toml

@@ -16,7 +16,7 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-jsonrpsee = { version = "0.22", features = ["server"] }
+jsonrpsee = { version = "0.23.1", features = ["server"] }
 node-primitives = { path = "../primitives" }
 pallet-transaction-payment-rpc = { path = "../../../frame/transaction-payment/rpc" }
 mmr-rpc = { path = "../../../client/merkle-mountain-range/rpc" }

substrate/client/consensus/babe/rpc/Cargo.toml

@@ -16,7 +16,7 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-jsonrpsee = { version = "0.22.5", features = ["client-core", "macros", "server-core"] }
+jsonrpsee = { version = "0.23.1", features = ["client-core", "macros", "server-core"] }
 futures = "0.3.30"
 serde = { features = ["derive"], workspace = true, default-features = true }
 thiserror = { workspace = true }

substrate/client/consensus/beefy/rpc/Cargo.toml

@@ -14,7 +14,7 @@ workspace = true
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.6.12", features = ["derive"] }
 futures = "0.3.30"
-jsonrpsee = { version = "0.22.5", features = ["client-core", "macros", "server-core"] }
+jsonrpsee = { version = "0.23.1", features = ["client-core", "macros", "server-core"] }
 log = { workspace = true, default-features = true }
 parking_lot = "0.12.1"
 serde = { features = ["derive"], workspace = true, default-features = true }

substrate/client/consensus/grandpa/rpc/Cargo.toml

@@ -15,7 +15,7 @@ workspace = true
 [dependencies]
 finality-grandpa = { version = "0.16.2", features = ["derive-codec"] }
 futures = "0.3.30"
-jsonrpsee = { version = "0.22.5", features = ["client-core", "macros", "server-core"] }
+jsonrpsee = { version = "0.23.1", features = ["client-core", "macros", "server-core"] }
 log = { workspace = true, default-features = true }
 codec = { package = "parity-scale-codec", version = "3.6.12", features = ["derive"] }
 serde = { features = ["derive"], workspace = true, default-features = true }

substrate/client/consensus/manual-seal/Cargo.toml

@@ -16,7 +16,7 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-jsonrpsee = { version = "0.22.5", features = ["client-core", "macros", "server-core"] }
+jsonrpsee = { version = "0.23.1", features = ["client-core", "macros", "server-core"] }
 assert_matches = "1.3.0"
 async-trait = "0.1.79"
 codec = { package = "parity-scale-codec", version = "3.6.12" }

substrate/client/merkle-mountain-range/rpc/Cargo.toml

@@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.6.12" }
-jsonrpsee = { version = "0.22.5", features = ["client-core", "macros", "server-core"] }
+jsonrpsee = { version = "0.23.1", features = ["client-core", "macros", "server-core"] }
 serde = { features = ["derive"], workspace = true, default-features = true }
 sp-api = { path = "../../../primitives/api" }
 sp-blockchain = { path = "../../../primitives/blockchain" }

substrate/client/rpc-api/Cargo.toml

@@ -28,4 +28,4 @@ sp-core = { path = "../../primitives/core" }
 sp-rpc = { path = "../../primitives/rpc" }
 sp-runtime = { path = "../../primitives/runtime" }
 sp-version = { path = "../../primitives/version" }
-jsonrpsee = { version = "0.22.5", features = ["client-core", "macros", "server-core"] }
+jsonrpsee = { version = "0.23.1", features = ["client-core", "macros", "server-core"] }

substrate/client/rpc-servers/Cargo.toml

@@ -19,14 +19,15 @@ targets = ["x86_64-unknown-linux-gnu"]
 forwarded-header-value = "0.1.1"
 futures = "0.3.30"
 governor = "0.6.0"
-http = "0.2.8"
-hyper = "0.14.27"
+http = "1.1"
+http-body-util = "0.1.2"
+hyper = "1.3"
 ip_network = "0.4.1"
-jsonrpsee = { version = "0.22", features = ["server"] }
+jsonrpsee = { version = "0.23.1", features = ["server"] }
 log = { workspace = true, default-features = true }
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" }
 serde = { workspace = true }
 serde_json = { workspace = true, default-features = true }
 tokio = { version = "1.22.0", features = ["parking_lot"] }
 tower = { version = "0.4.13", features = ["util"] }
-tower-http = { version = "0.4.0", features = ["cors"] }
+tower-http = { version = "0.5.2", features = ["cors"] }

substrate/client/rpc-servers/src/lib.rs

@@ -23,16 +23,13 @@
 pub mod middleware;
 pub mod utils;
 
-use std::{
-	convert::Infallible, error::Error as StdError, net::SocketAddr, num::NonZeroU32, time::Duration,
-};
+use std::{error::Error as StdError, net::SocketAddr, num::NonZeroU32, sync::Arc, time::Duration};
 
-use hyper::{
-	server::conn::AddrStream,
-	service::{make_service_fn, service_fn},
-};
 use jsonrpsee::{
-	server::{stop_channel, ws, PingConfig, StopHandle, TowerServiceBuilder},
+	core::BoxError,
+	server::{
+		serve_with_graceful_shutdown, stop_channel, ws, PingConfig, StopHandle, TowerServiceBuilder,
+	},
 	Methods, RpcModule,
 };
 use middleware::NodeHealthProxyLayer;
@@ -97,6 +94,7 @@ struct PerConnection<RpcMiddleware, HttpMiddleware> {
 	metrics: Option<RpcMetrics>,
 	tokio_handle: tokio::runtime::Handle,
 	service_builder: TowerServiceBuilder<RpcMiddleware, HttpMiddleware>,
+	rate_limit_whitelisted_ips: Arc<Vec<IpNetwork>>,
 }
 
 /// Start RPC server listening on given address.
@@ -124,8 +122,8 @@ where
 		rate_limit_trust_proxy_headers,
 	} = config;
 
-	let std_listener = TcpListener::bind(addrs.as_slice()).await?.into_std()?;
-	let local_addr = std_listener.local_addr().ok();
+	let listener = TcpListener::bind(addrs.as_slice()).await?;
+	let local_addr = listener.local_addr().ok();
 	let host_filter = host_filtering(cors.is_some(), local_addr);
 
 	let http_middleware = tower::ServiceBuilder::new()
@@ -161,20 +159,38 @@ where
 		methods: build_rpc_api(rpc_api).into(),
 		service_builder: builder.to_service_builder(),
 		metrics,
-		tokio_handle,
-		stop_handle: stop_handle.clone(),
+		tokio_handle: tokio_handle.clone(),
+		stop_handle,
+		rate_limit_whitelisted_ips: Arc::new(rate_limit_whitelisted_ips),
 	};
 
-	let make_service = make_service_fn(move |addr: &AddrStream| {
-		let cfg = cfg.clone();
-		let rate_limit_whitelisted_ips = rate_limit_whitelisted_ips.clone();
-		let ip = addr.remote_addr().ip();
-
-		async move {
-			let cfg = cfg.clone();
-			let rate_limit_whitelisted_ips = rate_limit_whitelisted_ips.clone();
+	tokio_handle.spawn(async move {
+		loop {
+			let (sock, remote_addr) = tokio::select! {
+				res = listener.accept() => {
+					match res {
+						Ok(s) => s,
+						Err(e) => {
+							log::debug!(target: "rpc", "Failed to accept ipv4 connection: {:?}", e);
+							continue;
+						}
+					}
+				}
+				_ = cfg.stop_handle.clone().shutdown() => break,
+			};
+
+			let ip = remote_addr.ip();
+			let cfg2 = cfg.clone();
+			let svc = tower::service_fn(move |req: http::Request<hyper::body::Incoming>| {
+				let PerConnection {
+					methods,
+					service_builder,
+					metrics,
+					tokio_handle,
+					stop_handle,
+					rate_limit_whitelisted_ips,
+				} = cfg2.clone();
 
-			Ok::<_, Infallible>(service_fn(move |req| {
 				let proxy_ip =
 					if rate_limit_trust_proxy_headers { get_proxy_ip(&req) } else { None };
 
@@ -191,9 +207,6 @@ where
 					rate_limit
 				};
 
-				let PerConnection { service_builder, metrics, tokio_handle, stop_handle, methods } =
-					cfg.clone();
-
 				let is_websocket = ws::is_upgrade_request(&req);
 				let transport_label = if is_websocket { "ws" } else { "http" };
 
@@ -213,7 +226,6 @@ where
 
 				let rpc_middleware =
 					RpcServiceBuilder::new().option_layer(middleware_layer.clone());
-
 				let mut svc =
 					service_builder.set_rpc_middleware(rpc_middleware).build(methods, stop_handle);
 
@@ -230,17 +242,19 @@ where
 						});
 					}
 
-					svc.call(req).await
+					// https://github.com/rust-lang/rust/issues/102211 the error type can't be inferred
+					// to be `Box<dyn std::error::Error + Send + Sync>` so we need to convert it to
+					// a concrete type as workaround.
+					svc.call(req).await.map_err(|e| BoxError::from(e))
 				}
-			}))
-		}
-	});
-
-	let server = hyper::Server::from_tcp(std_listener)?.serve(make_service);
+			});
 
-	tokio::spawn(async move {
-		let graceful = server.with_graceful_shutdown(async move { stop_handle.shutdown().await });
-		let _ = graceful.await;
+			cfg.tokio_handle.spawn(serve_with_graceful_shutdown(
+				sock,
+				svc,
+				cfg.stop_handle.clone().shutdown(),
+			));
+		}
 	});
 
 	log::info!(