From 4efc73e16da53caa1a7713eeee3f041c7b53c6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Wed, 10 Aug 2022 14:29:53 -0700 Subject: [PATCH 01/29] Introduce the 'ipfsd' feature to run the gateway as a single binary. --- iroh-gateway/Cargo.toml | 5 ++++- iroh-gateway/src/config.rs | 4 ++++ iroh-rpc-client/src/config.rs | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/iroh-gateway/Cargo.toml b/iroh-gateway/Cargo.toml index a902bb6517..0383c99bd4 100644 --- a/iroh-gateway/Cargo.toml +++ b/iroh-gateway/Cargo.toml @@ -10,6 +10,8 @@ repository = "https://github.com/dignifiedquire/iroh" [dependencies] iroh-rpc-client = { path = "../iroh-rpc-client", default-features = false } iroh-rpc-types = { path = "../iroh-rpc-types", default-features = false } +iroh-p2p = {path = "../iroh-p2p", default-features = false, features = ["rpc-mem"], optional = true} +iroh-store = {path = "../iroh-store", default-features = false, features = ["rpc-mem"], optional = true} cid = "0.8.6" @@ -63,6 +65,7 @@ axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messa [features] -default = ["rpc-grpc", "rpc-mem"] +default = ["rpc-mem", "rpc-grpc", "ipfsd"] +ipfsd = ["iroh-p2p/rpc-mem", "iroh-store/rpc-mem"] rpc-grpc = ["iroh-rpc-types/grpc", "iroh-rpc-client/grpc", "iroh-metrics/rpc-grpc"] rpc-mem = ["iroh-rpc-types/mem", "iroh-rpc-client/mem"] diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index 6d26dd7fdb..ddd97ca1fb 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -128,7 +128,11 @@ fn default_headers() -> HeaderMap { impl Default for Config { fn default() -> Self { + #[cfg(not(feature = "ipfsd"))] let rpc_client = RpcClientConfig::default_grpc(); + #[cfg(feature = "ipfsd")] + let rpc_client = RpcClientConfig::default_ipfsd(); + let mut t = Self { writeable: false, fetch: false, diff --git a/iroh-rpc-client/src/config.rs b/iroh-rpc-client/src/config.rs index 352acce228..07cfef32fc 100644 --- a/iroh-rpc-client/src/config.rs +++ b/iroh-rpc-client/src/config.rs @@ -42,6 +42,30 @@ impl Config { store_addr: Some("grpc://0.0.0.0:4402".parse().unwrap()), } } + + // When running in ipfsd mode, the resolver will use memory channels to + // communicate with the p2p and store modules. + // The gateway itself is exposing a UDS rpc endpoint to be also usable + // as a single entry point for other system services. + pub fn default_ipfsd() -> Self { + use iroh_rpc_types::Addr; + let path = { + #[cfg(target_os = "android")] + "/dev/socket/ipfsd".into(); + + #[cfg(not(target_os = "android"))] + { + let path = format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()); + path.into() + } + }; + + Self { + gateway_addr: Some(Addr::GrpcUds(path)), + p2p_addr: None, + store_addr: None, + } + } } #[cfg(test)] From 42963267974e2ca3d13969a8e32d2cb2c4b1b143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Thu, 11 Aug 2022 13:16:53 -0700 Subject: [PATCH 02/29] Add mem-rpc store to ipfsd --- iroh-gateway/src/lib.rs | 2 ++ iroh-gateway/src/main.rs | 17 ++++++++++++++ iroh-gateway/src/mem_store.rs | 44 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 iroh-gateway/src/mem_store.rs diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index 5f7df4a8c1..cfff0eca94 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -5,6 +5,8 @@ mod constants; pub mod core; mod error; mod headers; +#[cfg(feature = "ipfsd")] +pub mod mem_store; pub mod metrics; mod response; mod rpc; diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index 7116cfb294..81b331e857 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -11,6 +11,7 @@ use iroh_gateway::{ metrics, }; use iroh_metrics::gateway::Metrics; +use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; use tokio::sync::RwLock; @@ -74,6 +75,16 @@ async fn main() -> Result<()> { args.make_overrides_map(), ) .unwrap(); + + // When running in ipfsd mode, update the rpc client config to setup + // memory addresses for the p2p and store modules. + #[cfg(feature = "ipfsd")] + let store_rpc = { + let (store_recv, store_sender) = Addr::new_mem(); + config.rpc_client.store_addr = Some(store_sender); + iroh_gateway::mem_store::start(store_recv).await? + }; + config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); println!("{:#?}", config); @@ -109,6 +120,12 @@ async fn main() -> Result<()> { }); iroh_util::block_until_sigint().await; + + #[cfg(feature = "ipfsd")] + { + store_rpc.abort(); + } + core_task.abort(); metrics_handle.shutdown(); diff --git a/iroh-gateway/src/mem_store.rs b/iroh-gateway/src/mem_store.rs new file mode 100644 index 0000000000..4c1d4c163d --- /dev/null +++ b/iroh-gateway/src/mem_store.rs @@ -0,0 +1,44 @@ +/// A store instance listening on a memory rpc channel. +use iroh_metrics::store::Metrics; +use iroh_rpc_types::store::StoreServerAddr; +use iroh_store::{config::ENV_PREFIX, rpc, Config, Store}; +use iroh_util::make_config; +use prometheus_client::registry::Registry; +use std::collections::HashMap; +use std::path::PathBuf; +use tokio::task::JoinHandle; +use tracing::info; + +/// Starts a new store, using the given mem rpc channel. +/// TODO: refactor to share most of the setup with iroh-store/src/main.rs +pub async fn start(rpc_addr: StoreServerAddr) -> anyhow::Result> { + println!("Starting memory store with addr {}", rpc_addr); + + let overrides: HashMap = HashMap::new(); + let config: iroh_store::Config = make_config( + // default + Config::new_grpc(PathBuf::from("./iroh-store")), + // potential config files + vec![], + // env var prefix for this config + ENV_PREFIX, + // map of present command line arguments + overrides, + ) + .unwrap(); + + let mut prom_registry = Registry::default(); + let store_metrics = Metrics::new(&mut prom_registry); + + let store = if config.path.exists() { + info!("Opening store at {}", config.path.display()); + Store::open(config, store_metrics).await? + } else { + info!("Creating store at {}", config.path.display()); + Store::create(config, store_metrics).await? + }; + + let rpc_task = tokio::spawn(async move { rpc::new(rpc_addr, store).await.unwrap() }); + + Ok(rpc_task) +} From 00faa8aa61166c1459dfec19fe7cb201e10a4d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Thu, 11 Aug 2022 13:38:09 -0700 Subject: [PATCH 03/29] Add mem-rpc p2p module to ipfsd --- iroh-gateway/src/lib.rs | 2 ++ iroh-gateway/src/main.rs | 10 ++++++-- iroh-gateway/src/mem_p2p.rs | 43 +++++++++++++++++++++++++++++++++++ iroh-gateway/src/mem_store.rs | 2 -- 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 iroh-gateway/src/mem_p2p.rs diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index cfff0eca94..f9e1c0896e 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -6,6 +6,8 @@ pub mod core; mod error; mod headers; #[cfg(feature = "ipfsd")] +pub mod mem_p2p; +#[cfg(feature = "ipfsd")] pub mod mem_store; pub mod metrics; mod response; diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index 81b331e857..d17588f885 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -79,10 +79,15 @@ async fn main() -> Result<()> { // When running in ipfsd mode, update the rpc client config to setup // memory addresses for the p2p and store modules. #[cfg(feature = "ipfsd")] - let store_rpc = { + let (store_rpc, p2p_rpc) = { let (store_recv, store_sender) = Addr::new_mem(); config.rpc_client.store_addr = Some(store_sender); - iroh_gateway::mem_store::start(store_recv).await? + let store_rpc = iroh_gateway::mem_store::start(store_recv).await?; + + let (p2p_recv, p2p_sender) = Addr::new_mem(); + config.rpc_client.p2p_addr = Some(p2p_sender); + let p2p_rpc = iroh_gateway::mem_p2p::start(p2p_recv).await?; + (store_rpc, p2p_rpc) }; config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); @@ -124,6 +129,7 @@ async fn main() -> Result<()> { #[cfg(feature = "ipfsd")] { store_rpc.abort(); + p2p_rpc.abort(); } core_task.abort(); diff --git a/iroh-gateway/src/mem_p2p.rs b/iroh-gateway/src/mem_p2p.rs new file mode 100644 index 0000000000..af6e9fdfa4 --- /dev/null +++ b/iroh-gateway/src/mem_p2p.rs @@ -0,0 +1,43 @@ +/// A p2p instance listening on a memory rpc channel. +use iroh_p2p::config::{Config, ENV_PREFIX}; +use iroh_p2p::{DiskStorage, Keychain, Node}; +use iroh_rpc_types::p2p::P2pServerAddr; +use iroh_util::make_config; +use prometheus_client::registry::Registry; +use std::collections::HashMap; +use tokio::task; +use tokio::task::JoinHandle; +use tracing::error; + +/// Starts a new p2p node, using the given mem rpc channel. +/// TODO: refactor to share most of the setup with iroh-p2p/src/main.rs +pub async fn start(rpc_addr: P2pServerAddr) -> anyhow::Result> { + // TODO: configurable network + let overrides: HashMap = HashMap::new(); + let network_config = make_config( + // default + Config::default_grpc(), + // potential config files + vec![], + // env var prefix for this config + ENV_PREFIX, + // map of present command line arguments + overrides, + ) + .unwrap(); + + let mut prom_registry = Registry::default(); + + let kc = Keychain::::new().await?; + + let mut p2p = Node::new(network_config, rpc_addr, kc, &mut prom_registry).await?; + + // Start services + let p2p_task = task::spawn(async move { + if let Err(err) = p2p.run().await { + error!("{:?}", err); + } + }); + + Ok(p2p_task) +} diff --git a/iroh-gateway/src/mem_store.rs b/iroh-gateway/src/mem_store.rs index 4c1d4c163d..fb32a544ff 100644 --- a/iroh-gateway/src/mem_store.rs +++ b/iroh-gateway/src/mem_store.rs @@ -12,8 +12,6 @@ use tracing::info; /// Starts a new store, using the given mem rpc channel. /// TODO: refactor to share most of the setup with iroh-store/src/main.rs pub async fn start(rpc_addr: StoreServerAddr) -> anyhow::Result> { - println!("Starting memory store with addr {}", rpc_addr); - let overrides: HashMap = HashMap::new(); let config: iroh_store::Config = make_config( // default From cd22c63dddc81c22327c11ed7c14149e42fb5f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Thu, 11 Aug 2022 19:07:14 -0700 Subject: [PATCH 04/29] Add http over uds support --- iroh-gateway/src/core.rs | 46 +++++++++++++++---- iroh-gateway/src/lib.rs | 2 + iroh-gateway/src/main.rs | 15 ++++++- iroh-gateway/src/uds.rs | 95 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 iroh-gateway/src/uds.rs diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 1469ce6266..51e7fa8f1d 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -6,7 +6,7 @@ use axum::{ http::{header::*, StatusCode}, response::IntoResponse, routing::get, - BoxError, Router, + BoxError, Router, Server, }; use bytes::Bytes; use futures::TryStreamExt; @@ -33,12 +33,16 @@ use std::{ time::{self, Duration}, }; use tokio::sync::RwLock; +#[cfg(feature = "ipfsd")] +use tokio::net::UnixListener; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; use tracing::info_span; use url::Url; use urlencoding::encode; +#[cfg(feature = "ipfsd")] +use crate::uds; use crate::{ bad_bits::BadBits, client::{Client, FileResult, Request}, @@ -122,12 +126,9 @@ impl Core { }) } - pub fn server( - self, - ) -> axum::Server> - { + fn get_app(&self) -> Router { // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 - let app = Router::new() + Router::new() .route("/:scheme/:cid", get(get_handler)) .route("/:scheme/:cid/*cpath", get(get_handler)) .route("/health", get(health_check)) @@ -151,15 +152,42 @@ impl Core { uri = %request.uri(), ) }), - ); + ) + } + + pub fn http_server( + &self, + ) -> Server> { + let app = self.get_app(); // todo(arqu): make configurable let addr = format!("0.0.0.0:{}", self.state.config.port); - axum::Server::bind(&addr.parse().unwrap()) + Server::bind(&addr.parse().unwrap()) .http1_preserve_header_case(true) .http1_title_case_headers(true) .serve(app.into_make_service()) } + + #[cfg(feature = "ipfsd")] + pub fn uds_server( + &self, + ) -> Server< + uds::ServerAccept, + axum::extract::connect_info::IntoMakeServiceWithConnectInfo, + > { + #[cfg(target_os = "android")] + let path = "/dev/socket/ipfsd.http".to_owned(); + + #[cfg(not(target_os = "android"))] + let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); + + let _ = std::fs::remove_file(&path); + let uds = UnixListener::bind(&path).unwrap(); + println!("Binding to UDS at {}", path); + let app = self.get_app(); + Server::builder(uds::ServerAccept { uds }) + .serve(app.into_make_service_with_connect_info::()) + } } #[tracing::instrument(skip(state))] @@ -690,7 +718,7 @@ mod tests { ) .await .unwrap(); - let server = handler.server(); + let server = handler.http_server(); let addr = server.local_addr(); let core_task = tokio::spawn(async move { server.await.unwrap(); diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index f9e1c0896e..35132d6246 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -13,3 +13,5 @@ pub mod metrics; mod response; mod rpc; mod templates; +#[cfg(feature = "ipfsd")] +mod uds; diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index d17588f885..ea366f878c 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -11,6 +11,7 @@ use iroh_gateway::{ metrics, }; use iroh_metrics::gateway::Metrics; +#[cfg(feature = "ipfsd")] use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; @@ -118,18 +119,28 @@ async fn main() -> Result<()> { iroh_metrics::MetricsHandle::from_registry_with_tracer(metrics_config, prom_registry) .await .expect("failed to initialize metrics"); - let server = handler.server(); - println!("listening on {}", server.local_addr()); + let server = handler.http_server(); + println!("HTTP endpoint listening on {}", server.local_addr()); let core_task = tokio::spawn(async move { server.await.unwrap(); }); + #[cfg(feature = "ipfsd")] + let uds_server_task = { + let uds_server = handler.uds_server(); + let task = tokio::spawn(async move { + uds_server.await.unwrap(); + }); + task + }; + iroh_util::block_until_sigint().await; #[cfg(feature = "ipfsd")] { store_rpc.abort(); p2p_rpc.abort(); + uds_server_task.abort(); } core_task.abort(); diff --git a/iroh-gateway/src/uds.rs b/iroh-gateway/src/uds.rs new file mode 100644 index 0000000000..1469a32cfe --- /dev/null +++ b/iroh-gateway/src/uds.rs @@ -0,0 +1,95 @@ +/// HTTP over UDS support +/// From https://github.com/tokio-rs/axum/blob/1fe45583626a4c9c890cc01131d38c57f8728686/examples/unix-domain-socket/src/main.rs +use axum::extract::connect_info; +use futures::ready; +use hyper::{ + client::connect::{Connected, Connection}, + server::accept::Accept, +}; +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::{unix::UCred, UnixListener, UnixStream}, +}; +use tower::BoxError; +pub struct ServerAccept { + pub uds: UnixListener, +} + +impl Accept for ServerAccept { + type Conn = UnixStream; + type Error = BoxError; + + fn poll_accept( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + let (stream, _addr) = ready!(self.uds.poll_accept(cx))?; + Poll::Ready(Some(Ok(stream))) + } +} + +struct ClientConnection { + stream: UnixStream, +} + +impl AsyncWrite for ClientConnection { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.stream).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + +impl AsyncRead for ClientConnection { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.stream).poll_read(cx, buf) + } +} + +impl Connection for ClientConnection { + fn connected(&self) -> Connected { + Connected::new() + } +} + +#[derive(Clone, Debug)] +#[allow(dead_code)] +pub struct UdsConnectInfo { + peer_addr: Arc, + peer_cred: UCred, +} + +impl connect_info::Connected<&UnixStream> for UdsConnectInfo { + fn connect_info(target: &UnixStream) -> Self { + let peer_addr = target.peer_addr().unwrap(); + let peer_cred = target.peer_cred().unwrap(); + + Self { + peer_addr: Arc::new(peer_addr), + peer_cred, + } + } +} From 0ef6b63eddc5e0d1142426aa38c624087db48d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 10:50:54 -0700 Subject: [PATCH 05/29] iroh-one: split the all-in-one ipfsd from iroh-gateway into its own crate --- Cargo.toml | 1 + iroh-gateway/Cargo.toml | 5 +- iroh-gateway/src/config.rs | 4 - iroh-gateway/src/core.rs | 46 +- iroh-gateway/src/lib.rs | 6 - iroh-gateway/src/main.rs | 38 +- iroh-one/Cargo.toml | 65 ++ iroh-one/README.md | 28 + iroh-one/src/client.rs | 150 +++++ iroh-one/src/config.rs | 287 ++++++++ iroh-one/src/constants.rs | 35 + iroh-one/src/core.rs | 687 ++++++++++++++++++++ iroh-one/src/error.rs | 24 + iroh-one/src/headers.rs | 313 +++++++++ iroh-one/src/lib.rs | 13 + iroh-one/src/main.rs | 122 ++++ {iroh-gateway => iroh-one}/src/mem_p2p.rs | 0 {iroh-gateway => iroh-one}/src/mem_store.rs | 0 iroh-one/src/metrics.rs | 9 + iroh-one/src/response.rs | 242 +++++++ iroh-one/src/rpc.rs | 24 + iroh-one/src/templates.rs | 19 + {iroh-gateway => iroh-one}/src/uds.rs | 0 iroh-rpc-client/src/config.rs | 24 - 24 files changed, 2031 insertions(+), 111 deletions(-) create mode 100644 iroh-one/Cargo.toml create mode 100644 iroh-one/README.md create mode 100644 iroh-one/src/client.rs create mode 100644 iroh-one/src/config.rs create mode 100644 iroh-one/src/constants.rs create mode 100644 iroh-one/src/core.rs create mode 100644 iroh-one/src/error.rs create mode 100644 iroh-one/src/headers.rs create mode 100644 iroh-one/src/lib.rs create mode 100644 iroh-one/src/main.rs rename {iroh-gateway => iroh-one}/src/mem_p2p.rs (100%) rename {iroh-gateway => iroh-one}/src/mem_store.rs (100%) create mode 100644 iroh-one/src/metrics.rs create mode 100644 iroh-one/src/response.rs create mode 100644 iroh-one/src/rpc.rs create mode 100644 iroh-one/src/templates.rs rename {iroh-gateway => iroh-one}/src/uds.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index a442b2dae6..f3d4c77322 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "iroh-rpc-types", "iroh-gateway", "iroh-metrics", + "iroh-one", "iroh-p2p", "iroh-resolver", "iroh-store", diff --git a/iroh-gateway/Cargo.toml b/iroh-gateway/Cargo.toml index 0383c99bd4..a902bb6517 100644 --- a/iroh-gateway/Cargo.toml +++ b/iroh-gateway/Cargo.toml @@ -10,8 +10,6 @@ repository = "https://github.com/dignifiedquire/iroh" [dependencies] iroh-rpc-client = { path = "../iroh-rpc-client", default-features = false } iroh-rpc-types = { path = "../iroh-rpc-types", default-features = false } -iroh-p2p = {path = "../iroh-p2p", default-features = false, features = ["rpc-mem"], optional = true} -iroh-store = {path = "../iroh-store", default-features = false, features = ["rpc-mem"], optional = true} cid = "0.8.6" @@ -65,7 +63,6 @@ axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messa [features] -default = ["rpc-mem", "rpc-grpc", "ipfsd"] -ipfsd = ["iroh-p2p/rpc-mem", "iroh-store/rpc-mem"] +default = ["rpc-grpc", "rpc-mem"] rpc-grpc = ["iroh-rpc-types/grpc", "iroh-rpc-client/grpc", "iroh-metrics/rpc-grpc"] rpc-mem = ["iroh-rpc-types/mem", "iroh-rpc-client/mem"] diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index ddd97ca1fb..6d26dd7fdb 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -128,11 +128,7 @@ fn default_headers() -> HeaderMap { impl Default for Config { fn default() -> Self { - #[cfg(not(feature = "ipfsd"))] let rpc_client = RpcClientConfig::default_grpc(); - #[cfg(feature = "ipfsd")] - let rpc_client = RpcClientConfig::default_ipfsd(); - let mut t = Self { writeable: false, fetch: false, diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 51e7fa8f1d..1469ce6266 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -6,7 +6,7 @@ use axum::{ http::{header::*, StatusCode}, response::IntoResponse, routing::get, - BoxError, Router, Server, + BoxError, Router, }; use bytes::Bytes; use futures::TryStreamExt; @@ -33,16 +33,12 @@ use std::{ time::{self, Duration}, }; use tokio::sync::RwLock; -#[cfg(feature = "ipfsd")] -use tokio::net::UnixListener; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; use tracing::info_span; use url::Url; use urlencoding::encode; -#[cfg(feature = "ipfsd")] -use crate::uds; use crate::{ bad_bits::BadBits, client::{Client, FileResult, Request}, @@ -126,9 +122,12 @@ impl Core { }) } - fn get_app(&self) -> Router { + pub fn server( + self, + ) -> axum::Server> + { // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 - Router::new() + let app = Router::new() .route("/:scheme/:cid", get(get_handler)) .route("/:scheme/:cid/*cpath", get(get_handler)) .route("/health", get(health_check)) @@ -152,42 +151,15 @@ impl Core { uri = %request.uri(), ) }), - ) - } - - pub fn http_server( - &self, - ) -> Server> { - let app = self.get_app(); + ); // todo(arqu): make configurable let addr = format!("0.0.0.0:{}", self.state.config.port); - Server::bind(&addr.parse().unwrap()) + axum::Server::bind(&addr.parse().unwrap()) .http1_preserve_header_case(true) .http1_title_case_headers(true) .serve(app.into_make_service()) } - - #[cfg(feature = "ipfsd")] - pub fn uds_server( - &self, - ) -> Server< - uds::ServerAccept, - axum::extract::connect_info::IntoMakeServiceWithConnectInfo, - > { - #[cfg(target_os = "android")] - let path = "/dev/socket/ipfsd.http".to_owned(); - - #[cfg(not(target_os = "android"))] - let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); - - let _ = std::fs::remove_file(&path); - let uds = UnixListener::bind(&path).unwrap(); - println!("Binding to UDS at {}", path); - let app = self.get_app(); - Server::builder(uds::ServerAccept { uds }) - .serve(app.into_make_service_with_connect_info::()) - } } #[tracing::instrument(skip(state))] @@ -718,7 +690,7 @@ mod tests { ) .await .unwrap(); - let server = handler.http_server(); + let server = handler.server(); let addr = server.local_addr(); let core_task = tokio::spawn(async move { server.await.unwrap(); diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index 35132d6246..5f7df4a8c1 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -5,13 +5,7 @@ mod constants; pub mod core; mod error; mod headers; -#[cfg(feature = "ipfsd")] -pub mod mem_p2p; -#[cfg(feature = "ipfsd")] -pub mod mem_store; pub mod metrics; mod response; mod rpc; mod templates; -#[cfg(feature = "ipfsd")] -mod uds; diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index ea366f878c..7116cfb294 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -11,8 +11,6 @@ use iroh_gateway::{ metrics, }; use iroh_metrics::gateway::Metrics; -#[cfg(feature = "ipfsd")] -use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; use tokio::sync::RwLock; @@ -76,21 +74,6 @@ async fn main() -> Result<()> { args.make_overrides_map(), ) .unwrap(); - - // When running in ipfsd mode, update the rpc client config to setup - // memory addresses for the p2p and store modules. - #[cfg(feature = "ipfsd")] - let (store_rpc, p2p_rpc) = { - let (store_recv, store_sender) = Addr::new_mem(); - config.rpc_client.store_addr = Some(store_sender); - let store_rpc = iroh_gateway::mem_store::start(store_recv).await?; - - let (p2p_recv, p2p_sender) = Addr::new_mem(); - config.rpc_client.p2p_addr = Some(p2p_sender); - let p2p_rpc = iroh_gateway::mem_p2p::start(p2p_recv).await?; - (store_rpc, p2p_rpc) - }; - config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); println!("{:#?}", config); @@ -119,30 +102,13 @@ async fn main() -> Result<()> { iroh_metrics::MetricsHandle::from_registry_with_tracer(metrics_config, prom_registry) .await .expect("failed to initialize metrics"); - let server = handler.http_server(); - println!("HTTP endpoint listening on {}", server.local_addr()); + let server = handler.server(); + println!("listening on {}", server.local_addr()); let core_task = tokio::spawn(async move { server.await.unwrap(); }); - #[cfg(feature = "ipfsd")] - let uds_server_task = { - let uds_server = handler.uds_server(); - let task = tokio::spawn(async move { - uds_server.await.unwrap(); - }); - task - }; - iroh_util::block_until_sigint().await; - - #[cfg(feature = "ipfsd")] - { - store_rpc.abort(); - p2p_rpc.abort(); - uds_server_task.abort(); - } - core_task.abort(); metrics_handle.shutdown(); diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml new file mode 100644 index 0000000000..cf1b7fe9e5 --- /dev/null +++ b/iroh-one/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "iroh-one" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0/MIT" +readme = "README.md" +description = "Single binary IPFS gateway" +repository = "https://github.com/dignifiedquire/iroh" + +[dependencies] +iroh-rpc-client = { path = "../iroh-rpc-client", default-features = false } +iroh-rpc-types = { path = "../iroh-rpc-types", default-features = false } +iroh-p2p = {path = "../iroh-p2p", default-features = false, features = ["rpc-mem"]} +iroh-store = {path = "../iroh-store", default-features = false, features = ["rpc-mem"]} + +cid = "0.8.4" + +tokio = { version = "1", features = ["macros", "rt-multi-thread", "process"] } +axum = "0.5.1" +clap = { version = "3.1.14", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.78" +serde_qs = "0.10.1" +tower = { version = "0.4", features = ["util", "timeout", "load-shed", "limit"] } +mime_guess = "2.0.4" +iroh-metrics = { path = "../iroh-metrics", default-features = false } +tracing = "0.1.33" +names = { version = "0.14.0", default-features = false } +git-version = "0.3.5" +rand = "0.8.5" +tracing-opentelemetry = "0.17.2" +opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } +time = "0.3.9" +headers = "0.3.7" +hyper = "0.14.19" +libp2p = { version = "0.47", default-features = false } +iroh-util = { path = "../iroh-util" } +anyhow = "1" +futures = "0.3.21" +tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } +iroh-resolver = { path = "../iroh-resolver" } +prometheus-client = "0.17.0" +tokio-util = { version = "0.7", features = ["io"] } +bytes = "1.1.0" +tower-layer = { version = "0.3" } +tower-http = { version = "0.3", features = ["trace"] } +http = "0.2" +async-recursion = "1.0.0" +handlebars = "4" +url = "2.2.2" +urlencoding = "2.1.0" +dirs = "4.0.0" +toml = "0.5.9" +http-serde = "1.1.0" +config = "0.13.1" +async-trait = "0.1.56" + +[dev-dependencies] +axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers + + +[features] +default = ["rpc-mem", "rpc-grpc"] +rpc-grpc = ["iroh-rpc-types/grpc", "iroh-rpc-client/grpc", "iroh-metrics/rpc-grpc"] +rpc-mem = ["iroh-rpc-types/mem", "iroh-rpc-client/mem"] diff --git a/iroh-one/README.md b/iroh-one/README.md new file mode 100644 index 0000000000..4daf84c79d --- /dev/null +++ b/iroh-one/README.md @@ -0,0 +1,28 @@ +# Iroh Gateway + +A rust implementation of an IPFS gateway. + +## Running / Building + +`cargo run -- -p 10000` + +### Options + +- Run with `cargo run -- -h` for details +- `-wcf` Writeable, Cache, Fetch (options to toggle write enable, caching mechanics and fetching from the network); currently exists but is not implemented +- `-p` Port the gateway should listen on + +## ENV Variables + +- `IROH_INSTANCE_ID` - unique instance identifier, preferably some name than hard id (default: generated lower & snake case name) +- `IROH_ENV` - indicates the service environment (default: `dev`) + +## Endpoints + +| Endpoint | Flag | Description | Default | +|-----------------------------------|--------------------------------------------|-----------------------------------------------------------------------------------------|-------------| +| `/ipfs/:cid` & `/ipfs/:cid/:path` | `?format={"", "fs", "raw", "car"}` | Specifies the serving format & content-type | `""/fs` | +| | `?filename=DESIRED_FILE_NAME` | Specifies a filename for the attachment | `{cid}.bin` | +| | `?download={true, false}` | Sets content-disposition to attachment, browser prompts to save file instead of loading | `false` | +| | `?force_dir={true, false}` | Lists unixFS directories even if they contain an `index.html` file | `false` | +| | `?uri=ENCODED_URL` | Query parameter to handle navigator.registerProtocolHandler Web API ie. ipfs:// | `""` | \ No newline at end of file diff --git a/iroh-one/src/client.rs b/iroh-one/src/client.rs new file mode 100644 index 0000000000..64ed87a4ce --- /dev/null +++ b/iroh-one/src/client.rs @@ -0,0 +1,150 @@ +use std::sync::Arc; + +use axum::body::StreamBody; +use futures::StreamExt; +use iroh_metrics::gateway::Metrics; +use iroh_resolver::resolver::CidOrDomain; +use iroh_resolver::resolver::Metadata; +use iroh_resolver::resolver::OutMetrics; +use iroh_resolver::resolver::OutPrettyReader; +use iroh_resolver::resolver::Resolver; +use iroh_resolver::resolver::Source; +use prometheus_client::registry::Registry; +use tokio::io::AsyncReadExt; +use tokio_util::io::ReaderStream; +use tracing::info; +use tracing::warn; + +use crate::core::GetParams; +use crate::response::ResponseFormat; + +#[derive(Debug, Clone)] +pub struct Client { + resolver: Arc>, +} + +pub type PrettyStreamBody = StreamBody>>; + +impl Client { + pub fn new(rpc_client: &iroh_rpc_client::Client, registry: &mut Registry) -> Self { + Self { + resolver: Arc::new(Resolver::new(rpc_client.clone(), registry)), + } + } + + #[tracing::instrument(skip(self, rpc_client, metrics))] + pub async fn get_file( + &self, + path: iroh_resolver::resolver::Path, + rpc_client: &iroh_rpc_client::Client, + start_time: std::time::Instant, + metrics: &Metrics, + ) -> Result<(PrettyStreamBody, Metadata), String> { + info!("get file {}", path); + let res = self + .resolver + .resolve(path) + .await + .map_err(|e| e.to_string())?; + metrics + .ttf_block + .set(start_time.elapsed().as_millis() as u64); + let metadata = res.metadata().clone(); + if metadata.source == Source::Bitswap { + metrics + .hist_ttfb + .observe(start_time.elapsed().as_millis() as f64); + } else { + metrics + .hist_ttfb_cached + .observe(start_time.elapsed().as_millis() as f64); + } + let reader = res + .pretty( + rpc_client.clone(), + OutMetrics { + metrics: metrics.clone(), + start: start_time, + }, + ) + .map_err(|e| e.to_string())?; + let stream = ReaderStream::new(reader); + let body = StreamBody::new(stream); + + Ok((body, metadata)) + } + + #[tracing::instrument(skip(self, rpc_client, metrics))] + pub async fn get_file_recursive( + self, + path: iroh_resolver::resolver::Path, + rpc_client: iroh_rpc_client::Client, + start_time: std::time::Instant, + metrics: Metrics, + ) -> Result { + info!("get file {}", path); + let (mut sender, body) = axum::body::Body::channel(); + + tokio::spawn(async move { + let res = self.resolver.resolve_recursive(path); + tokio::pin!(res); + + while let Some(res) = res.next().await { + match res { + Ok(res) => { + metrics + .ttf_block + .set(start_time.elapsed().as_millis() as u64); + let metadata = res.metadata().clone(); + if metadata.source == Source::Bitswap { + metrics + .hist_ttfb + .observe(start_time.elapsed().as_millis() as f64); + } else { + metrics + .hist_ttfb_cached + .observe(start_time.elapsed().as_millis() as f64); + } + let reader = res.pretty( + rpc_client.clone(), + OutMetrics { + metrics: metrics.clone(), + start: start_time, + }, + ); + match reader { + Ok(mut reader) => { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await.unwrap(); + sender.send_data(bytes.into()).await.unwrap(); + } + Err(e) => { + warn!("failed to load recursively: {:?}", e); + sender.abort(); + break; + } + } + } + Err(e) => { + warn!("failed to load recursively: {:?}", e); + sender.abort(); + break; + } + } + } + }); + + Ok(body) + } +} + +#[derive(Debug, Clone)] +pub struct Request { + pub format: ResponseFormat, + pub cid: CidOrDomain, + pub resolved_path: iroh_resolver::resolver::Path, + pub query_file_name: String, + pub content_path: String, + pub download: bool, + pub query_params: GetParams, +} diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs new file mode 100644 index 0000000000..aac2d2aa65 --- /dev/null +++ b/iroh-one/src/config.rs @@ -0,0 +1,287 @@ +use crate::constants::*; +use anyhow::{bail, Result}; +use axum::http::{header::*, Method}; +use config::{ConfigError, Map, Source, Value}; +use headers::{ + AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlAllowOrigin, HeaderMapExt, +}; +use iroh_metrics::config::Config as MetricsConfig; +use iroh_rpc_client::Config as RpcClientConfig; +use iroh_rpc_types::{gateway::GatewayServerAddr, Addr}; +use iroh_util::insert_into_config_map; +use serde::{Deserialize, Serialize}; + +/// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory +pub const CONFIG_FILE_NAME: &str = "gateway.config.toml"; +/// ENV_PREFIX should be used along side the config field name to set a config field using +/// environment variables +/// For example, `IROH_GATEWAY_PORT=1000` would set the value of the `Config.port` field +pub const ENV_PREFIX: &str = "IROH_GATEWAY"; +pub const DEFAULT_PORT: u16 = 9050; + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Config { + /// flag to toggle whether the gateway allows writing/pushing data + pub writeable: bool, + /// flag to toggle whether the gateway allows fetching data from other nodes or is local only + pub fetch: bool, + /// flag to toggle whether the gateway enables/utilizes caching + pub cache: bool, + /// default port to listen on + pub port: u16, + // NOTE: for toml to serialize properly, the "table" values must be serialized at the end, and + // so much come at the end of the `Config` struct + /// set of user provided headers to attach to all responses + #[serde(with = "http_serde::header_map")] + pub headers: HeaderMap, + /// rpc addresses for the gateway & addresses for the rpc client to dial + pub rpc_client: RpcClientConfig, + /// metrics configuration + pub metrics: MetricsConfig, +} + +impl Config { + pub fn new( + writeable: bool, + fetch: bool, + cache: bool, + port: u16, + rpc_client: RpcClientConfig, + ) -> Self { + Self { + writeable, + fetch, + cache, + headers: HeaderMap::new(), + port, + rpc_client, + metrics: MetricsConfig::default(), + } + } + + pub fn set_default_headers(&mut self) { + self.headers = default_headers(); + } + + /// Derive server addr for non memory addrs. + pub fn server_rpc_addr(&self) -> Result> { + self.rpc_client + .gateway_addr + .as_ref() + .map(|addr| { + #[allow(unreachable_patterns)] + match addr { + #[cfg(feature = "rpc-grpc")] + Addr::GrpcHttp2(addr) => Ok(Addr::GrpcHttp2(*addr)), + #[cfg(all(feature = "rpc-grpc", unix))] + Addr::GrpcUds(path) => Ok(Addr::GrpcUds(path.clone())), + #[cfg(feature = "rpc-mem")] + Addr::Mem(_) => bail!("can not derive rpc_addr for mem addr"), + _ => bail!("invalid rpc_addr"), + } + }) + .transpose() + } + + // When running in ipfsd mode, the resolver will use memory channels to + // communicate with the p2p and store modules. + // The gateway itself is exposing a UDS rpc endpoint to be also usable + // as a single entry point for other system services. + pub fn default_ipfsd() -> RpcClientConfig { + let path = { + #[cfg(target_os = "android")] + "/dev/socket/ipfsd".into(); + + #[cfg(not(target_os = "android"))] + { + let path = format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()); + path.into() + } + }; + + RpcClientConfig { + gateway_addr: Some(Addr::GrpcUds(path)), + p2p_addr: None, + store_addr: None, + } + } +} + +fn default_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.typed_insert(AccessControlAllowOrigin::ANY); + headers.typed_insert( + [ + Method::GET, + Method::PUT, + Method::POST, + Method::DELETE, + Method::HEAD, + Method::OPTIONS, + ] + .into_iter() + .collect::(), + ); + headers.typed_insert( + [ + CONTENT_TYPE, + CONTENT_DISPOSITION, + LAST_MODIFIED, + CACHE_CONTROL, + ACCEPT_RANGES, + ETAG, + HEADER_SERVICE_WORKER.clone(), + HEADER_X_IPFS_GATEWAY_PREFIX.clone(), + HEADER_X_TRACE_ID.clone(), + HEADER_X_CONTENT_TYPE_OPTIONS.clone(), + HEADER_X_IPFS_PATH.clone(), + HEADER_X_IPFS_ROOTS.clone(), + ] + .into_iter() + .collect::(), + ); + // todo(arqu): remove these once propperly implmented + headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); + headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); + headers +} + +impl Default for Config { + fn default() -> Self { + let rpc_client = Self::default_ipfsd(); + + let mut t = Self { + writeable: false, + fetch: false, + cache: false, + headers: HeaderMap::new(), + port: DEFAULT_PORT, + rpc_client, + metrics: MetricsConfig::default(), + }; + t.set_default_headers(); + t + } +} + +impl Source for Config { + fn clone_into_box(&self) -> Box { + Box::new(self.clone()) + } + + fn collect(&self) -> Result, ConfigError> { + let rpc_client = self.rpc_client.collect()?; + let mut map: Map = Map::new(); + insert_into_config_map(&mut map, "writeable", self.writeable); + insert_into_config_map(&mut map, "fetch", self.fetch); + insert_into_config_map(&mut map, "cache", self.cache); + // Some issue between deserializing u64 & u16, converting this to + // an signed int fixes the issue + insert_into_config_map(&mut map, "port", self.port as i32); + insert_into_config_map(&mut map, "headers", collect_headers(&self.headers)?); + insert_into_config_map(&mut map, "rpc_client", rpc_client); + let metrics = self.metrics.collect()?; + insert_into_config_map(&mut map, "metrics", metrics); + Ok(map) + } +} + +fn collect_headers(headers: &HeaderMap) -> Result, ConfigError> { + let mut map = Map::new(); + for (key, value) in headers.iter() { + insert_into_config_map( + &mut map, + key.as_str(), + value.to_str().map_err(|e| ConfigError::Foreign(e.into()))?, + ); + } + Ok(map) +} + +#[cfg(test)] +mod tests { + use super::*; + use config::Config as ConfigBuilder; + + #[test] + fn test_default_headers() { + let headers = default_headers(); + assert_eq!(headers.len(), 5); + let h = headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(); + assert_eq!(h, "*"); + } + + #[test] + fn default_config() { + let config = Config::default(); + assert!(!config.writeable); + assert!(!config.fetch); + assert!(!config.cache); + assert_eq!(config.port, DEFAULT_PORT); + } + + #[test] + fn test_collect() { + let default = Config::default(); + let mut expect: Map = Map::new(); + expect.insert("writeable".to_string(), Value::new(None, default.writeable)); + expect.insert("fetch".to_string(), Value::new(None, default.fetch)); + expect.insert("cache".to_string(), Value::new(None, default.cache)); + expect.insert("port".to_string(), Value::new(None, default.port as i64)); + expect.insert( + "headers".to_string(), + Value::new(None, collect_headers(&default.headers).unwrap()), + ); + expect.insert( + "rpc_client".to_string(), + Value::new(None, default.rpc_client.collect().unwrap()), + ); + expect.insert( + "metrics".to_string(), + Value::new(None, default.metrics.collect().unwrap()), + ); + + let got = default.collect().unwrap(); + for key in got.keys() { + let left = expect.get(key).unwrap_or_else(|| panic!("{}", key)); + let right = got.get(key).unwrap(); + assert_eq!(left, right); + } + } + + #[test] + fn test_collect_headers() { + let mut expect = Map::new(); + expect.insert( + "access-control-allow-origin".to_string(), + Value::new(None, "*"), + ); + expect.insert( + "access-control-allow-methods".to_string(), + Value::new(None, "GET, PUT, POST, DELETE, HEAD, OPTIONS"), + ); + expect.insert("access-control-allow-headers".to_string(), Value::new(None, "content-type, content-disposition, last-modified, cache-control, accept-ranges, etag, service-worker, x-ipfs-gateway-prefix, x-trace-id, x-content-type-options, x-ipfs-path, x-ipfs-roots")); + expect.insert( + "cache-control".to_string(), + Value::new(None, "no-cache, no-transform"), + ); + expect.insert("accept-ranges".to_string(), Value::new(None, "none")); + let got = collect_headers(&default_headers()).unwrap(); + assert_eq!(expect, got); + } + + #[test] + fn test_build_config_from_struct() { + let mut expect = Config::default(); + expect.set_default_headers(); + let source = expect.clone(); + let got: Config = ConfigBuilder::builder() + .add_source(source) + .build() + .unwrap() + .try_deserialize() + .unwrap(); + + assert_eq!(expect, got); + } +} diff --git a/iroh-one/src/constants.rs b/iroh-one/src/constants.rs new file mode 100644 index 0000000000..0aad73854b --- /dev/null +++ b/iroh-one/src/constants.rs @@ -0,0 +1,35 @@ +use axum::http::{header::HeaderName, HeaderValue}; + +// Headers +pub static HEADER_X_IPFS_PATH: HeaderName = HeaderName::from_static("x-ipfs-path"); +pub static HEADER_X_CONTENT_TYPE_OPTIONS: HeaderName = + HeaderName::from_static("x-content-type-options"); +pub static HEADER_X_TRACE_ID: HeaderName = HeaderName::from_static("x-trace-id"); +pub static HEADER_X_IPFS_GATEWAY_PREFIX: HeaderName = + HeaderName::from_static("x-ipfs-gateway-prefix"); +pub static HEADER_X_IPFS_ROOTS: HeaderName = HeaderName::from_static("x-ipfs-roots"); +pub static HEADER_SERVICE_WORKER: HeaderName = HeaderName::from_static("service-worker"); + +// Common Header Values +pub static VALUE_XCTO_NOSNIFF: HeaderValue = HeaderValue::from_static("nosniff"); +pub static VALUE_NONE: HeaderValue = HeaderValue::from_static("none"); +pub static VALUE_NO_CACHE_NO_TRANSFORM: HeaderValue = + HeaderValue::from_static("no-cache, no-transform"); +pub static VAL_IMMUTABLE_MAX_AGE: HeaderValue = + HeaderValue::from_static("public, max-age=31536000, immutable"); + +// Dispositions +pub static DISPOSITION_ATTACHMENT: &str = "attachment"; +pub static DISPOSITION_INLINE: &str = "inline"; + +// Content Types +pub static CONTENT_TYPE_IPLD_RAW: HeaderValue = + HeaderValue::from_static("application/vnd.ipld.raw"); +pub static CONTENT_TYPE_IPLD_CAR: HeaderValue = + HeaderValue::from_static("application/vnd.ipld.car; version=1"); +pub static CONTENT_TYPE_OCTET_STREAM: HeaderValue = + HeaderValue::from_static("application/octet-stream"); + +// Schemes +pub static SCHEME_IPFS: &str = "ipfs"; +pub static SCHEME_IPNS: &str = "ipns"; diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs new file mode 100644 index 0000000000..2bebd27913 --- /dev/null +++ b/iroh-one/src/core.rs @@ -0,0 +1,687 @@ +use async_recursion::async_recursion; +use axum::{ + body::{self, Body, HttpBody}, + error_handling::HandleErrorLayer, + extract::{Extension, Path, Query}, + http::{header::*, StatusCode}, + response::IntoResponse, + routing::get, + BoxError, Router, Server, +}; +use bytes::Bytes; +use handlebars::Handlebars; +use iroh_metrics::{gateway::Metrics, get_current_trace_id}; +use iroh_resolver::resolver::{CidOrDomain, UnixfsType}; +use iroh_rpc_client::Client as RpcClient; +use iroh_rpc_types::gateway::GatewayServerAddr; +use prometheus_client::registry::Registry; +use serde::{Deserialize, Serialize}; +use serde_json::{ + json, + value::{Map, Value as Json}, +}; +use serde_qs; +use std::{ + collections::HashMap, + error::Error, + fmt::Write, + sync::Arc, + time::{self, Duration}, +}; +use tokio::net::UnixListener; +use tower::ServiceBuilder; +use tower_http::trace::TraceLayer; +use tracing::info_span; +use url::Url; +use urlencoding::encode; +use crate::{ + client::{Client, Request}, + config::Config, + constants::*, + error::GatewayError, + headers::*, + response::{get_response_format, GatewayResponse, ResponseFormat}, + rpc, + rpc::Gateway, + templates, + uds, +}; + +#[derive(Debug)] +pub struct Core { + state: Arc, +} + +#[derive(Debug)] +pub struct State { + config: Config, + client: Client, + rpc_client: iroh_rpc_client::Client, + handlebars: HashMap, + pub metrics: Metrics, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct GetParams { + // todo(arqu): swap this for ResponseFormat + /// specifies the expected format of the response + format: Option, + /// specifies the desired filename of the response + filename: Option, + /// specifies whether the response should be of disposition inline or attachment + download: Option, + /// specifies whether the response should render a directory even if index.html is present + force_dir: Option, + /// uri query parameter for handling navigator.registerProtocolHandler Web API requests + uri: Option, + recursive: Option, +} + +impl GetParams { + pub fn to_query_string(&self) -> String { + let q = serde_qs::to_string(self).unwrap(); + if q.is_empty() { + q + } else { + format!("?{}", q) + } + } +} + +impl Core { + pub async fn new( + config: Config, + rpc_addr: GatewayServerAddr, + metrics: Metrics, + registry: &mut Registry, + ) -> anyhow::Result { + tokio::spawn(async move { + // TODO: handle error + rpc::new(rpc_addr, Gateway::default()).await + }); + let rpc_client = RpcClient::new(config.rpc_client.clone()).await?; + let mut templates = HashMap::new(); + templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); + templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); + let client = Client::new(&rpc_client, registry); + + Ok(Self { + state: Arc::new(State { + config, + client, + rpc_client, + metrics, + handlebars: templates, + }), + }) + } + + fn get_app(&self) -> Router { + // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 + Router::new() + .route("/:scheme/:cid", get(get_handler)) + .route("/:scheme/:cid/*cpath", get(get_handler)) + .route("/health", get(health_check)) + .layer(Extension(Arc::clone(&self.state))) + .layer( + ServiceBuilder::new() + // Handle errors from middleware + .layer(Extension(Arc::clone(&self.state))) + .layer(HandleErrorLayer::new(middleware_error_handler)) + .load_shed() + .concurrency_limit(2048) + .timeout(Duration::from_secs(60)) + .into_inner(), + ) + .layer( + // Tracing span for each request + TraceLayer::new_for_http().make_span_with(|request: &http::Request| { + info_span!( + "request", + method = %request.method(), + uri = %request.uri(), + ) + }), + ) + } + + pub fn http_server( + &self, + ) -> Server> { + let app = self.get_app(); + // todo(arqu): make configurable + let addr = format!("0.0.0.0:{}", self.state.config.port); + + Server::bind(&addr.parse().unwrap()) + .http1_preserve_header_case(true) + .http1_title_case_headers(true) + .serve(app.into_make_service()) + } + + pub fn uds_server( + &self, + ) -> Server< + uds::ServerAccept, + axum::extract::connect_info::IntoMakeServiceWithConnectInfo, + > { + #[cfg(target_os = "android")] + let path = "/dev/socket/ipfsd.http".to_owned(); + + #[cfg(not(target_os = "android"))] + let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); + + let _ = std::fs::remove_file(&path); + let uds = UnixListener::bind(&path).unwrap(); + println!("Binding to UDS at {}", path); + let app = self.get_app(); + Server::builder(uds::ServerAccept { uds }) + .serve(app.into_make_service_with_connect_info::()) + } +} + +#[tracing::instrument(skip(state))] +async fn get_handler( + Extension(state): Extension>, + Path(params): Path>, + Query(query_params): Query, + request_headers: HeaderMap, +) -> Result { + state.metrics.requests_total.inc(); + let start_time = time::Instant::now(); + // parse path params + let scheme = params.get("scheme").unwrap(); + if scheme != SCHEME_IPFS && scheme != SCHEME_IPNS { + return Err(error( + StatusCode::BAD_REQUEST, + "invalid scheme, must be ipfs or ipns", + &state, + )); + } + let cid = params.get("cid").unwrap(); + let cpath = "".to_string(); + let cpath = params.get("cpath").unwrap_or(&cpath); + let query_params_copy = query_params.clone(); + + let uri_param = query_params.uri.clone().unwrap_or_default(); + if !uri_param.is_empty() { + return protocol_handler_redirect(uri_param, &state); + } + service_worker_check(&request_headers, cpath.to_string(), &state)?; + unsuported_header_check(&request_headers, &state)?; + + let full_content_path = format!("/{}/{}{}", scheme, cid, cpath); + let resolved_path: iroh_resolver::resolver::Path = full_content_path + .parse() + .map_err(|e: anyhow::Error| e.to_string()) + .map_err(|e| error(StatusCode::BAD_REQUEST, &e, &state))?; + let resolved_cid = resolved_path.root(); + + // parse query params + let format = match get_response_format(&request_headers, query_params.format) { + Ok(format) => format, + Err(err) => { + return Err(error(StatusCode::BAD_REQUEST, &err, &state)); + } + }; + + let query_file_name = query_params.filename.unwrap_or_default(); + let download = query_params.download.unwrap_or_default(); + let recursive = query_params.recursive.unwrap_or_default(); + + let mut headers = HeaderMap::new(); + + if let Some(resp) = etag_check(&request_headers, resolved_cid, &format, &state) { + return Ok(resp); + } + + // init headers + format.write_headers(&mut headers); + add_user_headers(&mut headers, state.config.headers.clone()); + headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str(&full_content_path).unwrap(), + ); + + // handle request and fetch data + let req = Request { + format, + cid: resolved_path.root().clone(), + resolved_path, + query_file_name, + content_path: full_content_path.to_string(), + download, + query_params: query_params_copy, + }; + + if recursive { + serve_car_recursive(&req, state, headers, start_time).await + } else { + match req.format { + ResponseFormat::Raw => serve_raw(&req, state, headers, start_time).await, + ResponseFormat::Car => serve_car(&req, state, headers, start_time).await, + ResponseFormat::Fs(_) => serve_fs(&req, state, headers, start_time).await, + } + } +} + +#[tracing::instrument()] +async fn health_check() -> String { + "OK".to_string() +} + +#[tracing::instrument()] +fn protocol_handler_redirect( + uri_param: String, + state: &State, +) -> Result { + let u = match Url::parse(&uri_param) { + Ok(u) => u, + Err(e) => { + return Err(error( + StatusCode::BAD_REQUEST, + &format!("invalid uri: {}", e), + state, + )); + } + }; + let uri_scheme = u.scheme(); + if uri_scheme != SCHEME_IPFS && uri_scheme != SCHEME_IPNS { + return Err(error( + StatusCode::BAD_REQUEST, + "invalid uri scheme, must be ipfs or ipns", + state, + )); + } + let mut uri_path = u.path().to_string(); + let uri_query = u.query(); + if uri_query.is_some() { + let encoded_query = encode(uri_query.unwrap()); + write!(uri_path, "?{}", encoded_query) + .map_err(|e| error(StatusCode::BAD_REQUEST, &e.to_string(), state))?; + } + let uri_host = u.host().unwrap().to_string(); + let redirect_uri = format!("{}://{}{}", uri_scheme, uri_host, uri_path); + Ok(GatewayResponse::redirect_permanently(&redirect_uri)) +} + +#[tracing::instrument()] +fn service_worker_check( + request_headers: &HeaderMap, + cpath: String, + state: &State, +) -> Result<(), GatewayError> { + if request_headers.contains_key(&HEADER_SERVICE_WORKER) { + let sw = request_headers.get(&HEADER_SERVICE_WORKER).unwrap(); + if sw.to_str().unwrap() == "script" && cpath.is_empty() { + return Err(error( + StatusCode::BAD_REQUEST, + "Service Worker not supported", + state, + )); + } + } + Ok(()) +} + +#[tracing::instrument()] +fn unsuported_header_check(request_headers: &HeaderMap, state: &State) -> Result<(), GatewayError> { + if request_headers.contains_key(&HEADER_X_IPFS_GATEWAY_PREFIX) { + return Err(error( + StatusCode::BAD_REQUEST, + "Unsupported HTTP header", + state, + )); + } + Ok(()) +} + +#[tracing::instrument()] +fn etag_check( + request_headers: &HeaderMap, + resolved_cid: &CidOrDomain, + format: &ResponseFormat, + state: &State, +) -> Option { + if request_headers.contains_key("If-None-Match") { + // todo(arqu): handle dir etags + let cid_etag = get_etag(resolved_cid, Some(format.clone())); + let inm = request_headers + .get("If-None-Match") + .unwrap() + .to_str() + .unwrap(); + if etag_matches(inm, &cid_etag) { + return Some(GatewayResponse::not_modified()); + } + } + None +} + +#[tracing::instrument()] +async fn serve_raw( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: we currently only retrieve full cids + let (body, metadata) = state + .client + .get_file( + req.resolved_path.clone(), + &state.rpc_client, + start_time, + &state.metrics, + ) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + set_content_disposition_headers( + &mut headers, + format!("{}.bin", req.cid).as_str(), + DISPOSITION_ATTACHMENT, + ); + set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); + add_cache_control_headers(&mut headers, metadata.clone()); + add_ipfs_roots_headers(&mut headers, metadata); + response(StatusCode::OK, body, headers) +} + +#[tracing::instrument()] +async fn serve_car( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: we currently only retrieve full cids + let (body, metadata) = state + .client + .get_file( + req.resolved_path.clone(), + &state.rpc_client, + start_time, + &state.metrics, + ) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + set_content_disposition_headers( + &mut headers, + format!("{}.car", req.cid).as_str(), + DISPOSITION_ATTACHMENT, + ); + + // todo(arqu): this should be root cid + let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); + set_etag_headers(&mut headers, etag); + // todo(arqu): check if etag matches for root cid + add_ipfs_roots_headers(&mut headers, metadata); + response(StatusCode::OK, body, headers) +} + +#[tracing::instrument()] +async fn serve_car_recursive( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: actually package as car file + + let body = state + .client + .clone() + .get_file_recursive( + req.resolved_path.clone(), + state.rpc_client.clone(), + start_time, + state.metrics.clone(), + ) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + set_content_disposition_headers( + &mut headers, + format!("{}.car", req.cid).as_str(), + DISPOSITION_ATTACHMENT, + ); + + // todo(arqu): this should be root cid + let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); + set_etag_headers(&mut headers, etag); + // todo(arqu): check if etag matches for root cid + // add_ipfs_roots_headers(&mut headers, metadata); + response(StatusCode::OK, body, headers) +} + +#[tracing::instrument()] +#[async_recursion] +async fn serve_fs( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: we currently only retrieve full cids + let (mut body, metadata) = state + .client + .get_file( + req.resolved_path.clone(), + &state.rpc_client, + start_time, + &state.metrics, + ) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + add_ipfs_roots_headers(&mut headers, metadata.clone()); + match metadata.unixfs_type { + Some(UnixfsType::Dir) => { + if let Some(dir_list_data) = body.data().await { + let dir_list = match dir_list_data { + Ok(b) => b, + Err(_) => { + return Err(error( + StatusCode::INTERNAL_SERVER_ERROR, + "failed to read dir listing", + &state, + )); + } + }; + return serve_fs_dir(&dir_list, req, state, headers, start_time).await; + } else { + return Err(error( + StatusCode::INTERNAL_SERVER_ERROR, + "failed to read dir listing", + &state, + )); + } + } + Some(_) => { + // todo(arqu): error on no size + // todo(arqu): add lazy seeking + add_cache_control_headers(&mut headers, metadata.clone()); + set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); + let name = add_content_disposition_headers( + &mut headers, + &req.query_file_name, + &req.content_path, + req.download, + ); + if metadata.unixfs_type == Some(UnixfsType::Symlink) { + headers.insert( + CONTENT_TYPE, + HeaderValue::from_str("inode/symlink").unwrap(), + ); + } else { + add_content_type_headers(&mut headers, &name); + } + } + None => { + return Err(error( + StatusCode::BAD_REQUEST, + "couldn't determine unixfs type", + &state, + )); + } + } + response(StatusCode::OK, body, headers) +} + +#[tracing::instrument()] +async fn serve_fs_dir( + dir_list: &Bytes, + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + let dir_list = std::str::from_utf8(&dir_list[..]).unwrap(); + let force_dir = req.query_params.force_dir.unwrap_or(false); + let has_index = dir_list.lines().any(|l| l.starts_with("index.html")); + if !force_dir && has_index { + if !req.content_path.ends_with('/') { + let redirect_path = format!( + "{}/{}", + req.content_path, + req.query_params.to_query_string() + ); + return Ok(GatewayResponse::redirect(&redirect_path)); + } + let mut new_req = req.clone(); + new_req.resolved_path.push("index.html"); + new_req.content_path = format!("{}/index.html", req.content_path); + return serve_fs(&new_req, state, headers, start_time).await; + } + + headers.insert(CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap()); + // todo(arqu): set etag + // set_etag_headers(&mut headers, metadata.dir_hash.clone()); + + let mut template_data: Map = Map::new(); + let mut root_path = req.content_path.clone(); + if !root_path.ends_with('/') { + root_path.push('/'); + } + let links = dir_list + .lines() + .map(|line| { + let mut link = Map::new(); + link.insert("name".to_string(), Json::String(get_filename(line))); + link.insert( + "path".to_string(), + Json::String(format!("{}{}", root_path, line)), + ); + link + }) + .collect::>>(); + template_data.insert("links".to_string(), json!(links)); + let reg = Handlebars::new(); + let dir_template = state.handlebars.get("dir_list").unwrap(); + let res = reg.render_template(dir_template, &template_data).unwrap(); + response(StatusCode::OK, Body::from(res), headers) +} + +#[tracing::instrument(skip(body))] +fn response( + status_code: StatusCode, + body: B, + headers: HeaderMap, +) -> Result +where + B: 'static + HttpBody + Send, + ::Error: Into>, +{ + Ok(GatewayResponse { + status_code, + body: body::boxed(body), + headers, + trace_id: get_current_trace_id().to_string(), + }) +} + +#[tracing::instrument()] +fn error(status_code: StatusCode, message: &str, state: &State) -> GatewayError { + state.metrics.error_count.inc(); + GatewayError { + status_code, + message: message.to_string(), + trace_id: get_current_trace_id().to_string(), + } +} + +#[tracing::instrument()] +async fn middleware_error_handler( + Extension(state): Extension>, + err: BoxError, +) -> impl IntoResponse { + state.metrics.fail_count.inc(); + if err.is::() { + return error(StatusCode::REQUEST_TIMEOUT, "request timed out", &state); + } + + if err.is::() { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "service is overloaded, try again later", + &state, + ); + } + + return error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("unhandled internal error: {}", err).as_str(), + &state, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use iroh_rpc_client::Config as RpcClientConfig; + use prometheus_client::registry::Registry; + + #[tokio::test] + async fn gateway_health() { + let mut config = Config::new( + false, + false, + false, + 0, + RpcClientConfig { + gateway_addr: None, + p2p_addr: None, + store_addr: None, + }, + ); + config.set_default_headers(); + + let mut prom_registry = Registry::default(); + let gw_metrics = Metrics::new(&mut prom_registry); + let rpc_addr = "grpc://0.0.0.0:0".parse().unwrap(); + let handler = Core::new(config, rpc_addr, gw_metrics, &mut prom_registry) + .await + .unwrap(); + let server = handler.http_server(); + let addr = server.local_addr(); + let core_task = tokio::spawn(async move { + server.await.unwrap(); + }); + + let uri = hyper::Uri::builder() + .scheme("http") + .authority(format!("localhost:{}", addr.port())) + .path_and_query("/health") + .build() + .unwrap(); + let client = hyper::Client::new(); + let res = client.get(uri).await.unwrap(); + + assert_eq!(StatusCode::OK, res.status()); + let body = hyper::body::to_bytes(res.into_body()).await.unwrap(); + assert_eq!(b"OK", &body[..]); + core_task.abort(); + core_task.await.unwrap_err(); + } +} diff --git a/iroh-one/src/error.rs b/iroh-one/src/error.rs new file mode 100644 index 0000000000..3494912ed3 --- /dev/null +++ b/iroh-one/src/error.rs @@ -0,0 +1,24 @@ +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde_json::json; + +#[derive(Debug)] +pub struct GatewayError { + pub status_code: StatusCode, + pub message: String, + pub trace_id: String, +} + +impl IntoResponse for GatewayError { + fn into_response(self) -> Response { + let body = axum::Json(json!({ + "code": self.status_code.as_u16(), + "success": false, + "message": self.message, + "trace_id": self.trace_id, + })); + (self.status_code, body).into_response() + } +} diff --git a/iroh-one/src/headers.rs b/iroh-one/src/headers.rs new file mode 100644 index 0000000000..397e745c76 --- /dev/null +++ b/iroh-one/src/headers.rs @@ -0,0 +1,313 @@ +use crate::{constants::*, response::ResponseFormat}; +use ::time::OffsetDateTime; +use axum::http::header::*; +use iroh_resolver::resolver::{CidOrDomain, Metadata, PathType}; +use std::{fmt::Write, time}; + +#[tracing::instrument()] +pub fn add_user_headers(headers: &mut HeaderMap, user_headers: HeaderMap) { + headers.extend(user_headers.into_iter()); +} + +#[tracing::instrument()] +pub fn add_content_type_headers(headers: &mut HeaderMap, name: &str) { + let guess = mime_guess::from_path(name); + let content_type = guess.first_or_octet_stream().to_string(); + // todo(arqu): deeper content type checking + // todo(arqu): if mime type starts with text/html; strip encoding to let browser detect + headers.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type).unwrap()); +} + +#[tracing::instrument()] +pub fn add_content_disposition_headers( + headers: &mut HeaderMap, + filename: &str, + content_path: &str, + should_download: bool, +) -> String { + let mut name = get_filename(content_path); + if !filename.is_empty() { + name = filename.to_string(); + } + if !name.is_empty() { + let disposition = if should_download { + DISPOSITION_ATTACHMENT + } else { + DISPOSITION_INLINE + }; + set_content_disposition_headers(headers, &name, disposition); + } + name +} + +#[tracing::instrument()] +pub fn set_content_disposition_headers(headers: &mut HeaderMap, filename: &str, disposition: &str) { + headers.insert( + CONTENT_DISPOSITION, + HeaderValue::from_str(&format!("{}; filename={}", disposition, filename)).unwrap(), + ); +} + +#[tracing::instrument()] +pub fn add_cache_control_headers(headers: &mut HeaderMap, metadata: Metadata) { + if metadata.path.typ() == PathType::Ipns { + let lmdt: OffsetDateTime = time::SystemTime::now().into(); + headers.insert( + LAST_MODIFIED, + HeaderValue::from_str(&lmdt.to_string()).unwrap(), + ); + } else { + headers.insert(LAST_MODIFIED, HeaderValue::from_str("0").unwrap()); + headers.insert(CACHE_CONTROL, VAL_IMMUTABLE_MAX_AGE.clone()); + } +} + +#[tracing::instrument()] +pub fn add_ipfs_roots_headers(headers: &mut HeaderMap, metadata: Metadata) { + let mut roots = "".to_string(); + for rcid in metadata.resolved_path { + write!(roots, "{},", rcid).unwrap(); + } + roots.pop(); + headers.insert(&HEADER_X_IPFS_ROOTS, HeaderValue::from_str(&roots).unwrap()); +} + +#[tracing::instrument()] +pub fn set_etag_headers(headers: &mut HeaderMap, etag: String) { + headers.insert(ETAG, HeaderValue::from_str(&etag).unwrap()); +} + +#[tracing::instrument()] +pub fn get_etag(cid: &CidOrDomain, response_format: Option) -> String { + match cid { + CidOrDomain::Cid(cid) => { + let mut suffix = "".to_string(); + if let Some(fmt) = response_format { + let ext = fmt.get_extenstion(); + if !ext.is_empty() { + suffix = format!(".{}", ext); + } + } + format!("\"{}{}\"", cid, suffix) + } + CidOrDomain::Domain(_) => { + // TODO: + String::new() + } + } +} + +#[tracing::instrument()] +pub fn etag_matches(inm: &str, cid_etag: &str) -> bool { + let mut buf = inm.trim(); + loop { + if buf.is_empty() { + break; + } + if buf.starts_with(',') { + buf = &buf[1..]; + continue; + } + if buf.starts_with('*') { + return true; + } + let (etag, remain) = scan_etag(buf); + if etag.is_empty() { + break; + } + if etag_weak_match(etag, cid_etag) { + return true; + } + buf = remain; + } + false +} + +#[tracing::instrument()] +pub fn scan_etag(buf: &str) -> (&str, &str) { + let s = buf.trim(); + let mut start = 0; + if s.starts_with("W/") { + start = 2; + } + if s.len() - start < 2 || s.chars().nth(start) != Some('"') { + return ("", ""); + } + for i in start + 1..s.len() { + let c = s.as_bytes().get(i).unwrap(); + if *c == 0x21 || (0x23..0x7E).contains(c) || *c >= 0x80 { + continue; + } + if *c == b'"' { + return (&s[..i + 1], &s[i + 1..]); + } + return ("", ""); + } + ("", "") +} + +#[tracing::instrument()] +pub fn etag_weak_match(etag: &str, cid_etag: &str) -> bool { + etag.trim_start_matches("W/") == cid_etag.trim_start_matches("W/") +} + +#[tracing::instrument()] +pub fn get_filename(content_path: &str) -> String { + content_path + .split('/') + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .last() + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use cid::Cid; + + use super::*; + + #[test] + fn add_user_headers_test() { + let mut headers = HeaderMap::new(); + let mut user_headers = HeaderMap::new(); + user_headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str("QmHeaderPath1").unwrap(), + ); + user_headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str("QmHeaderPath2").unwrap(), + ); + add_user_headers(&mut headers, user_headers); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&HEADER_X_IPFS_PATH).unwrap(), + &"QmHeaderPath2".to_string() + ); + } + + #[test] + fn add_content_type_headers_test() { + let mut headers = HeaderMap::new(); + let name = "test.txt"; + add_content_type_headers(&mut headers, name); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_TYPE).unwrap(), + &"text/plain".to_string() + ); + + let mut headers = HeaderMap::new(); + let name = "test.RAND_EXT"; + add_content_type_headers(&mut headers, name); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_TYPE).unwrap(), + &CONTENT_TYPE_OCTET_STREAM + ); + } + + #[test] + fn add_content_disposition_headers_test() { + // inline + let mut headers = HeaderMap::new(); + let filename = "test.txt"; + let content_path = "QmSomeCid"; + let download = false; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION).unwrap(), + &"inline; filename=test.txt".to_string() + ); + assert_eq!(name, "test.txt"); + + // attachment + let mut headers = HeaderMap::new(); + let filename = "test.txt"; + let content_path = "QmSomeCid"; + let download = true; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION).unwrap(), + &"attachment; filename=test.txt".to_string() + ); + assert_eq!(name, "test.txt"); + + // no filename & no content path filename + let mut headers = HeaderMap::new(); + let filename = ""; + let content_path = "QmSomeCid"; + let download = true; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!(name, "QmSomeCid"); + + // no filename & with content path filename + let mut headers = HeaderMap::new(); + let filename = ""; + let content_path = "QmSomeCid/folder/test.txt"; + let download = true; + let name = add_content_disposition_headers(&mut headers, filename, content_path, download); + assert_eq!(headers.len(), 1); + assert_eq!(name, "test.txt"); + } + + #[test] + fn set_content_disposition_headers_test() { + let mut headers = HeaderMap::new(); + let filename = "test_inline.txt"; + set_content_disposition_headers(&mut headers, filename, DISPOSITION_INLINE); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION).unwrap(), + &"inline; filename=test_inline.txt".to_string() + ); + + let mut headers = HeaderMap::new(); + let filename = "test_attachment.txt"; + set_content_disposition_headers(&mut headers, filename, DISPOSITION_ATTACHMENT); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_DISPOSITION).unwrap(), + &"attachment; filename=test_attachment.txt".to_string() + ); + } + + #[test] + fn get_filename_test() { + assert_eq!(get_filename("QmSomeCid/folder/test.txt"), "test.txt"); + assert_eq!(get_filename("QmSomeCid/folder"), "folder"); + assert_eq!(get_filename("QmSomeCid"), "QmSomeCid"); + assert_eq!(get_filename(""), ""); + } + + #[test] + fn etag_test() { + let any_etag = "*"; + let etag = get_etag( + &CidOrDomain::Cid( + Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy") + .unwrap(), + ), + Some(ResponseFormat::Raw), + ); + let wetag = format!("W/{}", etag); + let other_etag = get_etag( + &CidOrDomain::Cid( + Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4aaaaa") + .unwrap(), + ), + Some(ResponseFormat::Raw), + ); + let other_wetag = format!("W/{}", other_etag); + let long_etag = format!("{},{}", other_etag, wetag); + + assert!(etag_matches(any_etag, &etag)); + assert!(etag_matches(&etag, &wetag)); + assert!(etag_matches(&long_etag, &etag)); + assert!(!etag_matches(&etag, &other_wetag)); + } +} diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs new file mode 100644 index 0000000000..78ca4296bc --- /dev/null +++ b/iroh-one/src/lib.rs @@ -0,0 +1,13 @@ +mod client; +pub mod config; +mod constants; +pub mod core; +mod error; +mod headers; +pub mod mem_p2p; +pub mod mem_store; +pub mod metrics; +mod response; +mod rpc; +mod templates; +mod uds; diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs new file mode 100644 index 0000000000..f7e1043a59 --- /dev/null +++ b/iroh-one/src/main.rs @@ -0,0 +1,122 @@ +use std::collections::HashMap; +use std::path::PathBuf; + +use anyhow::{anyhow, Result}; +use clap::Parser; +use iroh_metrics::gateway::Metrics; +use iroh_one::{ + config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, + core::Core, + metrics, +}; +use iroh_rpc_types::Addr; +use iroh_util::{iroh_home_path, make_config}; +use prometheus_client::registry::Registry; + +#[derive(Parser, Debug, Clone)] +#[clap(author, version, about, long_about = None)] +struct Args { + #[clap(short, long)] + port: Option, + #[clap(short, long)] + writeable: Option, + #[clap(short, long)] + fetch: Option, + #[clap(short, long)] + cache: Option, + #[clap(long = "metrics")] + metrics: bool, + #[clap(long = "tracing")] + tracing: bool, + #[clap(long)] + cfg: Option, +} + +impl Args { + fn make_overrides_map(&self) -> HashMap<&str, String> { + let mut map: HashMap<&str, String> = HashMap::new(); + if let Some(port) = self.port { + map.insert("port", port.to_string()); + } + if let Some(writable) = self.writeable { + map.insert("writable", writable.to_string()); + } + if let Some(fetch) = self.fetch { + map.insert("fetch", fetch.to_string()); + } + if let Some(cache) = self.cache { + map.insert("cache", cache.to_string()); + } + map.insert("metrics.collect", self.metrics.to_string()); + map.insert("metrics.tracing", self.tracing.to_string()); + map + } +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let args = Args::parse(); + + let sources = vec![iroh_home_path(CONFIG_FILE_NAME), args.cfg.clone()]; + let mut config = make_config( + // default + Config::default(), + // potential config files + sources, + // env var prefix for this config + ENV_PREFIX, + // map of present command line arguments + args.make_overrides_map(), + ) + .unwrap(); + + let (store_rpc, p2p_rpc) = { + let (store_recv, store_sender) = Addr::new_mem(); + config.rpc_client.store_addr = Some(store_sender); + let store_rpc = iroh_one::mem_store::start(store_recv).await?; + + let (p2p_recv, p2p_sender) = Addr::new_mem(); + config.rpc_client.p2p_addr = Some(p2p_sender); + let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv).await?; + (store_rpc, p2p_rpc) + }; + + config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); + println!("{:#?}", config); + + let metrics_config = config.metrics.clone(); + let mut prom_registry = Registry::default(); + let gw_metrics = Metrics::new(&mut prom_registry); + let rpc_addr = config + .server_rpc_addr()? + .ok_or_else(|| anyhow!("missing gateway rpc addr"))?; + let handler = Core::new(config, rpc_addr, gw_metrics, &mut prom_registry).await?; + + let metrics_handle = + iroh_metrics::MetricsHandle::from_registry_with_tracer(metrics_config, prom_registry) + .await + .expect("failed to initialize metrics"); + let server = handler.http_server(); + println!("HTTP endpoint listening on {}", server.local_addr()); + let core_task = tokio::spawn(async move { + server.await.unwrap(); + }); + + let uds_server_task = { + let uds_server = handler.uds_server(); + let task = tokio::spawn(async move { + uds_server.await.unwrap(); + }); + task + }; + + iroh_util::block_until_sigint().await; + + store_rpc.abort(); + p2p_rpc.abort(); + uds_server_task.abort(); + core_task.abort(); + + metrics_handle.shutdown(); + Ok(()) +} diff --git a/iroh-gateway/src/mem_p2p.rs b/iroh-one/src/mem_p2p.rs similarity index 100% rename from iroh-gateway/src/mem_p2p.rs rename to iroh-one/src/mem_p2p.rs diff --git a/iroh-gateway/src/mem_store.rs b/iroh-one/src/mem_store.rs similarity index 100% rename from iroh-gateway/src/mem_store.rs rename to iroh-one/src/mem_store.rs diff --git a/iroh-one/src/metrics.rs b/iroh-one/src/metrics.rs new file mode 100644 index 0000000000..20ac886927 --- /dev/null +++ b/iroh-one/src/metrics.rs @@ -0,0 +1,9 @@ +use git_version::git_version; +use iroh_metrics::config::Config as MetricsConfig; + +pub fn metrics_config_with_compile_time_info(cfg: MetricsConfig) -> MetricsConfig { + // compile time configuration + cfg.with_service_name(env!("CARGO_PKG_NAME").to_string()) + .with_build(git_version!().to_string()) + .with_version(env!("CARGO_PKG_VERSION").to_string()) +} diff --git a/iroh-one/src/response.rs b/iroh-one/src/response.rs new file mode 100644 index 0000000000..23f2f00203 --- /dev/null +++ b/iroh-one/src/response.rs @@ -0,0 +1,242 @@ +use axum::{ + body::BoxBody, + http::{header::*, HeaderValue, StatusCode}, + response::{IntoResponse, Redirect, Response}, +}; + +use crate::constants::*; + +pub const ERR_UNSUPPORTED_FORMAT: &str = "unsuported format"; + +#[derive(Debug, Clone, PartialEq)] +pub enum ResponseFormat { + Raw, + Car, + Fs(String), +} + +impl std::convert::TryFrom<&str> for ResponseFormat { + type Error = String; + + fn try_from(s: &str) -> Result { + match s.to_lowercase().as_str() { + "application/vnd.ipld.raw" | "raw" => Ok(ResponseFormat::Raw), + "application/vnd.ipld.car" | "car" => Ok(ResponseFormat::Car), + "fs" | "" => Ok(ResponseFormat::Fs(String::new())), + rf => { + if rf.starts_with("application/vnd.ipld.") { + Ok(ResponseFormat::Fs(rf.to_string())) + } else { + Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, rf)) + } + } + } + } +} + +impl ResponseFormat { + pub fn write_headers(&self, headers: &mut HeaderMap) { + match self { + ResponseFormat::Raw => { + headers.insert(CONTENT_TYPE, CONTENT_TYPE_IPLD_RAW.clone()); + headers.insert(&HEADER_X_CONTENT_TYPE_OPTIONS, VALUE_XCTO_NOSNIFF.clone()); + } + ResponseFormat::Car => { + headers.insert(CONTENT_TYPE, CONTENT_TYPE_IPLD_CAR.clone()); + headers.insert(&HEADER_X_CONTENT_TYPE_OPTIONS, VALUE_XCTO_NOSNIFF.clone()); + headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); + headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); + } + ResponseFormat::Fs(_) => { + headers.insert(CONTENT_TYPE, CONTENT_TYPE_OCTET_STREAM.clone()); + } + } + } + + pub fn get_extenstion(&self) -> String { + match self { + ResponseFormat::Raw => "bin".to_string(), + ResponseFormat::Car => "car".to_string(), + ResponseFormat::Fs(s) => { + if s.is_empty() { + String::new() + } else { + s.split('.').last().unwrap().to_string() + } + } + } + } + + pub fn try_from_headers(headers: &HeaderMap) -> Result { + if headers.contains_key("Accept") { + if let Some(h_values) = headers.get("Accept") { + let h_values = h_values.to_str().unwrap().split(','); + for h_value in h_values { + let h_value = h_value.trim(); + if h_value.starts_with("application/vnd.ipld.") { + // if valid media type use it, otherwise return error + // todo(arqu): add support for better media type detection + if h_value != "application/vnd.ipld.raw" + && h_value != "application/vnd.ipld.car" + { + return Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, h_value)); + } + return ResponseFormat::try_from(h_value); + } + } + } + } + Ok(ResponseFormat::Fs(String::new())) + } +} + +#[tracing::instrument()] +pub fn get_response_format( + request_headers: &HeaderMap, + query_format: Option, +) -> Result { + let format = if let Some(format) = query_format { + if format.is_empty() { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + } + } else { + match ResponseFormat::try_from(format.as_str()) { + Ok(format) => format, + Err(_) => { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + }; + return Err("invalid format".to_string()); + } + } + } + } else { + match ResponseFormat::try_from_headers(request_headers) { + Ok(format) => format, + Err(_) => { + return Err("invalid format".to_string()); + } + } + }; + Ok(format) +} + +#[derive(Debug)] +pub struct GatewayResponse { + pub status_code: StatusCode, + pub body: BoxBody, + pub headers: HeaderMap, + pub trace_id: String, +} + +impl IntoResponse for GatewayResponse { + fn into_response(mut self) -> Response { + if self.status_code == StatusCode::SEE_OTHER { + if let Some(ref path) = self.headers.remove(LOCATION) { + if let Ok(path) = path.to_str() { + return Redirect::to(path).into_response(); + } + } + } + let mut rb = Response::builder().status(self.status_code); + self.headers.insert( + &HEADER_X_TRACE_ID, + HeaderValue::from_str(&self.trace_id).unwrap(), + ); + let rh = rb.headers_mut().unwrap(); + rh.extend(self.headers); + rb.body(self.body).unwrap() + } +} + +impl GatewayResponse { + fn _redirect(to: &str, status_code: StatusCode) -> Self { + let mut headers = HeaderMap::new(); + headers.insert(LOCATION, HeaderValue::from_str(to).unwrap()); + GatewayResponse { + status_code, + body: BoxBody::default(), + headers: HeaderMap::new(), + trace_id: String::new(), + } + } + + pub fn redirect(to: &str) -> Self { + Self::_redirect(to, StatusCode::SEE_OTHER) + } + + pub fn redirect_permanently(to: &str) -> Self { + Self::_redirect(to, StatusCode::MOVED_PERMANENTLY) + } + + pub fn not_modified() -> Self { + Self { + status_code: StatusCode::NOT_MODIFIED, + body: BoxBody::default(), + headers: HeaderMap::new(), + trace_id: String::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn response_format_try_from() { + let rf = ResponseFormat::try_from("raw"); + assert_eq!(rf, Ok(ResponseFormat::Raw)); + let rf = ResponseFormat::try_from("car"); + assert_eq!(rf, Ok(ResponseFormat::Car)); + let rf = ResponseFormat::try_from("fs"); + assert_eq!(rf, Ok(ResponseFormat::Fs(String::new()))); + let rf = ResponseFormat::try_from(""); + assert_eq!(rf, Ok(ResponseFormat::Fs(String::new()))); + + let rf = ResponseFormat::try_from("RaW"); + assert_eq!(rf, Ok(ResponseFormat::Raw)); + + let rf = ResponseFormat::try_from("UNKNOWN"); + assert!(rf.is_err()); + } + + #[test] + fn response_format_write_headers() { + let rf = ResponseFormat::try_from("raw").unwrap(); + let mut headers = HeaderMap::new(); + rf.write_headers(&mut headers); + assert_eq!(headers.len(), 2); + assert_eq!(headers.get(&CONTENT_TYPE).unwrap(), &CONTENT_TYPE_IPLD_RAW); + assert_eq!( + headers.get(&HEADER_X_CONTENT_TYPE_OPTIONS).unwrap(), + &VALUE_XCTO_NOSNIFF + ); + + let rf = ResponseFormat::try_from("car").unwrap(); + let mut headers = HeaderMap::new(); + rf.write_headers(&mut headers); + assert_eq!(headers.len(), 4); + assert_eq!(headers.get(&CONTENT_TYPE).unwrap(), &CONTENT_TYPE_IPLD_CAR); + assert_eq!( + headers.get(&HEADER_X_CONTENT_TYPE_OPTIONS).unwrap(), + &VALUE_XCTO_NOSNIFF + ); + + let rf = ResponseFormat::try_from("fs").unwrap(); + let mut headers = HeaderMap::new(); + rf.write_headers(&mut headers); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(&CONTENT_TYPE).unwrap(), + &CONTENT_TYPE_OCTET_STREAM + ); + } +} diff --git a/iroh-one/src/rpc.rs b/iroh-one/src/rpc.rs new file mode 100644 index 0000000000..90ca1f7288 --- /dev/null +++ b/iroh-one/src/rpc.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use async_trait::async_trait; +use iroh_rpc_types::gateway::{Gateway as RpcGateway, GatewayServerAddr, VersionResponse}; + +#[derive(Default)] +pub struct Gateway {} + +#[async_trait] +impl RpcGateway for Gateway { + #[tracing::instrument(skip(self))] + async fn version(&self, _: ()) -> Result { + let version = env!("CARGO_PKG_VERSION").to_string(); + Ok(VersionResponse { version }) + } +} + +#[cfg(feature = "grpc")] +impl iroh_rpc_types::NamedService for Gateway { + const NAME: &'static str = "gateway"; +} + +pub async fn new(addr: GatewayServerAddr, gateway: Gateway) -> Result<()> { + iroh_rpc_types::gateway::serve(addr, gateway).await +} diff --git a/iroh-one/src/templates.rs b/iroh-one/src/templates.rs new file mode 100644 index 0000000000..2a3c596a6c --- /dev/null +++ b/iroh-one/src/templates.rs @@ -0,0 +1,19 @@ +pub const DIR_LIST: &str = " +
+
    + {{#each links}} +
  1. + {{this.name}} +
    2. + {{/each}} +
+
"; + +pub const NOT_FOUND: &str = " +
+

404 Not Found

+

+ The requested resource was not found. +

+
+"; diff --git a/iroh-gateway/src/uds.rs b/iroh-one/src/uds.rs similarity index 100% rename from iroh-gateway/src/uds.rs rename to iroh-one/src/uds.rs diff --git a/iroh-rpc-client/src/config.rs b/iroh-rpc-client/src/config.rs index 07cfef32fc..352acce228 100644 --- a/iroh-rpc-client/src/config.rs +++ b/iroh-rpc-client/src/config.rs @@ -42,30 +42,6 @@ impl Config { store_addr: Some("grpc://0.0.0.0:4402".parse().unwrap()), } } - - // When running in ipfsd mode, the resolver will use memory channels to - // communicate with the p2p and store modules. - // The gateway itself is exposing a UDS rpc endpoint to be also usable - // as a single entry point for other system services. - pub fn default_ipfsd() -> Self { - use iroh_rpc_types::Addr; - let path = { - #[cfg(target_os = "android")] - "/dev/socket/ipfsd".into(); - - #[cfg(not(target_os = "android"))] - { - let path = format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()); - path.into() - } - }; - - Self { - gateway_addr: Some(Addr::GrpcUds(path)), - p2p_addr: None, - store_addr: None, - } - } } #[cfg(test)] From b40238bc73a7d1d96444c2be6b2405bef37150b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 11:00:09 -0700 Subject: [PATCH 06/29] iroh-one: share constants, headers and response with iroh-gateway --- iroh-gateway/src/lib.rs | 6 +- iroh-one/.gitignore | 1 + iroh-one/Cargo.toml | 82 +++++----- iroh-one/src/client.rs | 2 +- iroh-one/src/config.rs | 2 +- iroh-one/src/constants.rs | 35 ----- iroh-one/src/core.rs | 25 +-- iroh-one/src/headers.rs | 313 -------------------------------------- iroh-one/src/lib.rs | 3 - iroh-one/src/response.rs | 242 ----------------------------- 10 files changed, 59 insertions(+), 652 deletions(-) create mode 100644 iroh-one/.gitignore delete mode 100644 iroh-one/src/constants.rs delete mode 100644 iroh-one/src/headers.rs delete mode 100644 iroh-one/src/response.rs diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index 5f7df4a8c1..6d86f0b706 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -1,11 +1,11 @@ pub mod bad_bits; mod client; pub mod config; -mod constants; +pub mod constants; pub mod core; mod error; -mod headers; +pub mod headers; pub mod metrics; -mod response; +pub mod response; mod rpc; mod templates; diff --git a/iroh-one/.gitignore b/iroh-one/.gitignore new file mode 100644 index 0000000000..1c3f71732b --- /dev/null +++ b/iroh-one/.gitignore @@ -0,0 +1 @@ +iroh-store/ diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index cf1b7fe9e5..8736f648d4 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -1,64 +1,62 @@ [package] -name = "iroh-one" -version = "0.1.0" +description = "Single binary IPFS gateway" edition = "2021" license = "Apache-2.0/MIT" +name = "iroh-one" readme = "README.md" -description = "Single binary IPFS gateway" repository = "https://github.com/dignifiedquire/iroh" +version = "0.1.0" [dependencies] -iroh-rpc-client = { path = "../iroh-rpc-client", default-features = false } -iroh-rpc-types = { path = "../iroh-rpc-types", default-features = false } +anyhow = "1" +async-recursion = "1.0.0" +async-trait = "0.1.56" +axum = "0.5.1" +bytes = "1.1.0" +cid = "0.8.4" +clap = {version = "3.1.14", features = ["derive"]} +config = "0.13.1" +dirs = "4.0.0" +futures = "0.3.21" +git-version = "0.3.5" +handlebars = "4" +headers = "0.3.7" +http = "0.2" +http-serde = "1.1.0" +hyper = "0.14.19" +iroh-gateway = {path = "../iroh-gateway"} +iroh-metrics = {path = "../iroh-metrics", default-features = false} iroh-p2p = {path = "../iroh-p2p", default-features = false, features = ["rpc-mem"]} +iroh-resolver = {path = "../iroh-resolver"} +iroh-rpc-client = {path = "../iroh-rpc-client", default-features = false} +iroh-rpc-types = {path = "../iroh-rpc-types", default-features = false} iroh-store = {path = "../iroh-store", default-features = false, features = ["rpc-mem"]} - -cid = "0.8.4" - -tokio = { version = "1", features = ["macros", "rt-multi-thread", "process"] } -axum = "0.5.1" -clap = { version = "3.1.14", features = ["derive"] } -serde = { version = "1.0", features = ["derive"] } +iroh-util = {path = "../iroh-util"} +libp2p = {version = "0.47", default-features = false} +mime_guess = "2.0.4" +names = {version = "0.14.0", default-features = false} +opentelemetry = {version = "0.17.0", features = ["rt-tokio"]} +prometheus-client = "0.17.0" +rand = "0.8.5" +serde = {version = "1.0", features = ["derive"]} serde_json = "1.0.78" serde_qs = "0.10.1" -tower = { version = "0.4", features = ["util", "timeout", "load-shed", "limit"] } -mime_guess = "2.0.4" -iroh-metrics = { path = "../iroh-metrics", default-features = false } +time = "0.3.9" +tokio = {version = "1", features = ["macros", "rt-multi-thread", "process"]} +tokio-util = {version = "0.7", features = ["io"]} +toml = "0.5.9" +tower = {version = "0.4", features = ["util", "timeout", "load-shed", "limit"]} +tower-http = {version = "0.3", features = ["trace"]} +tower-layer = {version = "0.3"} tracing = "0.1.33" -names = { version = "0.14.0", default-features = false } -git-version = "0.3.5" -rand = "0.8.5" tracing-opentelemetry = "0.17.2" -opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } -time = "0.3.9" -headers = "0.3.7" -hyper = "0.14.19" -libp2p = { version = "0.47", default-features = false } -iroh-util = { path = "../iroh-util" } -anyhow = "1" -futures = "0.3.21" -tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } -iroh-resolver = { path = "../iroh-resolver" } -prometheus-client = "0.17.0" -tokio-util = { version = "0.7", features = ["io"] } -bytes = "1.1.0" -tower-layer = { version = "0.3" } -tower-http = { version = "0.3", features = ["trace"] } -http = "0.2" -async-recursion = "1.0.0" -handlebars = "4" +tracing-subscriber = {version = "0.3.11", features = ["env-filter"]} url = "2.2.2" urlencoding = "2.1.0" -dirs = "4.0.0" -toml = "0.5.9" -http-serde = "1.1.0" -config = "0.13.1" -async-trait = "0.1.56" [dev-dependencies] axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers - [features] default = ["rpc-mem", "rpc-grpc"] rpc-grpc = ["iroh-rpc-types/grpc", "iroh-rpc-client/grpc", "iroh-metrics/rpc-grpc"] diff --git a/iroh-one/src/client.rs b/iroh-one/src/client.rs index 64ed87a4ce..023714154c 100644 --- a/iroh-one/src/client.rs +++ b/iroh-one/src/client.rs @@ -16,7 +16,7 @@ use tracing::info; use tracing::warn; use crate::core::GetParams; -use crate::response::ResponseFormat; +use iroh_gateway::response::ResponseFormat; #[derive(Debug, Clone)] pub struct Client { diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index aac2d2aa65..d2bd581685 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -1,4 +1,4 @@ -use crate::constants::*; +use iroh_gateway::constants::*; use anyhow::{bail, Result}; use axum::http::{header::*, Method}; use config::{ConfigError, Map, Source, Value}; diff --git a/iroh-one/src/constants.rs b/iroh-one/src/constants.rs deleted file mode 100644 index 0aad73854b..0000000000 --- a/iroh-one/src/constants.rs +++ /dev/null @@ -1,35 +0,0 @@ -use axum::http::{header::HeaderName, HeaderValue}; - -// Headers -pub static HEADER_X_IPFS_PATH: HeaderName = HeaderName::from_static("x-ipfs-path"); -pub static HEADER_X_CONTENT_TYPE_OPTIONS: HeaderName = - HeaderName::from_static("x-content-type-options"); -pub static HEADER_X_TRACE_ID: HeaderName = HeaderName::from_static("x-trace-id"); -pub static HEADER_X_IPFS_GATEWAY_PREFIX: HeaderName = - HeaderName::from_static("x-ipfs-gateway-prefix"); -pub static HEADER_X_IPFS_ROOTS: HeaderName = HeaderName::from_static("x-ipfs-roots"); -pub static HEADER_SERVICE_WORKER: HeaderName = HeaderName::from_static("service-worker"); - -// Common Header Values -pub static VALUE_XCTO_NOSNIFF: HeaderValue = HeaderValue::from_static("nosniff"); -pub static VALUE_NONE: HeaderValue = HeaderValue::from_static("none"); -pub static VALUE_NO_CACHE_NO_TRANSFORM: HeaderValue = - HeaderValue::from_static("no-cache, no-transform"); -pub static VAL_IMMUTABLE_MAX_AGE: HeaderValue = - HeaderValue::from_static("public, max-age=31536000, immutable"); - -// Dispositions -pub static DISPOSITION_ATTACHMENT: &str = "attachment"; -pub static DISPOSITION_INLINE: &str = "inline"; - -// Content Types -pub static CONTENT_TYPE_IPLD_RAW: HeaderValue = - HeaderValue::from_static("application/vnd.ipld.raw"); -pub static CONTENT_TYPE_IPLD_CAR: HeaderValue = - HeaderValue::from_static("application/vnd.ipld.car; version=1"); -pub static CONTENT_TYPE_OCTET_STREAM: HeaderValue = - HeaderValue::from_static("application/octet-stream"); - -// Schemes -pub static SCHEME_IPFS: &str = "ipfs"; -pub static SCHEME_IPNS: &str = "ipns"; diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 2bebd27913..0698bb8b96 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -1,3 +1,11 @@ +use crate::{ + client::{Client, Request}, + config::Config, + error::GatewayError, + rpc, + rpc::Gateway, + templates, uds, +}; use async_recursion::async_recursion; use axum::{ body::{self, Body, HttpBody}, @@ -10,6 +18,11 @@ use axum::{ }; use bytes::Bytes; use handlebars::Handlebars; +use iroh_gateway::{ + constants::*, + headers::*, + response::{get_response_format, GatewayResponse, ResponseFormat}, +}; use iroh_metrics::{gateway::Metrics, get_current_trace_id}; use iroh_resolver::resolver::{CidOrDomain, UnixfsType}; use iroh_rpc_client::Client as RpcClient; @@ -34,18 +47,6 @@ use tower_http::trace::TraceLayer; use tracing::info_span; use url::Url; use urlencoding::encode; -use crate::{ - client::{Client, Request}, - config::Config, - constants::*, - error::GatewayError, - headers::*, - response::{get_response_format, GatewayResponse, ResponseFormat}, - rpc, - rpc::Gateway, - templates, - uds, -}; #[derive(Debug)] pub struct Core { diff --git a/iroh-one/src/headers.rs b/iroh-one/src/headers.rs deleted file mode 100644 index 397e745c76..0000000000 --- a/iroh-one/src/headers.rs +++ /dev/null @@ -1,313 +0,0 @@ -use crate::{constants::*, response::ResponseFormat}; -use ::time::OffsetDateTime; -use axum::http::header::*; -use iroh_resolver::resolver::{CidOrDomain, Metadata, PathType}; -use std::{fmt::Write, time}; - -#[tracing::instrument()] -pub fn add_user_headers(headers: &mut HeaderMap, user_headers: HeaderMap) { - headers.extend(user_headers.into_iter()); -} - -#[tracing::instrument()] -pub fn add_content_type_headers(headers: &mut HeaderMap, name: &str) { - let guess = mime_guess::from_path(name); - let content_type = guess.first_or_octet_stream().to_string(); - // todo(arqu): deeper content type checking - // todo(arqu): if mime type starts with text/html; strip encoding to let browser detect - headers.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type).unwrap()); -} - -#[tracing::instrument()] -pub fn add_content_disposition_headers( - headers: &mut HeaderMap, - filename: &str, - content_path: &str, - should_download: bool, -) -> String { - let mut name = get_filename(content_path); - if !filename.is_empty() { - name = filename.to_string(); - } - if !name.is_empty() { - let disposition = if should_download { - DISPOSITION_ATTACHMENT - } else { - DISPOSITION_INLINE - }; - set_content_disposition_headers(headers, &name, disposition); - } - name -} - -#[tracing::instrument()] -pub fn set_content_disposition_headers(headers: &mut HeaderMap, filename: &str, disposition: &str) { - headers.insert( - CONTENT_DISPOSITION, - HeaderValue::from_str(&format!("{}; filename={}", disposition, filename)).unwrap(), - ); -} - -#[tracing::instrument()] -pub fn add_cache_control_headers(headers: &mut HeaderMap, metadata: Metadata) { - if metadata.path.typ() == PathType::Ipns { - let lmdt: OffsetDateTime = time::SystemTime::now().into(); - headers.insert( - LAST_MODIFIED, - HeaderValue::from_str(&lmdt.to_string()).unwrap(), - ); - } else { - headers.insert(LAST_MODIFIED, HeaderValue::from_str("0").unwrap()); - headers.insert(CACHE_CONTROL, VAL_IMMUTABLE_MAX_AGE.clone()); - } -} - -#[tracing::instrument()] -pub fn add_ipfs_roots_headers(headers: &mut HeaderMap, metadata: Metadata) { - let mut roots = "".to_string(); - for rcid in metadata.resolved_path { - write!(roots, "{},", rcid).unwrap(); - } - roots.pop(); - headers.insert(&HEADER_X_IPFS_ROOTS, HeaderValue::from_str(&roots).unwrap()); -} - -#[tracing::instrument()] -pub fn set_etag_headers(headers: &mut HeaderMap, etag: String) { - headers.insert(ETAG, HeaderValue::from_str(&etag).unwrap()); -} - -#[tracing::instrument()] -pub fn get_etag(cid: &CidOrDomain, response_format: Option) -> String { - match cid { - CidOrDomain::Cid(cid) => { - let mut suffix = "".to_string(); - if let Some(fmt) = response_format { - let ext = fmt.get_extenstion(); - if !ext.is_empty() { - suffix = format!(".{}", ext); - } - } - format!("\"{}{}\"", cid, suffix) - } - CidOrDomain::Domain(_) => { - // TODO: - String::new() - } - } -} - -#[tracing::instrument()] -pub fn etag_matches(inm: &str, cid_etag: &str) -> bool { - let mut buf = inm.trim(); - loop { - if buf.is_empty() { - break; - } - if buf.starts_with(',') { - buf = &buf[1..]; - continue; - } - if buf.starts_with('*') { - return true; - } - let (etag, remain) = scan_etag(buf); - if etag.is_empty() { - break; - } - if etag_weak_match(etag, cid_etag) { - return true; - } - buf = remain; - } - false -} - -#[tracing::instrument()] -pub fn scan_etag(buf: &str) -> (&str, &str) { - let s = buf.trim(); - let mut start = 0; - if s.starts_with("W/") { - start = 2; - } - if s.len() - start < 2 || s.chars().nth(start) != Some('"') { - return ("", ""); - } - for i in start + 1..s.len() { - let c = s.as_bytes().get(i).unwrap(); - if *c == 0x21 || (0x23..0x7E).contains(c) || *c >= 0x80 { - continue; - } - if *c == b'"' { - return (&s[..i + 1], &s[i + 1..]); - } - return ("", ""); - } - ("", "") -} - -#[tracing::instrument()] -pub fn etag_weak_match(etag: &str, cid_etag: &str) -> bool { - etag.trim_start_matches("W/") == cid_etag.trim_start_matches("W/") -} - -#[tracing::instrument()] -pub fn get_filename(content_path: &str) -> String { - content_path - .split('/') - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .last() - .unwrap_or_default() -} - -#[cfg(test)] -mod tests { - use cid::Cid; - - use super::*; - - #[test] - fn add_user_headers_test() { - let mut headers = HeaderMap::new(); - let mut user_headers = HeaderMap::new(); - user_headers.insert( - &HEADER_X_IPFS_PATH, - HeaderValue::from_str("QmHeaderPath1").unwrap(), - ); - user_headers.insert( - &HEADER_X_IPFS_PATH, - HeaderValue::from_str("QmHeaderPath2").unwrap(), - ); - add_user_headers(&mut headers, user_headers); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&HEADER_X_IPFS_PATH).unwrap(), - &"QmHeaderPath2".to_string() - ); - } - - #[test] - fn add_content_type_headers_test() { - let mut headers = HeaderMap::new(); - let name = "test.txt"; - add_content_type_headers(&mut headers, name); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_TYPE).unwrap(), - &"text/plain".to_string() - ); - - let mut headers = HeaderMap::new(); - let name = "test.RAND_EXT"; - add_content_type_headers(&mut headers, name); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_TYPE).unwrap(), - &CONTENT_TYPE_OCTET_STREAM - ); - } - - #[test] - fn add_content_disposition_headers_test() { - // inline - let mut headers = HeaderMap::new(); - let filename = "test.txt"; - let content_path = "QmSomeCid"; - let download = false; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION).unwrap(), - &"inline; filename=test.txt".to_string() - ); - assert_eq!(name, "test.txt"); - - // attachment - let mut headers = HeaderMap::new(); - let filename = "test.txt"; - let content_path = "QmSomeCid"; - let download = true; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION).unwrap(), - &"attachment; filename=test.txt".to_string() - ); - assert_eq!(name, "test.txt"); - - // no filename & no content path filename - let mut headers = HeaderMap::new(); - let filename = ""; - let content_path = "QmSomeCid"; - let download = true; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!(name, "QmSomeCid"); - - // no filename & with content path filename - let mut headers = HeaderMap::new(); - let filename = ""; - let content_path = "QmSomeCid/folder/test.txt"; - let download = true; - let name = add_content_disposition_headers(&mut headers, filename, content_path, download); - assert_eq!(headers.len(), 1); - assert_eq!(name, "test.txt"); - } - - #[test] - fn set_content_disposition_headers_test() { - let mut headers = HeaderMap::new(); - let filename = "test_inline.txt"; - set_content_disposition_headers(&mut headers, filename, DISPOSITION_INLINE); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION).unwrap(), - &"inline; filename=test_inline.txt".to_string() - ); - - let mut headers = HeaderMap::new(); - let filename = "test_attachment.txt"; - set_content_disposition_headers(&mut headers, filename, DISPOSITION_ATTACHMENT); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_DISPOSITION).unwrap(), - &"attachment; filename=test_attachment.txt".to_string() - ); - } - - #[test] - fn get_filename_test() { - assert_eq!(get_filename("QmSomeCid/folder/test.txt"), "test.txt"); - assert_eq!(get_filename("QmSomeCid/folder"), "folder"); - assert_eq!(get_filename("QmSomeCid"), "QmSomeCid"); - assert_eq!(get_filename(""), ""); - } - - #[test] - fn etag_test() { - let any_etag = "*"; - let etag = get_etag( - &CidOrDomain::Cid( - Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy") - .unwrap(), - ), - Some(ResponseFormat::Raw), - ); - let wetag = format!("W/{}", etag); - let other_etag = get_etag( - &CidOrDomain::Cid( - Cid::try_from("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4aaaaa") - .unwrap(), - ), - Some(ResponseFormat::Raw), - ); - let other_wetag = format!("W/{}", other_etag); - let long_etag = format!("{},{}", other_etag, wetag); - - assert!(etag_matches(any_etag, &etag)); - assert!(etag_matches(&etag, &wetag)); - assert!(etag_matches(&long_etag, &etag)); - assert!(!etag_matches(&etag, &other_wetag)); - } -} diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs index 78ca4296bc..66f7b75172 100644 --- a/iroh-one/src/lib.rs +++ b/iroh-one/src/lib.rs @@ -1,13 +1,10 @@ mod client; pub mod config; -mod constants; pub mod core; mod error; -mod headers; pub mod mem_p2p; pub mod mem_store; pub mod metrics; -mod response; mod rpc; mod templates; mod uds; diff --git a/iroh-one/src/response.rs b/iroh-one/src/response.rs deleted file mode 100644 index 23f2f00203..0000000000 --- a/iroh-one/src/response.rs +++ /dev/null @@ -1,242 +0,0 @@ -use axum::{ - body::BoxBody, - http::{header::*, HeaderValue, StatusCode}, - response::{IntoResponse, Redirect, Response}, -}; - -use crate::constants::*; - -pub const ERR_UNSUPPORTED_FORMAT: &str = "unsuported format"; - -#[derive(Debug, Clone, PartialEq)] -pub enum ResponseFormat { - Raw, - Car, - Fs(String), -} - -impl std::convert::TryFrom<&str> for ResponseFormat { - type Error = String; - - fn try_from(s: &str) -> Result { - match s.to_lowercase().as_str() { - "application/vnd.ipld.raw" | "raw" => Ok(ResponseFormat::Raw), - "application/vnd.ipld.car" | "car" => Ok(ResponseFormat::Car), - "fs" | "" => Ok(ResponseFormat::Fs(String::new())), - rf => { - if rf.starts_with("application/vnd.ipld.") { - Ok(ResponseFormat::Fs(rf.to_string())) - } else { - Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, rf)) - } - } - } - } -} - -impl ResponseFormat { - pub fn write_headers(&self, headers: &mut HeaderMap) { - match self { - ResponseFormat::Raw => { - headers.insert(CONTENT_TYPE, CONTENT_TYPE_IPLD_RAW.clone()); - headers.insert(&HEADER_X_CONTENT_TYPE_OPTIONS, VALUE_XCTO_NOSNIFF.clone()); - } - ResponseFormat::Car => { - headers.insert(CONTENT_TYPE, CONTENT_TYPE_IPLD_CAR.clone()); - headers.insert(&HEADER_X_CONTENT_TYPE_OPTIONS, VALUE_XCTO_NOSNIFF.clone()); - headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); - headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); - } - ResponseFormat::Fs(_) => { - headers.insert(CONTENT_TYPE, CONTENT_TYPE_OCTET_STREAM.clone()); - } - } - } - - pub fn get_extenstion(&self) -> String { - match self { - ResponseFormat::Raw => "bin".to_string(), - ResponseFormat::Car => "car".to_string(), - ResponseFormat::Fs(s) => { - if s.is_empty() { - String::new() - } else { - s.split('.').last().unwrap().to_string() - } - } - } - } - - pub fn try_from_headers(headers: &HeaderMap) -> Result { - if headers.contains_key("Accept") { - if let Some(h_values) = headers.get("Accept") { - let h_values = h_values.to_str().unwrap().split(','); - for h_value in h_values { - let h_value = h_value.trim(); - if h_value.starts_with("application/vnd.ipld.") { - // if valid media type use it, otherwise return error - // todo(arqu): add support for better media type detection - if h_value != "application/vnd.ipld.raw" - && h_value != "application/vnd.ipld.car" - { - return Err(format!("{}: {}", ERR_UNSUPPORTED_FORMAT, h_value)); - } - return ResponseFormat::try_from(h_value); - } - } - } - } - Ok(ResponseFormat::Fs(String::new())) - } -} - -#[tracing::instrument()] -pub fn get_response_format( - request_headers: &HeaderMap, - query_format: Option, -) -> Result { - let format = if let Some(format) = query_format { - if format.is_empty() { - match ResponseFormat::try_from_headers(request_headers) { - Ok(format) => format, - Err(_) => { - return Err("invalid format".to_string()); - } - } - } else { - match ResponseFormat::try_from(format.as_str()) { - Ok(format) => format, - Err(_) => { - match ResponseFormat::try_from_headers(request_headers) { - Ok(format) => format, - Err(_) => { - return Err("invalid format".to_string()); - } - }; - return Err("invalid format".to_string()); - } - } - } - } else { - match ResponseFormat::try_from_headers(request_headers) { - Ok(format) => format, - Err(_) => { - return Err("invalid format".to_string()); - } - } - }; - Ok(format) -} - -#[derive(Debug)] -pub struct GatewayResponse { - pub status_code: StatusCode, - pub body: BoxBody, - pub headers: HeaderMap, - pub trace_id: String, -} - -impl IntoResponse for GatewayResponse { - fn into_response(mut self) -> Response { - if self.status_code == StatusCode::SEE_OTHER { - if let Some(ref path) = self.headers.remove(LOCATION) { - if let Ok(path) = path.to_str() { - return Redirect::to(path).into_response(); - } - } - } - let mut rb = Response::builder().status(self.status_code); - self.headers.insert( - &HEADER_X_TRACE_ID, - HeaderValue::from_str(&self.trace_id).unwrap(), - ); - let rh = rb.headers_mut().unwrap(); - rh.extend(self.headers); - rb.body(self.body).unwrap() - } -} - -impl GatewayResponse { - fn _redirect(to: &str, status_code: StatusCode) -> Self { - let mut headers = HeaderMap::new(); - headers.insert(LOCATION, HeaderValue::from_str(to).unwrap()); - GatewayResponse { - status_code, - body: BoxBody::default(), - headers: HeaderMap::new(), - trace_id: String::new(), - } - } - - pub fn redirect(to: &str) -> Self { - Self::_redirect(to, StatusCode::SEE_OTHER) - } - - pub fn redirect_permanently(to: &str) -> Self { - Self::_redirect(to, StatusCode::MOVED_PERMANENTLY) - } - - pub fn not_modified() -> Self { - Self { - status_code: StatusCode::NOT_MODIFIED, - body: BoxBody::default(), - headers: HeaderMap::new(), - trace_id: String::new(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn response_format_try_from() { - let rf = ResponseFormat::try_from("raw"); - assert_eq!(rf, Ok(ResponseFormat::Raw)); - let rf = ResponseFormat::try_from("car"); - assert_eq!(rf, Ok(ResponseFormat::Car)); - let rf = ResponseFormat::try_from("fs"); - assert_eq!(rf, Ok(ResponseFormat::Fs(String::new()))); - let rf = ResponseFormat::try_from(""); - assert_eq!(rf, Ok(ResponseFormat::Fs(String::new()))); - - let rf = ResponseFormat::try_from("RaW"); - assert_eq!(rf, Ok(ResponseFormat::Raw)); - - let rf = ResponseFormat::try_from("UNKNOWN"); - assert!(rf.is_err()); - } - - #[test] - fn response_format_write_headers() { - let rf = ResponseFormat::try_from("raw").unwrap(); - let mut headers = HeaderMap::new(); - rf.write_headers(&mut headers); - assert_eq!(headers.len(), 2); - assert_eq!(headers.get(&CONTENT_TYPE).unwrap(), &CONTENT_TYPE_IPLD_RAW); - assert_eq!( - headers.get(&HEADER_X_CONTENT_TYPE_OPTIONS).unwrap(), - &VALUE_XCTO_NOSNIFF - ); - - let rf = ResponseFormat::try_from("car").unwrap(); - let mut headers = HeaderMap::new(); - rf.write_headers(&mut headers); - assert_eq!(headers.len(), 4); - assert_eq!(headers.get(&CONTENT_TYPE).unwrap(), &CONTENT_TYPE_IPLD_CAR); - assert_eq!( - headers.get(&HEADER_X_CONTENT_TYPE_OPTIONS).unwrap(), - &VALUE_XCTO_NOSNIFF - ); - - let rf = ResponseFormat::try_from("fs").unwrap(); - let mut headers = HeaderMap::new(); - rf.write_headers(&mut headers); - assert_eq!(headers.len(), 1); - assert_eq!( - headers.get(&CONTENT_TYPE).unwrap(), - &CONTENT_TYPE_OCTET_STREAM - ); - } -} From e85d32df24a9891cf85b6527a55e09f57ea0f982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 12:09:15 -0700 Subject: [PATCH 07/29] refactor: make the http handling code shared between iroh-gateway and iroh-one --- iroh-gateway/src/bad_bits.rs | 2 +- iroh-gateway/src/client.rs | 2 +- iroh-gateway/src/config.rs | 14 + iroh-gateway/src/core.rs | 647 +++-------------------------------- iroh-gateway/src/handlers.rs | 577 +++++++++++++++++++++++++++++++ iroh-gateway/src/lib.rs | 3 +- iroh-gateway/src/main.rs | 2 +- iroh-one/src/config.rs | 14 + iroh-one/src/core.rs | 594 +------------------------------- iroh-one/src/lib.rs | 1 - iroh-one/src/main.rs | 3 +- 11 files changed, 679 insertions(+), 1180 deletions(-) create mode 100644 iroh-gateway/src/handlers.rs diff --git a/iroh-gateway/src/bad_bits.rs b/iroh-gateway/src/bad_bits.rs index 2b9695845f..c945b2b45f 100644 --- a/iroh-gateway/src/bad_bits.rs +++ b/iroh-gateway/src/bad_bits.rs @@ -194,7 +194,7 @@ mod tests { let gw_metrics = Metrics::new(&mut prom_registry); let rpc_addr = "grpc://0.0.0.0:0".parse().unwrap(); let handler = crate::core::Core::new( - config, + Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry, diff --git a/iroh-gateway/src/client.rs b/iroh-gateway/src/client.rs index 7fe64c3a80..10b56e8910 100644 --- a/iroh-gateway/src/client.rs +++ b/iroh-gateway/src/client.rs @@ -20,7 +20,7 @@ use tokio_util::io::ReaderStream; use tracing::info; use tracing::warn; -use crate::core::GetParams; +use crate::handlers::GetParams; use crate::response::ResponseFormat; #[derive(Debug, Clone)] diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index 6d26dd7fdb..a03f0942a9 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -167,6 +167,20 @@ impl Source for Config { } } +impl crate::handlers::StateConfig for Config { + fn rpc_client(&self) -> iroh_rpc_client::Config { + self.rpc_client.clone() + } + + fn port(&self) -> u16 { + self.port + } + + fn user_headers(&self) -> HeaderMap { + self.headers.clone() + } +} + fn collect_headers(headers: &HeaderMap) -> Result, ConfigError> { let mut map = Map::new(); for (key, value) in headers.iter() { diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 1469ce6266..fefb979ed3 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -1,52 +1,15 @@ -use async_recursion::async_recursion; -use axum::{ - body::{self, Body, HttpBody}, - error_handling::HandleErrorLayer, - extract::{Extension, Path, Query}, - http::{header::*, StatusCode}, - response::IntoResponse, - routing::get, - BoxError, Router, -}; -use bytes::Bytes; -use futures::TryStreamExt; -use handlebars::Handlebars; -use iroh_metrics::{gateway::Metrics, get_current_trace_id}; -use iroh_resolver::{ - resolver::{CidOrDomain, OutMetrics, UnixfsType}, - unixfs::Link, -}; +use axum::Router; +use iroh_metrics::gateway::Metrics; use iroh_rpc_client::Client as RpcClient; use iroh_rpc_types::gateway::GatewayServerAddr; use prometheus_client::registry::Registry; -use serde::{Deserialize, Serialize}; -use serde_json::{ - json, - value::{Map, Value as Json}, -}; -use serde_qs; -use std::{ - collections::HashMap, - error::Error, - fmt::Write, - sync::Arc, - time::{self, Duration}, -}; +use std::{collections::HashMap, sync::Arc}; use tokio::sync::RwLock; -use tower::ServiceBuilder; -use tower_http::trace::TraceLayer; -use tracing::info_span; -use url::Url; -use urlencoding::encode; use crate::{ bad_bits::BadBits, - client::{Client, FileResult, Request}, - config::Config, - constants::*, - error::GatewayError, - headers::*, - response::{get_response_format, GatewayResponse, ResponseFormat}, + client::Client, + handlers::{get_app_routes, StateConfig}, rpc, rpc::Gateway, templates, @@ -59,43 +22,16 @@ pub struct Core { #[derive(Debug)] pub struct State { - config: Config, - client: Client, - handlebars: HashMap, + pub config: Arc, + pub client: Client, + pub handlebars: HashMap, pub metrics: Metrics, - bad_bits: Arc>>, -} - -#[derive(Debug, Deserialize, Serialize, Clone)] -pub struct GetParams { - // todo(arqu): swap this for ResponseFormat - /// specifies the expected format of the response - format: Option, - /// specifies the desired filename of the response - filename: Option, - /// specifies whether the response should be of disposition inline or attachment - download: Option, - /// specifies whether the response should render a directory even if index.html is present - force_dir: Option, - /// uri query parameter for handling navigator.registerProtocolHandler Web API requests - uri: Option, - recursive: Option, -} - -impl GetParams { - pub fn to_query_string(&self) -> String { - let q = serde_qs::to_string(self).unwrap(); - if q.is_empty() { - q - } else { - format!("?{}", q) - } - } + pub bad_bits: Arc>>, } impl Core { pub async fn new( - config: Config, + config: Arc, rpc_addr: GatewayServerAddr, metrics: Metrics, registry: &mut Registry, @@ -105,7 +41,7 @@ impl Core { // TODO: handle error rpc::new(rpc_addr, Gateway::default()).await }); - let rpc_client = RpcClient::new(config.rpc_client.clone()).await?; + let rpc_client = RpcClient::new(config.rpc_client()).await?; let mut templates = HashMap::new(); templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); @@ -122,38 +58,45 @@ impl Core { }) } + pub async fn new_with_state( + rpc_addr: GatewayServerAddr, + state: Arc, + ) -> anyhow::Result { + tokio::spawn(async move { + // TODO: handle error + rpc::new(rpc_addr, Gateway::default()).await + }); + Ok(Self { state }) + } + + pub async fn make_state( + config: Arc, + metrics: Metrics, + registry: &mut Registry, + bad_bits: Arc>>, + ) -> anyhow::Result> { + let rpc_client = RpcClient::new(config.rpc_client()).await?; + let mut templates = HashMap::new(); + templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); + templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); + let client = Client::new(&rpc_client, registry); + Ok(Arc::new(State { + config, + client, + metrics, + handlebars: templates, + bad_bits, + })) + } + pub fn server( self, ) -> axum::Server> { - // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 - let app = Router::new() - .route("/:scheme/:cid", get(get_handler)) - .route("/:scheme/:cid/*cpath", get(get_handler)) - .route("/health", get(health_check)) - .layer(Extension(Arc::clone(&self.state))) - .layer( - ServiceBuilder::new() - // Handle errors from middleware - .layer(Extension(Arc::clone(&self.state))) - .layer(HandleErrorLayer::new(middleware_error_handler)) - .load_shed() - .concurrency_limit(2048) - .timeout(Duration::from_secs(60)) - .into_inner(), - ) - .layer( - // Tracing span for each request - TraceLayer::new_for_http().make_span_with(|request: &http::Request| { - info_span!( - "request", - method = %request.method(), - uri = %request.uri(), - ) - }), - ); + let app = get_app_routes(&self.state); + // todo(arqu): make configurable - let addr = format!("0.0.0.0:{}", self.state.config.port); + let addr = format!("0.0.0.0:{}", self.state.config.port()); axum::Server::bind(&addr.parse().unwrap()) .http1_preserve_header_case(true) @@ -162,504 +105,10 @@ impl Core { } } -#[tracing::instrument(skip(state))] -async fn get_handler( - Extension(state): Extension>, - Path(params): Path>, - Query(query_params): Query, - request_headers: HeaderMap, -) -> Result { - state.metrics.requests_total.inc(); - let start_time = time::Instant::now(); - // parse path params - let scheme = params.get("scheme").unwrap(); - if scheme != SCHEME_IPFS && scheme != SCHEME_IPNS { - return Err(error( - StatusCode::BAD_REQUEST, - "invalid scheme, must be ipfs or ipns", - &state, - )); - } - let cid = params.get("cid").unwrap(); - let cpath = "".to_string(); - let cpath = params.get("cpath").unwrap_or(&cpath); - let query_params_copy = query_params.clone(); - - let uri_param = query_params.uri.clone().unwrap_or_default(); - if !uri_param.is_empty() { - return protocol_handler_redirect(uri_param, &state); - } - service_worker_check(&request_headers, cpath.to_string(), &state)?; - unsuported_header_check(&request_headers, &state)?; - - if check_bad_bits(&state, cid, cpath).await { - return Err(error( - StatusCode::FORBIDDEN, - "CID is in the denylist", - &state, - )); - } - - let full_content_path = format!("/{}/{}{}", scheme, cid, cpath); - let resolved_path: iroh_resolver::resolver::Path = full_content_path - .parse() - .map_err(|e: anyhow::Error| e.to_string()) - .map_err(|e| error(StatusCode::BAD_REQUEST, &e, &state))?; - let resolved_cid = resolved_path.root(); - - if check_bad_bits(&state, resolved_cid.to_string().as_str(), cpath).await { - return Err(error( - StatusCode::FORBIDDEN, - "CID is in the denylist", - &state, - )); - } - - // parse query params - let format = match get_response_format(&request_headers, query_params.format) { - Ok(format) => format, - Err(err) => { - return Err(error(StatusCode::BAD_REQUEST, &err, &state)); - } - }; - - let query_file_name = query_params.filename.unwrap_or_default(); - let download = query_params.download.unwrap_or_default(); - let recursive = query_params.recursive.unwrap_or_default(); - - let mut headers = HeaderMap::new(); - - if let Some(resp) = etag_check(&request_headers, resolved_cid, &format, &state) { - return Ok(resp); - } - - // init headers - format.write_headers(&mut headers); - add_user_headers(&mut headers, state.config.headers.clone()); - headers.insert( - &HEADER_X_IPFS_PATH, - HeaderValue::from_str(&full_content_path).unwrap(), - ); - - // handle request and fetch data - let req = Request { - format, - cid: resolved_path.root().clone(), - resolved_path, - query_file_name, - content_path: full_content_path.to_string(), - download, - query_params: query_params_copy, - }; - - if recursive { - serve_car_recursive(&req, state, headers, start_time).await - } else { - match req.format { - ResponseFormat::Raw => serve_raw(&req, state, headers, start_time).await, - ResponseFormat::Car => serve_car(&req, state, headers, start_time).await, - ResponseFormat::Fs(_) => serve_fs(&req, state, headers, start_time).await, - } - } -} - -#[tracing::instrument()] -async fn health_check() -> String { - "OK".to_string() -} - -#[tracing::instrument()] -fn protocol_handler_redirect( - uri_param: String, - state: &State, -) -> Result { - let u = match Url::parse(&uri_param) { - Ok(u) => u, - Err(e) => { - return Err(error( - StatusCode::BAD_REQUEST, - &format!("invalid uri: {}", e), - state, - )); - } - }; - let uri_scheme = u.scheme(); - if uri_scheme != SCHEME_IPFS && uri_scheme != SCHEME_IPNS { - return Err(error( - StatusCode::BAD_REQUEST, - "invalid uri scheme, must be ipfs or ipns", - state, - )); - } - let mut uri_path = u.path().to_string(); - let uri_query = u.query(); - if uri_query.is_some() { - let encoded_query = encode(uri_query.unwrap()); - write!(uri_path, "?{}", encoded_query) - .map_err(|e| error(StatusCode::BAD_REQUEST, &e.to_string(), state))?; - } - let uri_host = u.host().unwrap().to_string(); - let redirect_uri = format!("{}://{}{}", uri_scheme, uri_host, uri_path); - Ok(GatewayResponse::redirect_permanently(&redirect_uri)) -} - -#[tracing::instrument()] -fn service_worker_check( - request_headers: &HeaderMap, - cpath: String, - state: &State, -) -> Result<(), GatewayError> { - if request_headers.contains_key(&HEADER_SERVICE_WORKER) { - let sw = request_headers.get(&HEADER_SERVICE_WORKER).unwrap(); - if sw.to_str().unwrap() == "script" && cpath.is_empty() { - return Err(error( - StatusCode::BAD_REQUEST, - "Service Worker not supported", - state, - )); - } - } - Ok(()) -} - -#[tracing::instrument()] -fn unsuported_header_check(request_headers: &HeaderMap, state: &State) -> Result<(), GatewayError> { - if request_headers.contains_key(&HEADER_X_IPFS_GATEWAY_PREFIX) { - return Err(error( - StatusCode::BAD_REQUEST, - "Unsupported HTTP header", - state, - )); - } - Ok(()) -} - -pub async fn check_bad_bits(state: &State, cid: &str, path: &str) -> bool { - // check if cid is in the denylist - if state.bad_bits.is_some() { - let bad_bits = state.bad_bits.as_ref(); - if let Some(bbits) = bad_bits { - if bbits.read().await.is_bad(cid, path) { - return true; - } - } - } - false -} - -#[tracing::instrument()] -fn etag_check( - request_headers: &HeaderMap, - resolved_cid: &CidOrDomain, - format: &ResponseFormat, - state: &State, -) -> Option { - if request_headers.contains_key("If-None-Match") { - // todo(arqu): handle dir etags - let cid_etag = get_etag(resolved_cid, Some(format.clone())); - let inm = request_headers - .get("If-None-Match") - .unwrap() - .to_str() - .unwrap(); - if etag_matches(inm, &cid_etag) { - return Some(GatewayResponse::not_modified()); - } - } - None -} - -#[tracing::instrument()] -async fn serve_raw( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: we currently only retrieve full cids - let (body, metadata) = state - .client - .get_file(req.resolved_path.clone(), start_time, &state.metrics) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - match body { - FileResult::File(body) => { - set_content_disposition_headers( - &mut headers, - format!("{}.bin", req.cid).as_str(), - DISPOSITION_ATTACHMENT, - ); - set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - add_cache_control_headers(&mut headers, metadata.clone()); - add_ipfs_roots_headers(&mut headers, metadata); - response(StatusCode::OK, body, headers) - } - FileResult::Directory(_) => Err(error( - StatusCode::INTERNAL_SERVER_ERROR, - "cannot serve directory as raw", - &state, - )), - } -} - -#[tracing::instrument()] -async fn serve_car( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: we currently only retrieve full cids - let (body, metadata) = state - .client - .get_file(req.resolved_path.clone(), start_time, &state.metrics) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - match body { - FileResult::File(body) => { - set_content_disposition_headers( - &mut headers, - format!("{}.car", req.cid).as_str(), - DISPOSITION_ATTACHMENT, - ); - - // todo(arqu): this should be root cid - let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); - set_etag_headers(&mut headers, etag); - // todo(arqu): check if etag matches for root cid - add_ipfs_roots_headers(&mut headers, metadata); - response(StatusCode::OK, body, headers) - } - FileResult::Directory(_) => Err(error( - StatusCode::INTERNAL_SERVER_ERROR, - "cannot serve directory as car file", - &state, - )), - } -} - -#[tracing::instrument()] -async fn serve_car_recursive( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: actually package as car file - - let body = state - .client - .clone() - .get_file_recursive(req.resolved_path.clone(), start_time, state.metrics.clone()) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - set_content_disposition_headers( - &mut headers, - format!("{}.car", req.cid).as_str(), - DISPOSITION_ATTACHMENT, - ); - - // todo(arqu): this should be root cid - let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); - set_etag_headers(&mut headers, etag); - // todo(arqu): check if etag matches for root cid - // add_ipfs_roots_headers(&mut headers, metadata); - response(StatusCode::OK, body, headers) -} - -#[tracing::instrument()] -#[async_recursion] -async fn serve_fs( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: we currently only retrieve full cids - let (body, metadata) = state - .client - .get_file(req.resolved_path.clone(), start_time, &state.metrics) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - add_ipfs_roots_headers(&mut headers, metadata.clone()); - match body { - FileResult::Directory(res) => { - let dir_list: anyhow::Result> = res - .unixfs_read_dir( - &state.client.resolver, - OutMetrics { - metrics: state.metrics.clone(), - start: start_time, - }, - ) - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string(), &state))? - .expect("already known this is a directory") - .try_collect() - .await; - match dir_list { - Ok(dir_list) => serve_fs_dir(&dir_list, req, state, headers, start_time).await, - Err(e) => { - tracing::warn!("failed to read dir: {:?}", e); - Err(error( - StatusCode::INTERNAL_SERVER_ERROR, - "failed to read dir listing", - &state, - )) - } - } - } - FileResult::File(body) => { - match metadata.unixfs_type { - Some(_) => { - // todo(arqu): error on no size - // todo(arqu): add lazy seeking - add_cache_control_headers(&mut headers, metadata.clone()); - set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - let name = add_content_disposition_headers( - &mut headers, - &req.query_file_name, - &req.content_path, - req.download, - ); - if metadata.unixfs_type == Some(UnixfsType::Symlink) { - headers.insert( - CONTENT_TYPE, - HeaderValue::from_str("inode/symlink").unwrap(), - ); - } else { - add_content_type_headers(&mut headers, &name); - } - response(StatusCode::OK, body, headers) - } - None => Err(error( - StatusCode::BAD_REQUEST, - "couldn't determine unixfs type", - &state, - )), - } - } - } -} - -#[tracing::instrument()] -async fn serve_fs_dir( - dir_list: &[Link], - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - let force_dir = req.query_params.force_dir.unwrap_or(false); - let has_index = dir_list.iter().any(|l| { - l.name - .as_ref() - .map(|l| l.starts_with("index.html")) - .unwrap_or_default() - }); - if !force_dir && has_index { - if !req.content_path.ends_with('/') { - let redirect_path = format!( - "{}/{}", - req.content_path, - req.query_params.to_query_string() - ); - return Ok(GatewayResponse::redirect(&redirect_path)); - } - let mut new_req = req.clone(); - new_req.resolved_path.push("index.html"); - new_req.content_path = format!("{}/index.html", req.content_path); - return serve_fs(&new_req, state, headers, start_time).await; - } - - headers.insert(CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap()); - // todo(arqu): set etag - // set_etag_headers(&mut headers, metadata.dir_hash.clone()); - - let mut template_data: Map = Map::new(); - let mut root_path = req.content_path.clone(); - if !root_path.ends_with('/') { - root_path.push('/'); - } - let links = dir_list - .iter() - .map(|l| { - let name = l.name.as_deref().unwrap_or_default(); - let mut link = Map::new(); - link.insert("name".to_string(), Json::String(get_filename(name))); - link.insert( - "path".to_string(), - Json::String(format!("{}{}", root_path, name)), - ); - link - }) - .collect::>>(); - template_data.insert("links".to_string(), json!(links)); - let reg = Handlebars::new(); - let dir_template = state.handlebars.get("dir_list").unwrap(); - let res = reg.render_template(dir_template, &template_data).unwrap(); - response(StatusCode::OK, Body::from(res), headers) -} - -#[tracing::instrument(skip(body))] -fn response( - status_code: StatusCode, - body: B, - headers: HeaderMap, -) -> Result -where - B: 'static + HttpBody + Send, - ::Error: Into>, -{ - Ok(GatewayResponse { - status_code, - body: body::boxed(body), - headers, - trace_id: get_current_trace_id().to_string(), - }) -} - -#[tracing::instrument()] -fn error(status_code: StatusCode, message: &str, state: &State) -> GatewayError { - state.metrics.error_count.inc(); - GatewayError { - status_code, - message: message.to_string(), - trace_id: get_current_trace_id().to_string(), - } -} - -#[tracing::instrument()] -async fn middleware_error_handler( - Extension(state): Extension>, - err: BoxError, -) -> impl IntoResponse { - state.metrics.fail_count.inc(); - if err.is::() { - return error(StatusCode::REQUEST_TIMEOUT, "request timed out", &state); - } - - if err.is::() { - return error( - StatusCode::SERVICE_UNAVAILABLE, - "service is overloaded, try again later", - &state, - ); - } - - return error( - StatusCode::INTERNAL_SERVER_ERROR, - format!("unhandled internal error: {}", err).as_str(), - &state, - ); -} - #[cfg(test)] mod tests { use super::*; + use crate::config::Config; use iroh_rpc_client::Config as RpcClientConfig; use prometheus_client::registry::Registry; @@ -682,7 +131,7 @@ mod tests { let gw_metrics = Metrics::new(&mut prom_registry); let rpc_addr = "grpc://0.0.0.0:0".parse().unwrap(); let handler = Core::new( - config, + Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry, @@ -705,7 +154,7 @@ mod tests { let client = hyper::Client::new(); let res = client.get(uri).await.unwrap(); - assert_eq!(StatusCode::OK, res.status()); + assert_eq!(http::StatusCode::OK, res.status()); let body = hyper::body::to_bytes(res.into_body()).await.unwrap(); assert_eq!(b"OK", &body[..]); core_task.abort(); diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs new file mode 100644 index 0000000000..d6cec630b8 --- /dev/null +++ b/iroh-gateway/src/handlers.rs @@ -0,0 +1,577 @@ +use async_recursion::async_recursion; +use axum::{ + body::{self, Body, HttpBody}, + error_handling::HandleErrorLayer, + extract::{Extension, Path, Query}, + http::{header::*, StatusCode}, + response::IntoResponse, + routing::get, + BoxError, Router, +}; +use bytes::Bytes; +use futures::TryStreamExt; +use handlebars::Handlebars; +use iroh_metrics::get_current_trace_id; +use iroh_resolver::{resolver::{CidOrDomain, UnixfsType, OutMetrics}, unixfs::Link}; +use serde::{Deserialize, Serialize}; +use serde_json::{ + json, + value::{Map, Value as Json}, +}; +use serde_qs; +use std::{ + collections::HashMap, + error::Error, + fmt::Write, + sync::Arc, + time::{self, Duration}, +}; +use tower::ServiceBuilder; +use tower_http::trace::TraceLayer; +use tracing::info_span; +use url::Url; +use urlencoding::encode; + +use crate::{ + client::{Request, FileResult}, + constants::*, + core::State, + error::GatewayError, + headers::*, + response::{get_response_format, GatewayResponse, ResponseFormat}, +}; + +/// Trait describing what needs to be accessed on the configuration +/// from the shared state. +pub trait StateConfig: std::fmt::Debug + Sync + Send { + fn rpc_client(&self) -> iroh_rpc_client::Config; + + fn port(&self) -> u16; + + fn user_headers(&self) -> HeaderMap; +} + +pub fn get_app_routes(state: &Arc) -> Router +where + T: Send + Sync + 'static +{ + // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 + Router::new() + .route("/:scheme/:cid", get(get_handler)) + .route("/:scheme/:cid/*cpath", get(get_handler)) + .route("/health", get(health_check)) + .layer(Extension(Arc::clone(&state))) + .layer( + ServiceBuilder::new() + // Handle errors from middleware + .layer(Extension(Arc::clone(&state))) + .layer(HandleErrorLayer::new(middleware_error_handler)) + .load_shed() + .concurrency_limit(2048) + .timeout(Duration::from_secs(60)) + .into_inner(), + ) + .layer( + // Tracing span for each request + TraceLayer::new_for_http().make_span_with(|request: &http::Request| { + info_span!( + "request", + method = %request.method(), + uri = %request.uri(), + ) + }), + ) +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct GetParams { + // todo(arqu): swap this for ResponseFormat + /// specifies the expected format of the response + format: Option, + /// specifies the desired filename of the response + filename: Option, + /// specifies whether the response should be of disposition inline or attachment + download: Option, + /// specifies whether the response should render a directory even if index.html is present + force_dir: Option, + /// uri query parameter for handling navigator.registerProtocolHandler Web API requests + uri: Option, + recursive: Option, +} + +impl GetParams { + pub fn to_query_string(&self) -> String { + let q = serde_qs::to_string(self).unwrap(); + if q.is_empty() { + q + } else { + format!("?{}", q) + } + } +} + +#[tracing::instrument(skip(state))] +pub async fn get_handler( + Extension(state): Extension>, + Path(params): Path>, + Query(query_params): Query, + request_headers: HeaderMap, +) -> Result { + state.metrics.requests_total.inc(); + let start_time = time::Instant::now(); + // parse path params + let scheme = params.get("scheme").unwrap(); + if scheme != SCHEME_IPFS && scheme != SCHEME_IPNS { + return Err(error( + StatusCode::BAD_REQUEST, + "invalid scheme, must be ipfs or ipns", + &state, + )); + } + let cid = params.get("cid").unwrap(); + let cpath = "".to_string(); + let cpath = params.get("cpath").unwrap_or(&cpath); + let query_params_copy = query_params.clone(); + + let uri_param = query_params.uri.clone().unwrap_or_default(); + if !uri_param.is_empty() { + return protocol_handler_redirect(uri_param, &state); + } + service_worker_check(&request_headers, cpath.to_string(), &state)?; + unsuported_header_check(&request_headers, &state)?; + + let full_content_path = format!("/{}/{}{}", scheme, cid, cpath); + let resolved_path: iroh_resolver::resolver::Path = full_content_path + .parse() + .map_err(|e: anyhow::Error| e.to_string()) + .map_err(|e| error(StatusCode::BAD_REQUEST, &e, &state))?; + let resolved_cid = resolved_path.root(); + + // parse query params + let format = match get_response_format(&request_headers, query_params.format) { + Ok(format) => format, + Err(err) => { + return Err(error(StatusCode::BAD_REQUEST, &err, &state)); + } + }; + + let query_file_name = query_params.filename.unwrap_or_default(); + let download = query_params.download.unwrap_or_default(); + let recursive = query_params.recursive.unwrap_or_default(); + + let mut headers = HeaderMap::new(); + + if let Some(resp) = etag_check(&request_headers, resolved_cid, &format, &state) { + return Ok(resp); + } + + // init headers + format.write_headers(&mut headers); + add_user_headers(&mut headers, state.config.user_headers()); + headers.insert( + &HEADER_X_IPFS_PATH, + HeaderValue::from_str(&full_content_path).unwrap(), + ); + + // handle request and fetch data + let req = Request { + format, + cid: resolved_path.root().clone(), + resolved_path, + query_file_name, + content_path: full_content_path.to_string(), + download, + query_params: query_params_copy, + }; + + if recursive { + serve_car_recursive(&req, state, headers, start_time).await + } else { + match req.format { + ResponseFormat::Raw => serve_raw(&req, state, headers, start_time).await, + ResponseFormat::Car => serve_car(&req, state, headers, start_time).await, + ResponseFormat::Fs(_) => serve_fs(&req, state, headers, start_time).await, + } + } +} + +#[tracing::instrument()] +pub async fn health_check() -> String { + "OK".to_string() +} + +#[tracing::instrument()] +fn protocol_handler_redirect( + uri_param: String, + state: &State, +) -> Result { + let u = match Url::parse(&uri_param) { + Ok(u) => u, + Err(e) => { + return Err(error( + StatusCode::BAD_REQUEST, + &format!("invalid uri: {}", e), + state, + )); + } + }; + let uri_scheme = u.scheme(); + if uri_scheme != SCHEME_IPFS && uri_scheme != SCHEME_IPNS { + return Err(error( + StatusCode::BAD_REQUEST, + "invalid uri scheme, must be ipfs or ipns", + state, + )); + } + let mut uri_path = u.path().to_string(); + let uri_query = u.query(); + if uri_query.is_some() { + let encoded_query = encode(uri_query.unwrap()); + write!(uri_path, "?{}", encoded_query) + .map_err(|e| error(StatusCode::BAD_REQUEST, &e.to_string(), state))?; + } + let uri_host = u.host().unwrap().to_string(); + let redirect_uri = format!("{}://{}{}", uri_scheme, uri_host, uri_path); + Ok(GatewayResponse::redirect_permanently(&redirect_uri)) +} + +#[tracing::instrument()] +fn service_worker_check( + request_headers: &HeaderMap, + cpath: String, + state: &State, +) -> Result<(), GatewayError> { + if request_headers.contains_key(&HEADER_SERVICE_WORKER) { + let sw = request_headers.get(&HEADER_SERVICE_WORKER).unwrap(); + if sw.to_str().unwrap() == "script" && cpath.is_empty() { + return Err(error( + StatusCode::BAD_REQUEST, + "Service Worker not supported", + state, + )); + } + } + Ok(()) +} + +#[tracing::instrument()] +fn unsuported_header_check(request_headers: &HeaderMap, state: &State) -> Result<(), GatewayError> { + if request_headers.contains_key(&HEADER_X_IPFS_GATEWAY_PREFIX) { + return Err(error( + StatusCode::BAD_REQUEST, + "Unsupported HTTP header", + state, + )); + } + Ok(()) +} + +#[tracing::instrument()] +fn etag_check( + request_headers: &HeaderMap, + resolved_cid: &CidOrDomain, + format: &ResponseFormat, + state: &State, +) -> Option { + if request_headers.contains_key("If-None-Match") { + // todo(arqu): handle dir etags + let cid_etag = get_etag(resolved_cid, Some(format.clone())); + let inm = request_headers + .get("If-None-Match") + .unwrap() + .to_str() + .unwrap(); + if etag_matches(inm, &cid_etag) { + return Some(GatewayResponse::not_modified()); + } + } + None +} + +#[tracing::instrument()] +async fn serve_raw( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: we currently only retrieve full cids + let (body, metadata) = state + .client + .get_file(req.resolved_path.clone(), start_time, &state.metrics) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + match body { + FileResult::File(body) => { + set_content_disposition_headers( + &mut headers, + format!("{}.bin", req.cid).as_str(), + DISPOSITION_ATTACHMENT, + ); + set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); + add_cache_control_headers(&mut headers, metadata.clone()); + add_ipfs_roots_headers(&mut headers, metadata); + response(StatusCode::OK, body, headers) + } + FileResult::Directory(_) => Err(error( + StatusCode::INTERNAL_SERVER_ERROR, + "cannot serve directory as raw", + &state, + )), + } +} + +#[tracing::instrument()] +async fn serve_car( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: we currently only retrieve full cids + let (body, metadata) = state + .client + .get_file(req.resolved_path.clone(), start_time, &state.metrics) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + match body { + FileResult::File(body) => { + set_content_disposition_headers( + &mut headers, + format!("{}.car", req.cid).as_str(), + DISPOSITION_ATTACHMENT, + ); + + // todo(arqu): this should be root cid + let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); + set_etag_headers(&mut headers, etag); + // todo(arqu): check if etag matches for root cid + add_ipfs_roots_headers(&mut headers, metadata); + response(StatusCode::OK, body, headers) + } + FileResult::Directory(_) => Err(error( + StatusCode::INTERNAL_SERVER_ERROR, + "cannot serve directory as car file", + &state, + )), + } +} + +#[tracing::instrument()] +async fn serve_car_recursive( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: actually package as car file + + let body = state + .client + .clone() + .get_file_recursive(req.resolved_path.clone(), start_time, state.metrics.clone()) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + set_content_disposition_headers( + &mut headers, + format!("{}.car", req.cid).as_str(), + DISPOSITION_ATTACHMENT, + ); + + // todo(arqu): this should be root cid + let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); + set_etag_headers(&mut headers, etag); + // todo(arqu): check if etag matches for root cid + // add_ipfs_roots_headers(&mut headers, metadata); + response(StatusCode::OK, body, headers) +} + +#[tracing::instrument()] +#[async_recursion] +async fn serve_fs( + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + // FIXME: we currently only retrieve full cids + let (body, metadata) = state + .client + .get_file(req.resolved_path.clone(), start_time, &state.metrics) + .await + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; + + add_ipfs_roots_headers(&mut headers, metadata.clone()); + match body { + FileResult::Directory(res) => { + let dir_list: anyhow::Result> = res + .unixfs_read_dir( + &state.client.resolver, + OutMetrics { + metrics: state.metrics.clone(), + start: start_time, + }, + ) + .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string(), &state))? + .expect("already known this is a directory") + .try_collect() + .await; + match dir_list { + Ok(dir_list) => serve_fs_dir(&dir_list, req, state, headers, start_time).await, + Err(e) => { + tracing::warn!("failed to read dir: {:?}", e); + Err(error( + StatusCode::INTERNAL_SERVER_ERROR, + "failed to read dir listing", + &state, + )) + } + } + } + FileResult::File(body) => { + match metadata.unixfs_type { + Some(_) => { + // todo(arqu): error on no size + // todo(arqu): add lazy seeking + add_cache_control_headers(&mut headers, metadata.clone()); + set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); + let name = add_content_disposition_headers( + &mut headers, + &req.query_file_name, + &req.content_path, + req.download, + ); + if metadata.unixfs_type == Some(UnixfsType::Symlink) { + headers.insert( + CONTENT_TYPE, + HeaderValue::from_str("inode/symlink").unwrap(), + ); + } else { + add_content_type_headers(&mut headers, &name); + } + response(StatusCode::OK, body, headers) + } + None => Err(error( + StatusCode::BAD_REQUEST, + "couldn't determine unixfs type", + &state, + )), + } + } + } +} + +#[tracing::instrument()] +async fn serve_fs_dir( + dir_list: &[Link], + req: &Request, + state: Arc, + mut headers: HeaderMap, + start_time: std::time::Instant, +) -> Result { + let force_dir = req.query_params.force_dir.unwrap_or(false); + let has_index = dir_list.iter().any(|l| { + l.name + .as_ref() + .map(|l| l.starts_with("index.html")) + .unwrap_or_default() + }); + if !force_dir && has_index { + if !req.content_path.ends_with('/') { + let redirect_path = format!( + "{}/{}", + req.content_path, + req.query_params.to_query_string() + ); + return Ok(GatewayResponse::redirect(&redirect_path)); + } + let mut new_req = req.clone(); + new_req.resolved_path.push("index.html"); + new_req.content_path = format!("{}/index.html", req.content_path); + return serve_fs(&new_req, state, headers, start_time).await; + } + + headers.insert(CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap()); + // todo(arqu): set etag + // set_etag_headers(&mut headers, metadata.dir_hash.clone()); + + let mut template_data: Map = Map::new(); + let mut root_path = req.content_path.clone(); + if !root_path.ends_with('/') { + root_path.push('/'); + } + let links = dir_list + .iter() + .map(|l| { + let name = l.name.as_deref().unwrap_or_default(); + let mut link = Map::new(); + link.insert("name".to_string(), Json::String(get_filename(name))); + link.insert( + "path".to_string(), + Json::String(format!("{}{}", root_path, name)), + ); + link + }) + .collect::>>(); + template_data.insert("links".to_string(), json!(links)); + let reg = Handlebars::new(); + let dir_template = state.handlebars.get("dir_list").unwrap(); + let res = reg.render_template(dir_template, &template_data).unwrap(); + response(StatusCode::OK, Body::from(res), headers) +} + +#[tracing::instrument(skip(body))] +fn response( + status_code: StatusCode, + body: B, + headers: HeaderMap, +) -> Result +where + B: 'static + HttpBody + Send, + ::Error: Into>, +{ + Ok(GatewayResponse { + status_code, + body: body::boxed(body), + headers, + trace_id: get_current_trace_id().to_string(), + }) +} + +#[tracing::instrument()] +fn error(status_code: StatusCode, message: &str, state: &State) -> GatewayError { + state.metrics.error_count.inc(); + GatewayError { + status_code, + message: message.to_string(), + trace_id: get_current_trace_id().to_string(), + } +} + +#[tracing::instrument()] +pub async fn middleware_error_handler( + Extension(state): Extension>, + err: BoxError, +) -> impl IntoResponse { + state.metrics.fail_count.inc(); + if err.is::() { + return error(StatusCode::REQUEST_TIMEOUT, "request timed out", &state); + } + + if err.is::() { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "service is overloaded, try again later", + &state, + ); + } + + return error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("unhandled internal error: {}", err).as_str(), + &state, + ); +} diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index 6d86f0b706..c8036de8ad 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -1,9 +1,10 @@ pub mod bad_bits; -mod client; +pub mod client; pub mod config; pub mod constants; pub mod core; mod error; +pub mod handlers; pub mod headers; pub mod metrics; pub mod response; diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index 7116cfb294..f10a1f2932 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -88,7 +88,7 @@ async fn main() -> Result<()> { .server_rpc_addr()? .ok_or_else(|| anyhow!("missing gateway rpc addr"))?; let handler = Core::new( - config, + Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry, diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index d2bd581685..f804164ea5 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -186,6 +186,20 @@ impl Source for Config { } } +impl iroh_gateway::handlers::StateConfig for Config { + fn rpc_client(&self) -> iroh_rpc_client::Config { + self.rpc_client.clone() + } + + fn port(&self) -> u16 { + self.port + } + + fn user_headers(&self) -> HeaderMap { + self.headers.clone() + } +} + fn collect_headers(headers: &HeaderMap) -> Result, ConfigError> { let mut map = Map::new(); for (key, value) in headers.iter() { diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 0698bb8b96..0f1c1cfc2f 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -1,157 +1,57 @@ -use crate::{ - client::{Client, Request}, - config::Config, - error::GatewayError, - rpc, - rpc::Gateway, - templates, uds, -}; -use async_recursion::async_recursion; -use axum::{ - body::{self, Body, HttpBody}, - error_handling::HandleErrorLayer, - extract::{Extension, Path, Query}, - http::{header::*, StatusCode}, - response::IntoResponse, - routing::get, - BoxError, Router, Server, -}; -use bytes::Bytes; -use handlebars::Handlebars; +use crate::{rpc, rpc::Gateway, templates, uds}; +use axum::{Router, Server}; use iroh_gateway::{ - constants::*, - headers::*, - response::{get_response_format, GatewayResponse, ResponseFormat}, + client::Client, + core::State, + handlers::{get_app_routes, StateConfig}, bad_bits::BadBits, }; -use iroh_metrics::{gateway::Metrics, get_current_trace_id}; -use iroh_resolver::resolver::{CidOrDomain, UnixfsType}; +use iroh_metrics::gateway::Metrics; use iroh_rpc_client::Client as RpcClient; use iroh_rpc_types::gateway::GatewayServerAddr; use prometheus_client::registry::Registry; -use serde::{Deserialize, Serialize}; -use serde_json::{ - json, - value::{Map, Value as Json}, -}; -use serde_qs; -use std::{ - collections::HashMap, - error::Error, - fmt::Write, - sync::Arc, - time::{self, Duration}, -}; -use tokio::net::UnixListener; -use tower::ServiceBuilder; -use tower_http::trace::TraceLayer; -use tracing::info_span; -use url::Url; -use urlencoding::encode; +use std::{collections::HashMap, sync::Arc}; +use tokio::{net::UnixListener, sync::RwLock}; #[derive(Debug)] pub struct Core { state: Arc, } -#[derive(Debug)] -pub struct State { - config: Config, - client: Client, - rpc_client: iroh_rpc_client::Client, - handlebars: HashMap, - pub metrics: Metrics, -} - -#[derive(Debug, Deserialize, Serialize, Clone)] -pub struct GetParams { - // todo(arqu): swap this for ResponseFormat - /// specifies the expected format of the response - format: Option, - /// specifies the desired filename of the response - filename: Option, - /// specifies whether the response should be of disposition inline or attachment - download: Option, - /// specifies whether the response should render a directory even if index.html is present - force_dir: Option, - /// uri query parameter for handling navigator.registerProtocolHandler Web API requests - uri: Option, - recursive: Option, -} - -impl GetParams { - pub fn to_query_string(&self) -> String { - let q = serde_qs::to_string(self).unwrap(); - if q.is_empty() { - q - } else { - format!("?{}", q) - } - } -} - impl Core { pub async fn new( - config: Config, + config: Arc, rpc_addr: GatewayServerAddr, metrics: Metrics, registry: &mut Registry, + bad_bits: Arc>>, ) -> anyhow::Result { tokio::spawn(async move { // TODO: handle error rpc::new(rpc_addr, Gateway::default()).await }); - let rpc_client = RpcClient::new(config.rpc_client.clone()).await?; + let rpc_client = RpcClient::new(config.rpc_client()).await?; let mut templates = HashMap::new(); templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); let client = Client::new(&rpc_client, registry); Ok(Self { - state: Arc::new(State { + state: Arc::new(iroh_gateway::core::State { config, client, - rpc_client, metrics, handlebars: templates, + bad_bits, }), }) } - fn get_app(&self) -> Router { - // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 - Router::new() - .route("/:scheme/:cid", get(get_handler)) - .route("/:scheme/:cid/*cpath", get(get_handler)) - .route("/health", get(health_check)) - .layer(Extension(Arc::clone(&self.state))) - .layer( - ServiceBuilder::new() - // Handle errors from middleware - .layer(Extension(Arc::clone(&self.state))) - .layer(HandleErrorLayer::new(middleware_error_handler)) - .load_shed() - .concurrency_limit(2048) - .timeout(Duration::from_secs(60)) - .into_inner(), - ) - .layer( - // Tracing span for each request - TraceLayer::new_for_http().make_span_with(|request: &http::Request| { - info_span!( - "request", - method = %request.method(), - uri = %request.uri(), - ) - }), - ) - } - pub fn http_server( &self, ) -> Server> { - let app = self.get_app(); + let app = get_app_routes(&self.state); // todo(arqu): make configurable - let addr = format!("0.0.0.0:{}", self.state.config.port); + let addr = format!("0.0.0.0:{}", self.state.config.port()); Server::bind(&addr.parse().unwrap()) .http1_preserve_header_case(true) @@ -174,472 +74,16 @@ impl Core { let _ = std::fs::remove_file(&path); let uds = UnixListener::bind(&path).unwrap(); println!("Binding to UDS at {}", path); - let app = self.get_app(); + let app = get_app_routes(&self.state); Server::builder(uds::ServerAccept { uds }) .serve(app.into_make_service_with_connect_info::()) } } -#[tracing::instrument(skip(state))] -async fn get_handler( - Extension(state): Extension>, - Path(params): Path>, - Query(query_params): Query, - request_headers: HeaderMap, -) -> Result { - state.metrics.requests_total.inc(); - let start_time = time::Instant::now(); - // parse path params - let scheme = params.get("scheme").unwrap(); - if scheme != SCHEME_IPFS && scheme != SCHEME_IPNS { - return Err(error( - StatusCode::BAD_REQUEST, - "invalid scheme, must be ipfs or ipns", - &state, - )); - } - let cid = params.get("cid").unwrap(); - let cpath = "".to_string(); - let cpath = params.get("cpath").unwrap_or(&cpath); - let query_params_copy = query_params.clone(); - - let uri_param = query_params.uri.clone().unwrap_or_default(); - if !uri_param.is_empty() { - return protocol_handler_redirect(uri_param, &state); - } - service_worker_check(&request_headers, cpath.to_string(), &state)?; - unsuported_header_check(&request_headers, &state)?; - - let full_content_path = format!("/{}/{}{}", scheme, cid, cpath); - let resolved_path: iroh_resolver::resolver::Path = full_content_path - .parse() - .map_err(|e: anyhow::Error| e.to_string()) - .map_err(|e| error(StatusCode::BAD_REQUEST, &e, &state))?; - let resolved_cid = resolved_path.root(); - - // parse query params - let format = match get_response_format(&request_headers, query_params.format) { - Ok(format) => format, - Err(err) => { - return Err(error(StatusCode::BAD_REQUEST, &err, &state)); - } - }; - - let query_file_name = query_params.filename.unwrap_or_default(); - let download = query_params.download.unwrap_or_default(); - let recursive = query_params.recursive.unwrap_or_default(); - - let mut headers = HeaderMap::new(); - - if let Some(resp) = etag_check(&request_headers, resolved_cid, &format, &state) { - return Ok(resp); - } - - // init headers - format.write_headers(&mut headers); - add_user_headers(&mut headers, state.config.headers.clone()); - headers.insert( - &HEADER_X_IPFS_PATH, - HeaderValue::from_str(&full_content_path).unwrap(), - ); - - // handle request and fetch data - let req = Request { - format, - cid: resolved_path.root().clone(), - resolved_path, - query_file_name, - content_path: full_content_path.to_string(), - download, - query_params: query_params_copy, - }; - - if recursive { - serve_car_recursive(&req, state, headers, start_time).await - } else { - match req.format { - ResponseFormat::Raw => serve_raw(&req, state, headers, start_time).await, - ResponseFormat::Car => serve_car(&req, state, headers, start_time).await, - ResponseFormat::Fs(_) => serve_fs(&req, state, headers, start_time).await, - } - } -} - -#[tracing::instrument()] -async fn health_check() -> String { - "OK".to_string() -} - -#[tracing::instrument()] -fn protocol_handler_redirect( - uri_param: String, - state: &State, -) -> Result { - let u = match Url::parse(&uri_param) { - Ok(u) => u, - Err(e) => { - return Err(error( - StatusCode::BAD_REQUEST, - &format!("invalid uri: {}", e), - state, - )); - } - }; - let uri_scheme = u.scheme(); - if uri_scheme != SCHEME_IPFS && uri_scheme != SCHEME_IPNS { - return Err(error( - StatusCode::BAD_REQUEST, - "invalid uri scheme, must be ipfs or ipns", - state, - )); - } - let mut uri_path = u.path().to_string(); - let uri_query = u.query(); - if uri_query.is_some() { - let encoded_query = encode(uri_query.unwrap()); - write!(uri_path, "?{}", encoded_query) - .map_err(|e| error(StatusCode::BAD_REQUEST, &e.to_string(), state))?; - } - let uri_host = u.host().unwrap().to_string(); - let redirect_uri = format!("{}://{}{}", uri_scheme, uri_host, uri_path); - Ok(GatewayResponse::redirect_permanently(&redirect_uri)) -} - -#[tracing::instrument()] -fn service_worker_check( - request_headers: &HeaderMap, - cpath: String, - state: &State, -) -> Result<(), GatewayError> { - if request_headers.contains_key(&HEADER_SERVICE_WORKER) { - let sw = request_headers.get(&HEADER_SERVICE_WORKER).unwrap(); - if sw.to_str().unwrap() == "script" && cpath.is_empty() { - return Err(error( - StatusCode::BAD_REQUEST, - "Service Worker not supported", - state, - )); - } - } - Ok(()) -} - -#[tracing::instrument()] -fn unsuported_header_check(request_headers: &HeaderMap, state: &State) -> Result<(), GatewayError> { - if request_headers.contains_key(&HEADER_X_IPFS_GATEWAY_PREFIX) { - return Err(error( - StatusCode::BAD_REQUEST, - "Unsupported HTTP header", - state, - )); - } - Ok(()) -} - -#[tracing::instrument()] -fn etag_check( - request_headers: &HeaderMap, - resolved_cid: &CidOrDomain, - format: &ResponseFormat, - state: &State, -) -> Option { - if request_headers.contains_key("If-None-Match") { - // todo(arqu): handle dir etags - let cid_etag = get_etag(resolved_cid, Some(format.clone())); - let inm = request_headers - .get("If-None-Match") - .unwrap() - .to_str() - .unwrap(); - if etag_matches(inm, &cid_etag) { - return Some(GatewayResponse::not_modified()); - } - } - None -} - -#[tracing::instrument()] -async fn serve_raw( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: we currently only retrieve full cids - let (body, metadata) = state - .client - .get_file( - req.resolved_path.clone(), - &state.rpc_client, - start_time, - &state.metrics, - ) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - set_content_disposition_headers( - &mut headers, - format!("{}.bin", req.cid).as_str(), - DISPOSITION_ATTACHMENT, - ); - set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - add_cache_control_headers(&mut headers, metadata.clone()); - add_ipfs_roots_headers(&mut headers, metadata); - response(StatusCode::OK, body, headers) -} - -#[tracing::instrument()] -async fn serve_car( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: we currently only retrieve full cids - let (body, metadata) = state - .client - .get_file( - req.resolved_path.clone(), - &state.rpc_client, - start_time, - &state.metrics, - ) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - set_content_disposition_headers( - &mut headers, - format!("{}.car", req.cid).as_str(), - DISPOSITION_ATTACHMENT, - ); - - // todo(arqu): this should be root cid - let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); - set_etag_headers(&mut headers, etag); - // todo(arqu): check if etag matches for root cid - add_ipfs_roots_headers(&mut headers, metadata); - response(StatusCode::OK, body, headers) -} - -#[tracing::instrument()] -async fn serve_car_recursive( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: actually package as car file - - let body = state - .client - .clone() - .get_file_recursive( - req.resolved_path.clone(), - state.rpc_client.clone(), - start_time, - state.metrics.clone(), - ) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - set_content_disposition_headers( - &mut headers, - format!("{}.car", req.cid).as_str(), - DISPOSITION_ATTACHMENT, - ); - - // todo(arqu): this should be root cid - let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); - set_etag_headers(&mut headers, etag); - // todo(arqu): check if etag matches for root cid - // add_ipfs_roots_headers(&mut headers, metadata); - response(StatusCode::OK, body, headers) -} - -#[tracing::instrument()] -#[async_recursion] -async fn serve_fs( - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - // FIXME: we currently only retrieve full cids - let (mut body, metadata) = state - .client - .get_file( - req.resolved_path.clone(), - &state.rpc_client, - start_time, - &state.metrics, - ) - .await - .map_err(|e| error(StatusCode::INTERNAL_SERVER_ERROR, &e, &state))?; - - add_ipfs_roots_headers(&mut headers, metadata.clone()); - match metadata.unixfs_type { - Some(UnixfsType::Dir) => { - if let Some(dir_list_data) = body.data().await { - let dir_list = match dir_list_data { - Ok(b) => b, - Err(_) => { - return Err(error( - StatusCode::INTERNAL_SERVER_ERROR, - "failed to read dir listing", - &state, - )); - } - }; - return serve_fs_dir(&dir_list, req, state, headers, start_time).await; - } else { - return Err(error( - StatusCode::INTERNAL_SERVER_ERROR, - "failed to read dir listing", - &state, - )); - } - } - Some(_) => { - // todo(arqu): error on no size - // todo(arqu): add lazy seeking - add_cache_control_headers(&mut headers, metadata.clone()); - set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - let name = add_content_disposition_headers( - &mut headers, - &req.query_file_name, - &req.content_path, - req.download, - ); - if metadata.unixfs_type == Some(UnixfsType::Symlink) { - headers.insert( - CONTENT_TYPE, - HeaderValue::from_str("inode/symlink").unwrap(), - ); - } else { - add_content_type_headers(&mut headers, &name); - } - } - None => { - return Err(error( - StatusCode::BAD_REQUEST, - "couldn't determine unixfs type", - &state, - )); - } - } - response(StatusCode::OK, body, headers) -} - -#[tracing::instrument()] -async fn serve_fs_dir( - dir_list: &Bytes, - req: &Request, - state: Arc, - mut headers: HeaderMap, - start_time: std::time::Instant, -) -> Result { - let dir_list = std::str::from_utf8(&dir_list[..]).unwrap(); - let force_dir = req.query_params.force_dir.unwrap_or(false); - let has_index = dir_list.lines().any(|l| l.starts_with("index.html")); - if !force_dir && has_index { - if !req.content_path.ends_with('/') { - let redirect_path = format!( - "{}/{}", - req.content_path, - req.query_params.to_query_string() - ); - return Ok(GatewayResponse::redirect(&redirect_path)); - } - let mut new_req = req.clone(); - new_req.resolved_path.push("index.html"); - new_req.content_path = format!("{}/index.html", req.content_path); - return serve_fs(&new_req, state, headers, start_time).await; - } - - headers.insert(CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap()); - // todo(arqu): set etag - // set_etag_headers(&mut headers, metadata.dir_hash.clone()); - - let mut template_data: Map = Map::new(); - let mut root_path = req.content_path.clone(); - if !root_path.ends_with('/') { - root_path.push('/'); - } - let links = dir_list - .lines() - .map(|line| { - let mut link = Map::new(); - link.insert("name".to_string(), Json::String(get_filename(line))); - link.insert( - "path".to_string(), - Json::String(format!("{}{}", root_path, line)), - ); - link - }) - .collect::>>(); - template_data.insert("links".to_string(), json!(links)); - let reg = Handlebars::new(); - let dir_template = state.handlebars.get("dir_list").unwrap(); - let res = reg.render_template(dir_template, &template_data).unwrap(); - response(StatusCode::OK, Body::from(res), headers) -} - -#[tracing::instrument(skip(body))] -fn response( - status_code: StatusCode, - body: B, - headers: HeaderMap, -) -> Result -where - B: 'static + HttpBody + Send, - ::Error: Into>, -{ - Ok(GatewayResponse { - status_code, - body: body::boxed(body), - headers, - trace_id: get_current_trace_id().to_string(), - }) -} - -#[tracing::instrument()] -fn error(status_code: StatusCode, message: &str, state: &State) -> GatewayError { - state.metrics.error_count.inc(); - GatewayError { - status_code, - message: message.to_string(), - trace_id: get_current_trace_id().to_string(), - } -} - -#[tracing::instrument()] -async fn middleware_error_handler( - Extension(state): Extension>, - err: BoxError, -) -> impl IntoResponse { - state.metrics.fail_count.inc(); - if err.is::() { - return error(StatusCode::REQUEST_TIMEOUT, "request timed out", &state); - } - - if err.is::() { - return error( - StatusCode::SERVICE_UNAVAILABLE, - "service is overloaded, try again later", - &state, - ); - } - - return error( - StatusCode::INTERNAL_SERVER_ERROR, - format!("unhandled internal error: {}", err).as_str(), - &state, - ); -} - #[cfg(test)] mod tests { use super::*; + use crate::config::Config; use iroh_rpc_client::Config as RpcClientConfig; use prometheus_client::registry::Registry; @@ -661,7 +105,7 @@ mod tests { let mut prom_registry = Registry::default(); let gw_metrics = Metrics::new(&mut prom_registry); let rpc_addr = "grpc://0.0.0.0:0".parse().unwrap(); - let handler = Core::new(config, rpc_addr, gw_metrics, &mut prom_registry) + let handler = Core::new(Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry) .await .unwrap(); let server = handler.http_server(); @@ -679,7 +123,7 @@ mod tests { let client = hyper::Client::new(); let res = client.get(uri).await.unwrap(); - assert_eq!(StatusCode::OK, res.status()); + assert_eq!(http::StatusCode::OK, res.status()); let body = hyper::body::to_bytes(res.into_body()).await.unwrap(); assert_eq!(b"OK", &body[..]); core_task.abort(); diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs index 66f7b75172..748a6aab11 100644 --- a/iroh-one/src/lib.rs +++ b/iroh-one/src/lib.rs @@ -1,4 +1,3 @@ -mod client; pub mod config; pub mod core; mod error; diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index f7e1043a59..68ca5b05d8 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; @@ -90,7 +91,7 @@ async fn main() -> Result<()> { let rpc_addr = config .server_rpc_addr()? .ok_or_else(|| anyhow!("missing gateway rpc addr"))?; - let handler = Core::new(config, rpc_addr, gw_metrics, &mut prom_registry).await?; + let handler = Core::new(Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry).await?; let metrics_handle = iroh_metrics::MetricsHandle::from_registry_with_tracer(metrics_config, prom_registry) From f345f989a1a13fbfdf8e9d91c389dd1238868034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 12:19:30 -0700 Subject: [PATCH 08/29] iroh-one: remove error and metrics --- iroh-one/src/error.rs | 24 ------------------------ iroh-one/src/lib.rs | 2 -- iroh-one/src/main.rs | 2 +- iroh-one/src/metrics.rs | 9 --------- 4 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 iroh-one/src/error.rs delete mode 100644 iroh-one/src/metrics.rs diff --git a/iroh-one/src/error.rs b/iroh-one/src/error.rs deleted file mode 100644 index 3494912ed3..0000000000 --- a/iroh-one/src/error.rs +++ /dev/null @@ -1,24 +0,0 @@ -use axum::{ - http::StatusCode, - response::{IntoResponse, Response}, -}; -use serde_json::json; - -#[derive(Debug)] -pub struct GatewayError { - pub status_code: StatusCode, - pub message: String, - pub trace_id: String, -} - -impl IntoResponse for GatewayError { - fn into_response(self) -> Response { - let body = axum::Json(json!({ - "code": self.status_code.as_u16(), - "success": false, - "message": self.message, - "trace_id": self.trace_id, - })); - (self.status_code, body).into_response() - } -} diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs index 748a6aab11..76479ccdec 100644 --- a/iroh-one/src/lib.rs +++ b/iroh-one/src/lib.rs @@ -1,9 +1,7 @@ pub mod config; pub mod core; -mod error; pub mod mem_p2p; pub mod mem_store; -pub mod metrics; mod rpc; mod templates; mod uds; diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 68ca5b05d8..91d3842490 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; +use iroh_gateway::metrics; use iroh_metrics::gateway::Metrics; use iroh_one::{ config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, core::Core, - metrics, }; use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; diff --git a/iroh-one/src/metrics.rs b/iroh-one/src/metrics.rs deleted file mode 100644 index 20ac886927..0000000000 --- a/iroh-one/src/metrics.rs +++ /dev/null @@ -1,9 +0,0 @@ -use git_version::git_version; -use iroh_metrics::config::Config as MetricsConfig; - -pub fn metrics_config_with_compile_time_info(cfg: MetricsConfig) -> MetricsConfig { - // compile time configuration - cfg.with_service_name(env!("CARGO_PKG_NAME").to_string()) - .with_build(git_version!().to_string()) - .with_version(env!("CARGO_PKG_VERSION").to_string()) -} From 0b46c89ff218a8f2ee69ae94e8cfdf9c4d34b28d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 12:23:24 -0700 Subject: [PATCH 09/29] iroh-one: share templates with iroh-gateway --- iroh-gateway/src/lib.rs | 2 +- iroh-one/src/client.rs | 150 -------------------------------------- iroh-one/src/core.rs | 3 +- iroh-one/src/lib.rs | 1 - iroh-one/src/templates.rs | 19 ----- 5 files changed, 3 insertions(+), 172 deletions(-) delete mode 100644 iroh-one/src/client.rs delete mode 100644 iroh-one/src/templates.rs diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index c8036de8ad..166ba3486d 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -9,4 +9,4 @@ pub mod headers; pub mod metrics; pub mod response; mod rpc; -mod templates; +pub mod templates; diff --git a/iroh-one/src/client.rs b/iroh-one/src/client.rs deleted file mode 100644 index 023714154c..0000000000 --- a/iroh-one/src/client.rs +++ /dev/null @@ -1,150 +0,0 @@ -use std::sync::Arc; - -use axum::body::StreamBody; -use futures::StreamExt; -use iroh_metrics::gateway::Metrics; -use iroh_resolver::resolver::CidOrDomain; -use iroh_resolver::resolver::Metadata; -use iroh_resolver::resolver::OutMetrics; -use iroh_resolver::resolver::OutPrettyReader; -use iroh_resolver::resolver::Resolver; -use iroh_resolver::resolver::Source; -use prometheus_client::registry::Registry; -use tokio::io::AsyncReadExt; -use tokio_util::io::ReaderStream; -use tracing::info; -use tracing::warn; - -use crate::core::GetParams; -use iroh_gateway::response::ResponseFormat; - -#[derive(Debug, Clone)] -pub struct Client { - resolver: Arc>, -} - -pub type PrettyStreamBody = StreamBody>>; - -impl Client { - pub fn new(rpc_client: &iroh_rpc_client::Client, registry: &mut Registry) -> Self { - Self { - resolver: Arc::new(Resolver::new(rpc_client.clone(), registry)), - } - } - - #[tracing::instrument(skip(self, rpc_client, metrics))] - pub async fn get_file( - &self, - path: iroh_resolver::resolver::Path, - rpc_client: &iroh_rpc_client::Client, - start_time: std::time::Instant, - metrics: &Metrics, - ) -> Result<(PrettyStreamBody, Metadata), String> { - info!("get file {}", path); - let res = self - .resolver - .resolve(path) - .await - .map_err(|e| e.to_string())?; - metrics - .ttf_block - .set(start_time.elapsed().as_millis() as u64); - let metadata = res.metadata().clone(); - if metadata.source == Source::Bitswap { - metrics - .hist_ttfb - .observe(start_time.elapsed().as_millis() as f64); - } else { - metrics - .hist_ttfb_cached - .observe(start_time.elapsed().as_millis() as f64); - } - let reader = res - .pretty( - rpc_client.clone(), - OutMetrics { - metrics: metrics.clone(), - start: start_time, - }, - ) - .map_err(|e| e.to_string())?; - let stream = ReaderStream::new(reader); - let body = StreamBody::new(stream); - - Ok((body, metadata)) - } - - #[tracing::instrument(skip(self, rpc_client, metrics))] - pub async fn get_file_recursive( - self, - path: iroh_resolver::resolver::Path, - rpc_client: iroh_rpc_client::Client, - start_time: std::time::Instant, - metrics: Metrics, - ) -> Result { - info!("get file {}", path); - let (mut sender, body) = axum::body::Body::channel(); - - tokio::spawn(async move { - let res = self.resolver.resolve_recursive(path); - tokio::pin!(res); - - while let Some(res) = res.next().await { - match res { - Ok(res) => { - metrics - .ttf_block - .set(start_time.elapsed().as_millis() as u64); - let metadata = res.metadata().clone(); - if metadata.source == Source::Bitswap { - metrics - .hist_ttfb - .observe(start_time.elapsed().as_millis() as f64); - } else { - metrics - .hist_ttfb_cached - .observe(start_time.elapsed().as_millis() as f64); - } - let reader = res.pretty( - rpc_client.clone(), - OutMetrics { - metrics: metrics.clone(), - start: start_time, - }, - ); - match reader { - Ok(mut reader) => { - let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes).await.unwrap(); - sender.send_data(bytes.into()).await.unwrap(); - } - Err(e) => { - warn!("failed to load recursively: {:?}", e); - sender.abort(); - break; - } - } - } - Err(e) => { - warn!("failed to load recursively: {:?}", e); - sender.abort(); - break; - } - } - } - }); - - Ok(body) - } -} - -#[derive(Debug, Clone)] -pub struct Request { - pub format: ResponseFormat, - pub cid: CidOrDomain, - pub resolved_path: iroh_resolver::resolver::Path, - pub query_file_name: String, - pub content_path: String, - pub download: bool, - pub query_params: GetParams, -} diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 0f1c1cfc2f..987f906646 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -1,9 +1,10 @@ -use crate::{rpc, rpc::Gateway, templates, uds}; +use crate::{rpc, rpc::Gateway, uds}; use axum::{Router, Server}; use iroh_gateway::{ client::Client, core::State, handlers::{get_app_routes, StateConfig}, bad_bits::BadBits, + templates, }; use iroh_metrics::gateway::Metrics; use iroh_rpc_client::Client as RpcClient; diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs index 76479ccdec..c66279d346 100644 --- a/iroh-one/src/lib.rs +++ b/iroh-one/src/lib.rs @@ -3,5 +3,4 @@ pub mod core; pub mod mem_p2p; pub mod mem_store; mod rpc; -mod templates; mod uds; diff --git a/iroh-one/src/templates.rs b/iroh-one/src/templates.rs deleted file mode 100644 index 2a3c596a6c..0000000000 --- a/iroh-one/src/templates.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub const DIR_LIST: &str = " -
-
    - {{#each links}} -
  1. - {{this.name}} -
    2. - {{/each}} -
-
"; - -pub const NOT_FOUND: &str = " -
-

404 Not Found

-

- The requested resource was not found. -

-
-"; From 9d888d9ee73dbe43c81b0892268076fe97626c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 12:50:11 -0700 Subject: [PATCH 10/29] iroh-gateway: No need for a fully generic state when creating routes --- iroh-gateway/src/handlers.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index d6cec630b8..e38c933368 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -51,10 +51,7 @@ pub trait StateConfig: std::fmt::Debug + Sync + Send { fn user_headers(&self) -> HeaderMap; } -pub fn get_app_routes(state: &Arc) -> Router -where - T: Send + Sync + 'static -{ +pub fn get_app_routes(state: &Arc) -> Router { // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 Router::new() .route("/:scheme/:cid", get(get_handler)) From b0e316e5d58fc80268649240f22b959ea538a7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 14:07:32 -0700 Subject: [PATCH 11/29] iroh-one: cleanup Cargo.toml --- iroh-one/Cargo.toml | 25 +------------------------ iroh-one/src/config.rs | 4 ++-- iroh-one/src/uds.rs | 4 ++-- 3 files changed, 5 insertions(+), 28 deletions(-) diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index 8736f648d4..ac5439360b 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -9,19 +9,12 @@ version = "0.1.0" [dependencies] anyhow = "1" -async-recursion = "1.0.0" async-trait = "0.1.56" axum = "0.5.1" -bytes = "1.1.0" -cid = "0.8.4" clap = {version = "3.1.14", features = ["derive"]} config = "0.13.1" -dirs = "4.0.0" futures = "0.3.21" -git-version = "0.3.5" -handlebars = "4" headers = "0.3.7" -http = "0.2" http-serde = "1.1.0" hyper = "0.14.19" iroh-gateway = {path = "../iroh-gateway"} @@ -32,30 +25,14 @@ iroh-rpc-client = {path = "../iroh-rpc-client", default-features = false} iroh-rpc-types = {path = "../iroh-rpc-types", default-features = false} iroh-store = {path = "../iroh-store", default-features = false, features = ["rpc-mem"]} iroh-util = {path = "../iroh-util"} -libp2p = {version = "0.47", default-features = false} -mime_guess = "2.0.4" -names = {version = "0.14.0", default-features = false} -opentelemetry = {version = "0.17.0", features = ["rt-tokio"]} prometheus-client = "0.17.0" -rand = "0.8.5" serde = {version = "1.0", features = ["derive"]} -serde_json = "1.0.78" -serde_qs = "0.10.1" -time = "0.3.9" tokio = {version = "1", features = ["macros", "rt-multi-thread", "process"]} -tokio-util = {version = "0.7", features = ["io"]} -toml = "0.5.9" -tower = {version = "0.4", features = ["util", "timeout", "load-shed", "limit"]} -tower-http = {version = "0.3", features = ["trace"]} -tower-layer = {version = "0.3"} tracing = "0.1.33" -tracing-opentelemetry = "0.17.2" -tracing-subscriber = {version = "0.3.11", features = ["env-filter"]} -url = "2.2.2" -urlencoding = "2.1.0" [dev-dependencies] axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers +http = "0.2" [features] default = ["rpc-mem", "rpc-grpc"] diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index f804164ea5..073fed5903 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -15,8 +15,8 @@ use serde::{Deserialize, Serialize}; pub const CONFIG_FILE_NAME: &str = "gateway.config.toml"; /// ENV_PREFIX should be used along side the config field name to set a config field using /// environment variables -/// For example, `IROH_GATEWAY_PORT=1000` would set the value of the `Config.port` field -pub const ENV_PREFIX: &str = "IROH_GATEWAY"; +/// For example, `IROH_ONE_PORT=1000` would set the value of the `Config.port` field +pub const ENV_PREFIX: &str = "IROH_ONE"; pub const DEFAULT_PORT: u16 = 9050; #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] diff --git a/iroh-one/src/uds.rs b/iroh-one/src/uds.rs index 1469a32cfe..713af0c643 100644 --- a/iroh-one/src/uds.rs +++ b/iroh-one/src/uds.rs @@ -16,14 +16,14 @@ use tokio::{ io::{AsyncRead, AsyncWrite}, net::{unix::UCred, UnixListener, UnixStream}, }; -use tower::BoxError; + pub struct ServerAccept { pub uds: UnixListener, } impl Accept for ServerAccept { type Conn = UnixStream; - type Error = BoxError; + type Error = Box; fn poll_accept( self: Pin<&mut Self>, From f6818247041e72706c032de6ca6b4fa0ee9bdb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 12 Aug 2022 17:47:33 -0700 Subject: [PATCH 12/29] iroh-one: Implement configuration structure --- iroh-gateway/src/handlers.rs | 4 +- iroh-one/.gitignore | 3 +- iroh-one/src/config.rs | 181 +++++++++++++++++++++++++++-------- iroh-one/src/core.rs | 19 +--- iroh-one/src/main.rs | 9 +- iroh-one/src/mem_p2p.rs | 23 +---- iroh-one/src/mem_store.rs | 21 +--- 7 files changed, 161 insertions(+), 99 deletions(-) diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index e38c933368..15b8fe7e94 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -57,11 +57,11 @@ pub fn get_app_routes(state: &Arc) -> Router { .route("/:scheme/:cid", get(get_handler)) .route("/:scheme/:cid/*cpath", get(get_handler)) .route("/health", get(health_check)) - .layer(Extension(Arc::clone(&state))) + .layer(Extension(Arc::clone(state))) .layer( ServiceBuilder::new() // Handle errors from middleware - .layer(Extension(Arc::clone(&state))) + .layer(Extension(Arc::clone(state))) .layer(HandleErrorLayer::new(middleware_error_handler)) .load_shed() .concurrency_limit(2048) diff --git a/iroh-one/.gitignore b/iroh-one/.gitignore index 1c3f71732b..48180fec31 100644 --- a/iroh-one/.gitignore +++ b/iroh-one/.gitignore @@ -1 +1,2 @@ -iroh-store/ +# Default path for the store. +iroh-store-db/ diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 073fed5903..a2331bc54a 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -1,26 +1,77 @@ -use iroh_gateway::constants::*; use anyhow::{bail, Result}; use axum::http::{header::*, Method}; use config::{ConfigError, Map, Source, Value}; use headers::{ AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlAllowOrigin, HeaderMapExt, }; +use iroh_gateway::constants::*; use iroh_metrics::config::Config as MetricsConfig; +use iroh_p2p::{Config as FullP2pConfig, Libp2pConfig}; use iroh_rpc_client::Config as RpcClientConfig; use iroh_rpc_types::{gateway::GatewayServerAddr, Addr}; +use iroh_store::Config as FullStoreConfig; use iroh_util::insert_into_config_map; use serde::{Deserialize, Serialize}; +use std::path::PathBuf; /// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory -pub const CONFIG_FILE_NAME: &str = "gateway.config.toml"; +pub const CONFIG_FILE_NAME: &str = "one.config.toml"; /// ENV_PREFIX should be used along side the config field name to set a config field using /// environment variables /// For example, `IROH_ONE_PORT=1000` would set the value of the `Config.port` field pub const ENV_PREFIX: &str = "IROH_ONE"; pub const DEFAULT_PORT: u16 = 9050; +/// The configuration includes gateway, store and p2p specific items +/// as well as the common rpc & metrics ones. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct Config { + // Gateway specific configuration. + pub gateway: GatewayConfig, + // Store specific configuration. + pub store: StoreConfig, + // P2P specific configuration. + pub p2p: iroh_p2p::Libp2pConfig, + + /// rpc addresses for the gateway & addresses for the rpc client to dial + pub rpc_client: RpcClientConfig, + /// metrics configuration + pub metrics: MetricsConfig, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct StoreConfig { + /// The location of the content database. + pub path: PathBuf, +} + +impl Source for StoreConfig { + fn clone_into_box(&self) -> Box { + Box::new(self.clone()) + } + + fn collect(&self) -> Result, ConfigError> { + let mut map: Map = Map::new(); + + let path = self + .path + .to_str() + .ok_or_else(|| ConfigError::Foreign("No `path` set. Path is required.".into()))?; + insert_into_config_map(&mut map, "path", path); + Ok(map) + } +} + +impl Default for StoreConfig { + fn default() -> Self { + Self { + path: PathBuf::from("./iroh-store-db"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct GatewayConfig { /// flag to toggle whether the gateway allows writing/pushing data pub writeable: bool, /// flag to toggle whether the gateway allows fetching data from other nodes or is local only @@ -34,35 +85,24 @@ pub struct Config { /// set of user provided headers to attach to all responses #[serde(with = "http_serde::header_map")] pub headers: HeaderMap, - /// rpc addresses for the gateway & addresses for the rpc client to dial - pub rpc_client: RpcClientConfig, - /// metrics configuration - pub metrics: MetricsConfig, } impl Config { pub fn new( - writeable: bool, - fetch: bool, - cache: bool, - port: u16, + gateway: GatewayConfig, + store: StoreConfig, + p2p: Libp2pConfig, rpc_client: RpcClientConfig, ) -> Self { Self { - writeable, - fetch, - cache, - headers: HeaderMap::new(), - port, + gateway, + store, + p2p, rpc_client, metrics: MetricsConfig::default(), } } - pub fn set_default_headers(&mut self) { - self.headers = default_headers(); - } - /// Derive server addr for non memory addrs. pub fn server_rpc_addr(&self) -> Result> { self.rpc_client @@ -107,6 +147,44 @@ impl Config { } } +#[allow(clippy::from_over_into)] +impl Into for Config { + fn into(self) -> FullStoreConfig { + FullStoreConfig { + path: self.store.path.clone(), + rpc_client: self.rpc_client.clone(), + metrics: self.metrics, + } + } +} + +#[allow(clippy::from_over_into)] +impl Into for Config { + fn into(self) -> FullP2pConfig { + FullP2pConfig { + libp2p: self.p2p.clone(), + rpc_client: self.rpc_client.clone(), + metrics: self.metrics, + } + } +} + +impl GatewayConfig { + pub fn new(writeable: bool, fetch: bool, cache: bool, port: u16) -> Self { + Self { + writeable, + fetch, + cache, + headers: HeaderMap::new(), + port, + } + } + + pub fn set_default_headers(&mut self) { + self.headers = default_headers(); + } +} + fn default_headers() -> HeaderMap { let mut headers = HeaderMap::new(); headers.typed_insert(AccessControlAllowOrigin::ANY); @@ -148,16 +226,24 @@ fn default_headers() -> HeaderMap { impl Default for Config { fn default() -> Self { - let rpc_client = Self::default_ipfsd(); + Self { + rpc_client: Self::default_ipfsd(), + metrics: MetricsConfig::default(), + gateway: GatewayConfig::default(), + store: StoreConfig::default(), + p2p: Libp2pConfig::default(), + } + } +} +impl Default for GatewayConfig { + fn default() -> Self { let mut t = Self { writeable: false, fetch: false, cache: false, headers: HeaderMap::new(), port: DEFAULT_PORT, - rpc_client, - metrics: MetricsConfig::default(), }; t.set_default_headers(); t @@ -170,7 +256,23 @@ impl Source for Config { } fn collect(&self) -> Result, ConfigError> { - let rpc_client = self.rpc_client.collect()?; + let mut map: Map = Map::new(); + + insert_into_config_map(&mut map, "gateway", self.gateway.collect()?); + insert_into_config_map(&mut map, "store", self.store.collect()?); + insert_into_config_map(&mut map, "p2p", self.p2p.collect()?); + insert_into_config_map(&mut map, "rpc_client", self.rpc_client.collect()?); + insert_into_config_map(&mut map, "metrics", self.metrics.collect()?); + Ok(map) + } +} + +impl Source for GatewayConfig { + fn clone_into_box(&self) -> Box { + Box::new(self.clone()) + } + + fn collect(&self) -> Result, ConfigError> { let mut map: Map = Map::new(); insert_into_config_map(&mut map, "writeable", self.writeable); insert_into_config_map(&mut map, "fetch", self.fetch); @@ -179,9 +281,6 @@ impl Source for Config { // an signed int fixes the issue insert_into_config_map(&mut map, "port", self.port as i32); insert_into_config_map(&mut map, "headers", collect_headers(&self.headers)?); - insert_into_config_map(&mut map, "rpc_client", rpc_client); - let metrics = self.metrics.collect()?; - insert_into_config_map(&mut map, "metrics", metrics); Ok(map) } } @@ -192,11 +291,11 @@ impl iroh_gateway::handlers::StateConfig for Config { } fn port(&self) -> u16 { - self.port + self.gateway.port } fn user_headers(&self) -> HeaderMap { - self.headers.clone() + self.gateway.headers.clone() } } @@ -228,23 +327,29 @@ mod tests { #[test] fn default_config() { let config = Config::default(); - assert!(!config.writeable); - assert!(!config.fetch); - assert!(!config.cache); - assert_eq!(config.port, DEFAULT_PORT); + assert!(!config.gateway.writeable); + assert!(!config.gateway.fetch); + assert!(!config.gateway.cache); + assert_eq!(config.gateway.port, DEFAULT_PORT); } #[test] fn test_collect() { let default = Config::default(); let mut expect: Map = Map::new(); - expect.insert("writeable".to_string(), Value::new(None, default.writeable)); - expect.insert("fetch".to_string(), Value::new(None, default.fetch)); - expect.insert("cache".to_string(), Value::new(None, default.cache)); - expect.insert("port".to_string(), Value::new(None, default.port as i64)); + expect.insert( + "writeable".to_string(), + Value::new(None, default.gateway.writeable), + ); + expect.insert("fetch".to_string(), Value::new(None, default.gateway.fetch)); + expect.insert("cache".to_string(), Value::new(None, default.gateway.cache)); + expect.insert( + "port".to_string(), + Value::new(None, default.gateway.port as i64), + ); expect.insert( "headers".to_string(), - Value::new(None, collect_headers(&default.headers).unwrap()), + Value::new(None, collect_headers(&default.gateway.headers).unwrap()), ); expect.insert( "rpc_client".to_string(), @@ -287,7 +392,7 @@ mod tests { #[test] fn test_build_config_from_struct() { let mut expect = Config::default(); - expect.set_default_headers(); + expect.gateway.set_default_headers(); let source = expect.clone(); let got: Config = ConfigBuilder::builder() .add_source(source) diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 987f906646..3c0f293db2 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -84,24 +84,15 @@ impl Core { #[cfg(test)] mod tests { use super::*; - use crate::config::Config; - use iroh_rpc_client::Config as RpcClientConfig; + use crate::config::{Config, GatewayConfig}; use prometheus_client::registry::Registry; #[tokio::test] async fn gateway_health() { - let mut config = Config::new( - false, - false, - false, - 0, - RpcClientConfig { - gateway_addr: None, - p2p_addr: None, - store_addr: None, - }, - ); - config.set_default_headers(); + let mut gateway = GatewayConfig::new(false, false, false, 0); + gateway.set_default_headers(); + let mut config = Config::default(); + config.gateway = gateway; let mut prom_registry = Registry::default(); let gw_metrics = Metrics::new(&mut prom_registry); diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 91d3842490..e17c90be78 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -74,11 +74,11 @@ async fn main() -> Result<()> { let (store_rpc, p2p_rpc) = { let (store_recv, store_sender) = Addr::new_mem(); config.rpc_client.store_addr = Some(store_sender); - let store_rpc = iroh_one::mem_store::start(store_recv).await?; + let store_rpc = iroh_one::mem_store::start(store_recv, config.clone().into()).await?; let (p2p_recv, p2p_sender) = Addr::new_mem(); config.rpc_client.p2p_addr = Some(p2p_sender); - let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv).await?; + let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv, config.clone().into()).await?; (store_rpc, p2p_rpc) }; @@ -105,10 +105,9 @@ async fn main() -> Result<()> { let uds_server_task = { let uds_server = handler.uds_server(); - let task = tokio::spawn(async move { + tokio::spawn(async move { uds_server.await.unwrap(); - }); - task + }) }; iroh_util::block_until_sigint().await; diff --git a/iroh-one/src/mem_p2p.rs b/iroh-one/src/mem_p2p.rs index af6e9fdfa4..652dc6d04c 100644 --- a/iroh-one/src/mem_p2p.rs +++ b/iroh-one/src/mem_p2p.rs @@ -1,36 +1,19 @@ /// A p2p instance listening on a memory rpc channel. -use iroh_p2p::config::{Config, ENV_PREFIX}; +use iroh_p2p::config::Config; use iroh_p2p::{DiskStorage, Keychain, Node}; use iroh_rpc_types::p2p::P2pServerAddr; -use iroh_util::make_config; use prometheus_client::registry::Registry; -use std::collections::HashMap; use tokio::task; use tokio::task::JoinHandle; use tracing::error; /// Starts a new p2p node, using the given mem rpc channel. -/// TODO: refactor to share most of the setup with iroh-p2p/src/main.rs -pub async fn start(rpc_addr: P2pServerAddr) -> anyhow::Result> { - // TODO: configurable network - let overrides: HashMap = HashMap::new(); - let network_config = make_config( - // default - Config::default_grpc(), - // potential config files - vec![], - // env var prefix for this config - ENV_PREFIX, - // map of present command line arguments - overrides, - ) - .unwrap(); - +pub async fn start(rpc_addr: P2pServerAddr, config: Config) -> anyhow::Result> { let mut prom_registry = Registry::default(); let kc = Keychain::::new().await?; - let mut p2p = Node::new(network_config, rpc_addr, kc, &mut prom_registry).await?; + let mut p2p = Node::new(config, rpc_addr, kc, &mut prom_registry).await?; // Start services let p2p_task = task::spawn(async move { diff --git a/iroh-one/src/mem_store.rs b/iroh-one/src/mem_store.rs index fb32a544ff..f85e066869 100644 --- a/iroh-one/src/mem_store.rs +++ b/iroh-one/src/mem_store.rs @@ -1,30 +1,13 @@ /// A store instance listening on a memory rpc channel. use iroh_metrics::store::Metrics; use iroh_rpc_types::store::StoreServerAddr; -use iroh_store::{config::ENV_PREFIX, rpc, Config, Store}; -use iroh_util::make_config; +use iroh_store::{rpc, Config, Store}; use prometheus_client::registry::Registry; -use std::collections::HashMap; -use std::path::PathBuf; use tokio::task::JoinHandle; use tracing::info; /// Starts a new store, using the given mem rpc channel. -/// TODO: refactor to share most of the setup with iroh-store/src/main.rs -pub async fn start(rpc_addr: StoreServerAddr) -> anyhow::Result> { - let overrides: HashMap = HashMap::new(); - let config: iroh_store::Config = make_config( - // default - Config::new_grpc(PathBuf::from("./iroh-store")), - // potential config files - vec![], - // env var prefix for this config - ENV_PREFIX, - // map of present command line arguments - overrides, - ) - .unwrap(); - +pub async fn start(rpc_addr: StoreServerAddr, config: Config) -> anyhow::Result> { let mut prom_registry = Registry::default(); let store_metrics = Metrics::new(&mut prom_registry); From dccd9d4854a2348b96b02f69f4ff9f479d371da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Mon, 15 Aug 2022 09:42:09 -0700 Subject: [PATCH 13/29] resolver: race p2p with raw CID download from a http gateway --- iroh-gateway/src/core.rs | 1 + iroh-one/src/config.rs | 8 ++- iroh-one/src/core.rs | 2 +- iroh-one/src/main.rs | 2 + iroh-resolver/Cargo.toml | 4 +- iroh-resolver/src/resolver.rs | 128 +++++++++++++++++++++++++--------- iroh-rpc-client/src/client.rs | 9 +++ iroh-rpc-client/src/config.rs | 3 + iroh-share/src/p2p_node.rs | 2 + 9 files changed, 123 insertions(+), 36 deletions(-) diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index fefb979ed3..8bec49de2f 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -123,6 +123,7 @@ mod tests { gateway_addr: None, p2p_addr: None, store_addr: None, + raw_gateway: None, }, ); config.set_default_headers(); diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index a2331bc54a..1487041b37 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -80,6 +80,8 @@ pub struct GatewayConfig { pub cache: bool, /// default port to listen on pub port: u16, + /// Gateway from which to fetch raw CIDs. TODO: move to p2p config? + pub raw_gateway: String, // NOTE: for toml to serialize properly, the "table" values must be serialized at the end, and // so much come at the end of the `Config` struct /// set of user provided headers to attach to all responses @@ -143,6 +145,7 @@ impl Config { gateway_addr: Some(Addr::GrpcUds(path)), p2p_addr: None, store_addr: None, + raw_gateway: None, } } } @@ -170,13 +173,14 @@ impl Into for Config { } impl GatewayConfig { - pub fn new(writeable: bool, fetch: bool, cache: bool, port: u16) -> Self { + pub fn new(writeable: bool, fetch: bool, cache: bool, port: u16, raw_gateway: &str) -> Self { Self { writeable, fetch, cache, headers: HeaderMap::new(), port, + raw_gateway: raw_gateway.to_owned(), } } @@ -244,6 +248,7 @@ impl Default for GatewayConfig { cache: false, headers: HeaderMap::new(), port: DEFAULT_PORT, + raw_gateway: "dweb.link".to_owned(), }; t.set_default_headers(); t @@ -277,6 +282,7 @@ impl Source for GatewayConfig { insert_into_config_map(&mut map, "writeable", self.writeable); insert_into_config_map(&mut map, "fetch", self.fetch); insert_into_config_map(&mut map, "cache", self.cache); + insert_into_config_map(&mut map, "raw_gateway", self.raw_gateway.clone()); // Some issue between deserializing u64 & u16, converting this to // an signed int fixes the issue insert_into_config_map(&mut map, "port", self.port as i32); diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 3c0f293db2..010fbc77f3 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -89,7 +89,7 @@ mod tests { #[tokio::test] async fn gateway_health() { - let mut gateway = GatewayConfig::new(false, false, false, 0); + let mut gateway = GatewayConfig::default(); gateway.set_default_headers(); let mut config = Config::default(); config.gateway = gateway; diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index e17c90be78..79a461dd4a 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -82,6 +82,8 @@ async fn main() -> Result<()> { (store_rpc, p2p_rpc) }; + config.rpc_client.raw_gateway = Some(config.gateway.raw_gateway.clone()); + config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); println!("{:#?}", config); diff --git a/iroh-resolver/Cargo.toml b/iroh-resolver/Cargo.toml index 2b7dd1e1b5..8982a89516 100644 --- a/iroh-resolver/Cargo.toml +++ b/iroh-resolver/Cargo.toml @@ -15,6 +15,7 @@ num_enum = "0.5.7" prost = "0.11" bytes = "1.1.0" iroh-rpc-client = { path = "../iroh-rpc-client", default-features = false } +iroh-util = { path = "../iroh-util", default-features = false } tokio = { version = "1" } futures = "0.3.21" tracing = "0.1.34" @@ -27,6 +28,8 @@ async-stream = "0.3.3" fastmurmur3 = "0.1.2" once_cell = "1.13.0" tokio-util = { version = "0.7", features = ["io"] } +reqwest = "0.11" + [dev-dependencies] criterion = { version = "0.3.5", features = ["async_tokio"] } @@ -35,7 +38,6 @@ tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } iroh-store = { path = "../iroh-store", default-features = false } iroh-rpc-types = { path = "../iroh-rpc-types", default-features = false } iroh-car = { path = "../iroh-car" } -iroh-util = { path = "../iroh-util", default-features = false } [build-dependencies] prost-build = "0.11.1" diff --git a/iroh-resolver/src/resolver.rs b/iroh-resolver/src/resolver.rs index fd94e143a7..f20417bb6b 100644 --- a/iroh-resolver/src/resolver.rs +++ b/iroh-resolver/src/resolver.rs @@ -9,8 +9,9 @@ use std::time::Instant; use anyhow::{anyhow, bail, ensure, Context as _, Result}; use async_trait::async_trait; use bytes::Bytes; -use cid::Cid; +use cid::{multibase, Cid}; use futures::Stream; +use futures::{future::FutureExt, pin_mut, select}; use iroh_metrics::resolver::Metrics; use iroh_rpc_client::Client; use libipld::codec::{Decode, Encode}; @@ -18,7 +19,7 @@ use libipld::prelude::Codec as _; use libipld::{Ipld, IpldCodec}; use prometheus_client::registry::Registry; use tokio::io::AsyncRead; -use tracing::{debug, trace, warn}; +use tracing::{debug, error, trace, warn}; use crate::codecs::Codec; use crate::unixfs::{ @@ -381,6 +382,7 @@ pub struct LoadedCid { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Source { Bitswap, + Http, Store(&'static str), } @@ -403,6 +405,31 @@ impl ContentLoader for Arc { } } +#[async_trait] +trait CidFetcher { + async fn fetch_p2p(&self, cid: &Cid) -> Result; + + async fn fetch_http(&self, cid: &Cid) -> Result; +} + +#[async_trait] +impl CidFetcher for Client { + async fn fetch_p2p(&self, cid: &Cid) -> Result { + let p2p = self.try_p2p()?; + let providers = p2p.fetch_providers(cid).await?; + p2p.fetch_bitswap(*cid, providers).await + } + + async fn fetch_http(&self, cid: &Cid) -> Result { + let gateway = self.try_raw_gateway()?; + let cid_str = multibase::encode(multibase::Base::Base32Lower, cid.to_bytes().as_slice()); + let gateway_url = format!("https://{}.ipfs.{}?format=raw", cid_str, gateway); + debug!("Will fetch {}", gateway_url); + let response = reqwest::get(gateway_url).await?; + response.bytes().await.map_err(|e| e.into()) + } +} + #[async_trait] impl ContentLoader for Client { async fn load_cid(&self, cid: &Cid) -> Result { @@ -422,40 +449,75 @@ impl ContentLoader for Client { warn!("failed to fetch data from store {}: {:?}", cid, err); } } - let p2p = self.try_p2p()?; - let providers = p2p.fetch_providers(&cid).await?; - let bytes = p2p.fetch_bitswap(cid, providers).await?; - - // trigger storage in the background - let cloned = bytes.clone(); - let rpc = self.clone(); - tokio::spawn(async move { - let clone2 = cloned.clone(); - let links = - tokio::task::spawn_blocking(move || parse_links(&cid, &clone2).unwrap_or_default()) - .await - .unwrap_or_default(); - - let len = cloned.len(); - let links_len = links.len(); - if let Some(store_rpc) = rpc.store.as_ref() { - match store_rpc.put(cid, cloned, links).await { - Ok(_) => debug!("stored {} ({}bytes, {}links)", cid, len, links_len), - Err(err) => { - warn!("failed to store {}: {:?}", cid, err); + + let p2p_fut = self.fetch_p2p(&cid).fuse(); + let http_fut = self.fetch_http(&cid).fuse(); + pin_mut!(p2p_fut, http_fut); + + let mut bytes: Option = None; + let mut source = Source::Bitswap; + + // Race the p2p and http fetches. + loop { + select! { + res = http_fut => { + if let Ok(data) = res { + debug!("retrieved from http"); + if let Some(true) = iroh_util::verify_hash(&cid, &data) { + source = Source::Http; + bytes = Some(data); + break; + } else { + error!("Got http data, but CID verification failed!"); + } } } - } else { - warn!("failed to store: missing store rpc conn"); + res = p2p_fut => { + if let Ok(data) = res { + debug!("retrieved from p2p"); + bytes = Some(data); + break; + } + } + complete => { break; } } - }); + } - trace!("retrieved from p2p"); + if let Some(bytes) = bytes { + // trigger storage in the background + let cloned = bytes.clone(); + let rpc = self.clone(); + tokio::spawn(async move { + let clone2 = cloned.clone(); + let links = tokio::task::spawn_blocking(move || { + parse_links(&cid, &clone2).unwrap_or_default() + }) + .await + .unwrap_or_default(); + + if let Some(store_rpc) = rpc.store.as_ref() { + let len = cloned.len(); + let links_len = links.len(); + match store_rpc.put(cid, cloned, links).await { + Ok(_) => { + debug!("stored {} ({} bytes, {} links)", cid, len, links_len) + } + Err(err) => { + error!("failed to store {}: {:?}", cid, err); + } + } + } else { + error!("failed to store: missing store rpc conn"); + } + }); - Ok(LoadedCid { - data: bytes, - source: Source::Bitswap, - }) + Ok(LoadedCid { + data: bytes, + source, + }) + } else { + Err(anyhow::anyhow!("Failed to load from p2p & http")) + } } } @@ -538,7 +600,7 @@ impl Resolver { let next_link = current .get_link_by_name(&part) .await? - .ok_or_else(|| anyhow!("link {} not found", part))?; + .ok_or_else(|| anyhow!("UnixfsNode::Directory link '{}' not found", part))?; let loaded_cid = self.load_cid(&next_link.cid).await?; let next_node = UnixfsNode::decode(&next_link.cid, loaded_cid.data)?; resolved_path.push(next_link.cid); @@ -549,7 +611,7 @@ impl Resolver { let (next_link, next_node) = hamt .get(self, part.as_bytes()) .await? - .ok_or_else(|| anyhow!("link {} not found", part))?; + .ok_or_else(|| anyhow!("UnixfsNode::HamtShard link '{}' not found", part))?; // TODO: is this the right way to to resolved path here? resolved_path.push(next_link.cid); diff --git a/iroh-rpc-client/src/client.rs b/iroh-rpc-client/src/client.rs index 16e6f530c6..befc7aa3d3 100644 --- a/iroh-rpc-client/src/client.rs +++ b/iroh-rpc-client/src/client.rs @@ -12,6 +12,7 @@ pub struct Client { pub gateway: Option, pub p2p: Option, pub store: Option, + pub raw_gateway: Option, } impl Client { @@ -20,6 +21,7 @@ impl Client { gateway_addr, p2p_addr, store_addr, + raw_gateway, } = cfg; let gateway = if let Some(addr) = gateway_addr { @@ -55,9 +57,16 @@ impl Client { gateway, p2p, store, + raw_gateway, }) } + pub fn try_raw_gateway(&self) -> Result<&String> { + self.raw_gateway + .as_ref() + .ok_or_else(|| anyhow!("no gateway configured to fetch raw CIDs")) + } + pub fn try_p2p(&self) -> Result<&P2pClient> { self.p2p .as_ref() diff --git a/iroh-rpc-client/src/config.rs b/iroh-rpc-client/src/config.rs index 352acce228..efd92681bd 100644 --- a/iroh-rpc-client/src/config.rs +++ b/iroh-rpc-client/src/config.rs @@ -12,6 +12,8 @@ pub struct Config { pub p2p_addr: Option, // store rpc address pub store_addr: Option, + // Domain name of a gateway to fetch raw CIDs. + pub raw_gateway: Option, } impl Source for Config { @@ -40,6 +42,7 @@ impl Config { gateway_addr: Some("grpc://0.0.0.0:4400".parse().unwrap()), p2p_addr: Some("grpc://0.0.0.0:4401".parse().unwrap()), store_addr: Some("grpc://0.0.0.0:4402".parse().unwrap()), + raw_gateway: None, } } } diff --git a/iroh-share/src/p2p_node.rs b/iroh-share/src/p2p_node.rs index 1a6581ced6..60be79b5bd 100644 --- a/iroh-share/src/p2p_node.rs +++ b/iroh-share/src/p2p_node.rs @@ -120,11 +120,13 @@ impl P2pNode { p2p_addr: Some(rpc_p2p_addr_client.clone()), store_addr: Some(rpc_store_addr_client.clone()), gateway_addr: None, + raw_gateway: None, }; let rpc_p2p_client_config = iroh_rpc_client::Config { p2p_addr: Some(rpc_p2p_addr_client.clone()), store_addr: Some(rpc_store_addr_client.clone()), gateway_addr: None, + raw_gateway: None, }; let config = config::Config { libp2p: config::Libp2pConfig { From 25ce66b35ed651a95a80723661d38d2ac9f82bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Mon, 15 Aug 2022 09:54:07 -0700 Subject: [PATCH 14/29] Update description and README --- iroh-one/Cargo.toml | 2 +- iroh-one/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index ac5439360b..0049b0cbb1 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Single binary IPFS gateway" +description = "all of iroh in a single binary" edition = "2021" license = "Apache-2.0/MIT" name = "iroh-one" diff --git a/iroh-one/README.md b/iroh-one/README.md index 4daf84c79d..a74c03aebc 100644 --- a/iroh-one/README.md +++ b/iroh-one/README.md @@ -1,14 +1,14 @@ -# Iroh Gateway +# Iroh One Gateway A rust implementation of an IPFS gateway. ## Running / Building -`cargo run -- -p 10000` +`cargo run --release -- -p 10000` ### Options -- Run with `cargo run -- -h` for details +- Run with `cargo run --release -- -h` for details - `-wcf` Writeable, Cache, Fetch (options to toggle write enable, caching mechanics and fetching from the network); currently exists but is not implemented - `-p` Port the gateway should listen on From 59a27a14c0b26057017c1cc7d361b1a7b4ba5408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Mon, 15 Aug 2022 10:26:29 -0700 Subject: [PATCH 15/29] chore: sync up with BadBits support --- iroh-gateway/src/bad_bits.rs | 1 + iroh-gateway/src/handlers.rs | 29 +++++++++++++++++++++++++++++ iroh-one/src/config.rs | 5 +++++ iroh-one/src/core.rs | 15 +++++++++++---- iroh-one/src/main.rs | 22 ++++++++++++++++++---- 5 files changed, 64 insertions(+), 8 deletions(-) diff --git a/iroh-gateway/src/bad_bits.rs b/iroh-gateway/src/bad_bits.rs index c945b2b45f..f67d1ea570 100644 --- a/iroh-gateway/src/bad_bits.rs +++ b/iroh-gateway/src/bad_bits.rs @@ -186,6 +186,7 @@ mod tests { gateway_addr: None, p2p_addr: None, store_addr: None, + raw_gateway: None, }, ); config.set_default_headers(); diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index 15b8fe7e94..7badceb5a6 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -137,6 +137,14 @@ pub async fn get_handler( service_worker_check(&request_headers, cpath.to_string(), &state)?; unsuported_header_check(&request_headers, &state)?; + if check_bad_bits(&state, cid, cpath).await { + return Err(error( + StatusCode::FORBIDDEN, + "CID is in the denylist", + &state, + )); + } + let full_content_path = format!("/{}/{}{}", scheme, cid, cpath); let resolved_path: iroh_resolver::resolver::Path = full_content_path .parse() @@ -144,6 +152,14 @@ pub async fn get_handler( .map_err(|e| error(StatusCode::BAD_REQUEST, &e, &state))?; let resolved_cid = resolved_path.root(); + if check_bad_bits(&state, resolved_cid.to_string().as_str(), cpath).await { + return Err(error( + StatusCode::FORBIDDEN, + "CID is in the denylist", + &state, + )); + } + // parse query params let format = match get_response_format(&request_headers, query_params.format) { Ok(format) => format, @@ -263,6 +279,19 @@ fn unsuported_header_check(request_headers: &HeaderMap, state: &State) -> Result Ok(()) } +pub async fn check_bad_bits(state: &State, cid: &str, path: &str) -> bool { + // check if cid is in the denylist + if state.bad_bits.is_some() { + let bad_bits = state.bad_bits.as_ref(); + if let Some(bbits) = bad_bits { + if bbits.read().await.is_bad(cid, path) { + return true; + } + } + } + false +} + #[tracing::instrument()] fn etag_check( request_headers: &HeaderMap, diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 1487041b37..2a2cdc8e99 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -87,6 +87,8 @@ pub struct GatewayConfig { /// set of user provided headers to attach to all responses #[serde(with = "http_serde::header_map")] pub headers: HeaderMap, + /// flag to toggle whether the gateway should use denylist on requests + pub denylist: bool, } impl Config { @@ -181,6 +183,7 @@ impl GatewayConfig { headers: HeaderMap::new(), port, raw_gateway: raw_gateway.to_owned(), + denylist: false, } } @@ -249,6 +252,7 @@ impl Default for GatewayConfig { headers: HeaderMap::new(), port: DEFAULT_PORT, raw_gateway: "dweb.link".to_owned(), + denylist: false, }; t.set_default_headers(); t @@ -282,6 +286,7 @@ impl Source for GatewayConfig { insert_into_config_map(&mut map, "writeable", self.writeable); insert_into_config_map(&mut map, "fetch", self.fetch); insert_into_config_map(&mut map, "cache", self.cache); + insert_into_config_map(&mut map, "denylist", self.denylist); insert_into_config_map(&mut map, "raw_gateway", self.raw_gateway.clone()); // Some issue between deserializing u64 & u16, converting this to // an signed int fixes the issue diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 010fbc77f3..17e14de843 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -1,9 +1,10 @@ use crate::{rpc, rpc::Gateway, uds}; use axum::{Router, Server}; use iroh_gateway::{ + bad_bits::BadBits, client::Client, core::State, - handlers::{get_app_routes, StateConfig}, bad_bits::BadBits, + handlers::{get_app_routes, StateConfig}, templates, }; use iroh_metrics::gateway::Metrics; @@ -97,9 +98,15 @@ mod tests { let mut prom_registry = Registry::default(); let gw_metrics = Metrics::new(&mut prom_registry); let rpc_addr = "grpc://0.0.0.0:0".parse().unwrap(); - let handler = Core::new(Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry) - .await - .unwrap(); + let handler = Core::new( + Arc::new(config), + rpc_addr, + gw_metrics, + &mut prom_registry, + Arc::new(None), + ) + .await + .unwrap(); let server = handler.http_server(); let addr = server.local_addr(); let core_task = tokio::spawn(async move { diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 79a461dd4a..cc64a95135 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; -use iroh_gateway::metrics; +use iroh_gateway::{metrics, bad_bits::BadBits}; use iroh_metrics::gateway::Metrics; use iroh_one::{ config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, @@ -13,6 +13,7 @@ use iroh_one::{ use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; +use tokio::sync::RwLock; #[derive(Parser, Debug, Clone)] #[clap(author, version, about, long_about = None)] @@ -25,9 +26,9 @@ struct Args { fetch: Option, #[clap(short, long)] cache: Option, - #[clap(long = "metrics")] + #[clap(long)] metrics: bool, - #[clap(long = "tracing")] + #[clap(long)] tracing: bool, #[clap(long)] cfg: Option, @@ -93,7 +94,20 @@ async fn main() -> Result<()> { let rpc_addr = config .server_rpc_addr()? .ok_or_else(|| anyhow!("missing gateway rpc addr"))?; - let handler = Core::new(Arc::new(config), rpc_addr, gw_metrics, &mut prom_registry).await?; + + let bad_bits = match config.gateway.denylist { + true => Arc::new(Some(RwLock::new(BadBits::new()))), + false => Arc::new(None), + }; + + let handler = Core::new( + Arc::new(config), + rpc_addr, + gw_metrics, + &mut prom_registry, + Arc::clone(&bad_bits), + ) + .await?; let metrics_handle = iroh_metrics::MetricsHandle::from_registry_with_tracer(metrics_config, prom_registry) From 659cc1531b7578acd045001d8913ed439cfa89e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Wed, 17 Aug 2022 10:35:46 -0700 Subject: [PATCH 16/29] fix: use rustls with reqwest --- iroh-resolver/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iroh-resolver/Cargo.toml b/iroh-resolver/Cargo.toml index 8982a89516..289b736512 100644 --- a/iroh-resolver/Cargo.toml +++ b/iroh-resolver/Cargo.toml @@ -28,8 +28,7 @@ async-stream = "0.3.3" fastmurmur3 = "0.1.2" once_cell = "1.13.0" tokio-util = { version = "0.7", features = ["io"] } -reqwest = "0.11" - +reqwest = { version = "0.11.10", features = ["rustls-tls"], default-features = false} [dev-dependencies] criterion = { version = "0.3.5", features = ["async_tokio"] } From 8049bce80fbe0188be35a4bd3336adcb02b1a546 Mon Sep 17 00:00:00 2001 From: The Capyloon Team Date: Wed, 17 Aug 2022 19:42:11 +0200 Subject: [PATCH 17/29] fix: android specific oopsy --- iroh-one/src/config.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 2a2cdc8e99..98c274bed9 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -134,7 +134,9 @@ impl Config { pub fn default_ipfsd() -> RpcClientConfig { let path = { #[cfg(target_os = "android")] - "/dev/socket/ipfsd".into(); + { + "/dev/socket/ipfsd".into() + } #[cfg(not(target_os = "android"))] { From 4867a014dcf6fa89e44b193d921792defe8c9c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Thu, 18 Aug 2022 13:05:54 -0700 Subject: [PATCH 18/29] fix: make clippy happy --- iroh-one/src/core.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 17e14de843..443b98428c 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -92,8 +92,10 @@ mod tests { async fn gateway_health() { let mut gateway = GatewayConfig::default(); gateway.set_default_headers(); - let mut config = Config::default(); - config.gateway = gateway; + let config = Config { + gateway, + ..Default::default() + }; let mut prom_registry = Registry::default(); let gw_metrics = Metrics::new(&mut prom_registry); From 64f7aee6aed28619960e302c5ecd5b7baf7c90da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Thu, 18 Aug 2022 13:15:29 -0700 Subject: [PATCH 19/29] fix: make cargo-fmt happy --- iroh-one/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index cc64a95135..7e28390ca5 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; -use iroh_gateway::{metrics, bad_bits::BadBits}; +use iroh_gateway::{bad_bits::BadBits, metrics}; use iroh_metrics::gateway::Metrics; use iroh_one::{ config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, From 07c2b9a91f4417816fd38da882337577ca90dc0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Thu, 18 Aug 2022 13:22:47 -0700 Subject: [PATCH 20/29] iroh-one: fix tests --- iroh-one/src/config.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 98c274bed9..63dc1932ea 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -351,27 +351,25 @@ mod tests { let default = Config::default(); let mut expect: Map = Map::new(); expect.insert( - "writeable".to_string(), - Value::new(None, default.gateway.writeable), + "gateway".to_string(), + Value::new(None, default.gateway.collect().unwrap()), ); - expect.insert("fetch".to_string(), Value::new(None, default.gateway.fetch)); - expect.insert("cache".to_string(), Value::new(None, default.gateway.cache)); expect.insert( - "port".to_string(), - Value::new(None, default.gateway.port as i64), + "store".to_string(), + Value::new(None, default.store.collect().unwrap()), ); expect.insert( - "headers".to_string(), - Value::new(None, collect_headers(&default.gateway.headers).unwrap()), - ); - expect.insert( - "rpc_client".to_string(), - Value::new(None, default.rpc_client.collect().unwrap()), + "p2p".to_string(), + Value::new(None, default.p2p.collect().unwrap()), ); expect.insert( "metrics".to_string(), Value::new(None, default.metrics.collect().unwrap()), ); + expect.insert( + "rpc_client".to_string(), + Value::new(None, default.rpc_client.collect().unwrap()), + ); let got = default.collect().unwrap(); for key in got.keys() { From e4a3258dffa287de25ddf16b4a4ed9c92a33e344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 19 Aug 2022 13:23:32 -0700 Subject: [PATCH 21/29] fix: Add missing raw_gateway in Config::collect --- iroh-rpc-client/src/config.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iroh-rpc-client/src/config.rs b/iroh-rpc-client/src/config.rs index efd92681bd..072d2ffa52 100644 --- a/iroh-rpc-client/src/config.rs +++ b/iroh-rpc-client/src/config.rs @@ -32,6 +32,9 @@ impl Source for Config { if let Some(addr) = &self.store_addr { insert_into_config_map(&mut map, "store_addr", addr.to_string()); } + if let Some(path) = &self.raw_gateway { + insert_into_config_map(&mut map, "raw_gateway", path.clone()); + } Ok(map) } } From f9176e6f6e2102ffb0614492685b81fb57f00f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabrice=20Desr=C3=A9?= Date: Fri, 19 Aug 2022 13:24:11 -0700 Subject: [PATCH 22/29] refactor: share CLI args among iroh-gateway and iroh-one --- iroh-gateway/src/cli.rs | 47 ++++++++++++++++++++++++++++++++++++++++ iroh-gateway/src/lib.rs | 1 + iroh-gateway/src/main.rs | 46 +-------------------------------------- iroh-one/src/main.rs | 44 +------------------------------------ 4 files changed, 50 insertions(+), 88 deletions(-) create mode 100644 iroh-gateway/src/cli.rs diff --git a/iroh-gateway/src/cli.rs b/iroh-gateway/src/cli.rs new file mode 100644 index 0000000000..e0b6c0bfb7 --- /dev/null +++ b/iroh-gateway/src/cli.rs @@ -0,0 +1,47 @@ +/// CLI arguments support. +use clap::Parser; +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Parser, Debug, Clone)] +#[clap(author, version, about, long_about = None)] +pub struct Args { + #[clap(short, long)] + port: Option, + #[clap(short, long)] + writeable: Option, + #[clap(short, long)] + fetch: Option, + #[clap(short, long)] + cache: Option, + #[clap(long)] + metrics: bool, + #[clap(long)] + tracing: bool, + #[clap(long)] + pub cfg: Option, + #[clap(long)] + denylist: bool, +} + +impl Args { + pub fn make_overrides_map(&self) -> HashMap<&str, String> { + let mut map: HashMap<&str, String> = HashMap::new(); + if let Some(port) = self.port { + map.insert("port", port.to_string()); + } + if let Some(writable) = self.writeable { + map.insert("writable", writable.to_string()); + } + if let Some(fetch) = self.fetch { + map.insert("fetch", fetch.to_string()); + } + if let Some(cache) = self.cache { + map.insert("cache", cache.to_string()); + } + map.insert("denylist", self.denylist.to_string()); + map.insert("metrics.collect", self.metrics.to_string()); + map.insert("metrics.tracing", self.tracing.to_string()); + map + } +} diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index 166ba3486d..c5e610e3b9 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -1,4 +1,5 @@ pub mod bad_bits; +pub mod cli; pub mod client; pub mod config; pub mod constants; diff --git a/iroh-gateway/src/main.rs b/iroh-gateway/src/main.rs index f10a1f2932..8a00c999cf 100644 --- a/iroh-gateway/src/main.rs +++ b/iroh-gateway/src/main.rs @@ -1,11 +1,10 @@ -use std::collections::HashMap; -use std::path::PathBuf; use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; use iroh_gateway::{ bad_bits::{self, BadBits}, + cli::Args, config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, core::Core, metrics, @@ -15,49 +14,6 @@ use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; use tokio::sync::RwLock; -#[derive(Parser, Debug, Clone)] -#[clap(author, version, about, long_about = None)] -struct Args { - #[clap(short, long)] - port: Option, - #[clap(short, long)] - writeable: Option, - #[clap(short, long)] - fetch: Option, - #[clap(short, long)] - cache: Option, - #[clap(long)] - metrics: bool, - #[clap(long)] - tracing: bool, - #[clap(long)] - cfg: Option, - #[clap(long)] - denylist: bool, -} - -impl Args { - fn make_overrides_map(&self) -> HashMap<&str, String> { - let mut map: HashMap<&str, String> = HashMap::new(); - if let Some(port) = self.port { - map.insert("port", port.to_string()); - } - if let Some(writable) = self.writeable { - map.insert("writable", writable.to_string()); - } - if let Some(fetch) = self.fetch { - map.insert("fetch", fetch.to_string()); - } - if let Some(cache) = self.cache { - map.insert("cache", cache.to_string()); - } - map.insert("denylist", self.denylist.to_string()); - map.insert("metrics.collect", self.metrics.to_string()); - map.insert("metrics.tracing", self.tracing.to_string()); - map - } -} - #[tokio::main(flavor = "multi_thread")] async fn main() -> Result<()> { let args = Args::parse(); diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 7e28390ca5..767fb58758 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -1,10 +1,8 @@ -use std::collections::HashMap; -use std::path::PathBuf; use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; -use iroh_gateway::{bad_bits::BadBits, metrics}; +use iroh_gateway::{bad_bits::BadBits, cli::Args, metrics}; use iroh_metrics::gateway::Metrics; use iroh_one::{ config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, @@ -15,46 +13,6 @@ use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; use tokio::sync::RwLock; -#[derive(Parser, Debug, Clone)] -#[clap(author, version, about, long_about = None)] -struct Args { - #[clap(short, long)] - port: Option, - #[clap(short, long)] - writeable: Option, - #[clap(short, long)] - fetch: Option, - #[clap(short, long)] - cache: Option, - #[clap(long)] - metrics: bool, - #[clap(long)] - tracing: bool, - #[clap(long)] - cfg: Option, -} - -impl Args { - fn make_overrides_map(&self) -> HashMap<&str, String> { - let mut map: HashMap<&str, String> = HashMap::new(); - if let Some(port) = self.port { - map.insert("port", port.to_string()); - } - if let Some(writable) = self.writeable { - map.insert("writable", writable.to_string()); - } - if let Some(fetch) = self.fetch { - map.insert("fetch", fetch.to_string()); - } - if let Some(cache) = self.cache { - map.insert("cache", cache.to_string()); - } - map.insert("metrics.collect", self.metrics.to_string()); - map.insert("metrics.tracing", self.tracing.to_string()); - map - } -} - #[tokio::main(flavor = "multi_thread")] async fn main() -> Result<()> { let args = Args::parse(); From 036ba8afcf56f275c26ade505d3dca0287ab850d Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Sat, 20 Aug 2022 00:01:49 +0200 Subject: [PATCH 23/29] cleanup --- iroh-gateway/src/config.rs | 6 + iroh-gateway/src/core.rs | 5 +- iroh-one/src/config.rs | 319 ++++--------------------------------- iroh-one/src/core.rs | 10 +- iroh-one/src/main.rs | 23 +-- iroh-one/src/rpc.rs | 1 + 6 files changed, 62 insertions(+), 302 deletions(-) diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index a03f0942a9..1d2e1d0dd0 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -29,6 +29,8 @@ pub struct Config { pub cache: bool, /// default port to listen on pub port: u16, + /// Gateway from which to fetch raw CIDs. TODO: move to p2p config? + pub raw_gateway: String, // NOTE: for toml to serialize properly, the "table" values must be serialized at the end, and // so much come at the end of the `Config` struct /// set of user provided headers to attach to all responses @@ -47,6 +49,7 @@ impl Config { writeable: bool, fetch: bool, cache: bool, + raw_gateway: &str, port: u16, rpc_client: RpcClientConfig, ) -> Self { @@ -56,6 +59,7 @@ impl Config { cache, headers: HeaderMap::new(), port, + raw_gateway: raw_gateway.to_owned(), rpc_client, metrics: MetricsConfig::default(), denylist: false, @@ -135,6 +139,7 @@ impl Default for Config { cache: false, headers: HeaderMap::new(), port: DEFAULT_PORT, + raw_gateway: String::new(), rpc_client, metrics: MetricsConfig::default(), denylist: false, @@ -159,6 +164,7 @@ impl Source for Config { // Some issue between deserializing u64 & u16, converting this to // an signed int fixes the issue insert_into_config_map(&mut map, "port", self.port as i32); + insert_into_config_map(&mut map, "raw_gateway", self.raw_gateway.clone()); insert_into_config_map(&mut map, "headers", collect_headers(&self.headers)?); insert_into_config_map(&mut map, "rpc_client", rpc_client); let metrics = self.metrics.collect()?; diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 8bec49de2f..d56179a210 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -15,12 +15,12 @@ use crate::{ templates, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Core { state: Arc, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct State { pub config: Arc, pub client: Client, @@ -118,6 +118,7 @@ mod tests { false, false, false, + "", 0, RpcClientConfig { gateway_addr: None, diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 63dc1932ea..47a0978fe2 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -1,15 +1,11 @@ use anyhow::{bail, Result}; -use axum::http::{header::*, Method}; +use axum::http::{header::*}; use config::{ConfigError, Map, Source, Value}; -use headers::{ - AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlAllowOrigin, HeaderMapExt, -}; -use iroh_gateway::constants::*; + use iroh_metrics::config::Config as MetricsConfig; -use iroh_p2p::{Config as FullP2pConfig, Libp2pConfig}; +use iroh_p2p::{Libp2pConfig}; use iroh_rpc_client::Config as RpcClientConfig; use iroh_rpc_types::{gateway::GatewayServerAddr, Addr}; -use iroh_store::Config as FullStoreConfig; use iroh_util::insert_into_config_map; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -27,11 +23,11 @@ pub const DEFAULT_PORT: u16 = 9050; #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct Config { // Gateway specific configuration. - pub gateway: GatewayConfig, + pub gateway: iroh_gateway::config::Config, // Store specific configuration. - pub store: StoreConfig, + pub store: iroh_store::config::Config, // P2P specific configuration. - pub p2p: iroh_p2p::Libp2pConfig, + pub p2p: iroh_p2p::config::Config, /// rpc addresses for the gateway & addresses for the rpc client to dial pub rpc_client: RpcClientConfig, @@ -39,63 +35,11 @@ pub struct Config { pub metrics: MetricsConfig, } -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub struct StoreConfig { - /// The location of the content database. - pub path: PathBuf, -} - -impl Source for StoreConfig { - fn clone_into_box(&self) -> Box { - Box::new(self.clone()) - } - - fn collect(&self) -> Result, ConfigError> { - let mut map: Map = Map::new(); - - let path = self - .path - .to_str() - .ok_or_else(|| ConfigError::Foreign("No `path` set. Path is required.".into()))?; - insert_into_config_map(&mut map, "path", path); - Ok(map) - } -} - -impl Default for StoreConfig { - fn default() -> Self { - Self { - path: PathBuf::from("./iroh-store-db"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub struct GatewayConfig { - /// flag to toggle whether the gateway allows writing/pushing data - pub writeable: bool, - /// flag to toggle whether the gateway allows fetching data from other nodes or is local only - pub fetch: bool, - /// flag to toggle whether the gateway enables/utilizes caching - pub cache: bool, - /// default port to listen on - pub port: u16, - /// Gateway from which to fetch raw CIDs. TODO: move to p2p config? - pub raw_gateway: String, - // NOTE: for toml to serialize properly, the "table" values must be serialized at the end, and - // so much come at the end of the `Config` struct - /// set of user provided headers to attach to all responses - #[serde(with = "http_serde::header_map")] - pub headers: HeaderMap, - /// flag to toggle whether the gateway should use denylist on requests - pub denylist: bool, -} - impl Config { pub fn new( - gateway: GatewayConfig, - store: StoreConfig, - p2p: Libp2pConfig, + gateway: iroh_gateway::config::Config, + store: iroh_store::config::Config, + p2p: iroh_p2p::config::Config, rpc_client: RpcClientConfig, ) -> Self { Self { @@ -154,110 +98,39 @@ impl Config { } } -#[allow(clippy::from_over_into)] -impl Into for Config { - fn into(self) -> FullStoreConfig { - FullStoreConfig { - path: self.store.path.clone(), - rpc_client: self.rpc_client.clone(), - metrics: self.metrics, - } - } -} - -#[allow(clippy::from_over_into)] -impl Into for Config { - fn into(self) -> FullP2pConfig { - FullP2pConfig { - libp2p: self.p2p.clone(), - rpc_client: self.rpc_client.clone(), - metrics: self.metrics, - } - } -} - -impl GatewayConfig { - pub fn new(writeable: bool, fetch: bool, cache: bool, port: u16, raw_gateway: &str) -> Self { +impl Default for Config { + fn default() -> Self { + let ipfsd = Self::default_ipfsd(); + let metrics_config = MetricsConfig::default(); Self { - writeable, - fetch, - cache, - headers: HeaderMap::new(), - port, - raw_gateway: raw_gateway.to_owned(), - denylist: false, + rpc_client: ipfsd.clone(), + metrics: metrics_config.clone(), + gateway: iroh_gateway::config::Config::default(), + store: default_store_config(ipfsd.clone(), metrics_config.clone()), + p2p: default_p2p_config(ipfsd, metrics_config), } } - - pub fn set_default_headers(&mut self) { - self.headers = default_headers(); - } } -fn default_headers() -> HeaderMap { - let mut headers = HeaderMap::new(); - headers.typed_insert(AccessControlAllowOrigin::ANY); - headers.typed_insert( - [ - Method::GET, - Method::PUT, - Method::POST, - Method::DELETE, - Method::HEAD, - Method::OPTIONS, - ] - .into_iter() - .collect::(), - ); - headers.typed_insert( - [ - CONTENT_TYPE, - CONTENT_DISPOSITION, - LAST_MODIFIED, - CACHE_CONTROL, - ACCEPT_RANGES, - ETAG, - HEADER_SERVICE_WORKER.clone(), - HEADER_X_IPFS_GATEWAY_PREFIX.clone(), - HEADER_X_TRACE_ID.clone(), - HEADER_X_CONTENT_TYPE_OPTIONS.clone(), - HEADER_X_IPFS_PATH.clone(), - HEADER_X_IPFS_ROOTS.clone(), - ] - .into_iter() - .collect::(), - ); - // todo(arqu): remove these once propperly implmented - headers.insert(CACHE_CONTROL, VALUE_NO_CACHE_NO_TRANSFORM.clone()); - headers.insert(ACCEPT_RANGES, VALUE_NONE.clone()); - headers -} - -impl Default for Config { - fn default() -> Self { - Self { - rpc_client: Self::default_ipfsd(), - metrics: MetricsConfig::default(), - gateway: GatewayConfig::default(), - store: StoreConfig::default(), - p2p: Libp2pConfig::default(), - } +fn default_store_config( + ipfsd: RpcClientConfig, + metrics: iroh_metrics::config::Config, +) -> iroh_store::config::Config { + iroh_store::config::Config { + path: PathBuf::new(), + rpc_client: ipfsd, + metrics, } } -impl Default for GatewayConfig { - fn default() -> Self { - let mut t = Self { - writeable: false, - fetch: false, - cache: false, - headers: HeaderMap::new(), - port: DEFAULT_PORT, - raw_gateway: "dweb.link".to_owned(), - denylist: false, - }; - t.set_default_headers(); - t +fn default_p2p_config( + ipfsd: RpcClientConfig, + metrics: iroh_metrics::config::Config, +) -> iroh_p2p::config::Config { + iroh_p2p::config::Config { + libp2p: Libp2pConfig::default(), + rpc_client: ipfsd, + metrics, } } @@ -278,26 +151,6 @@ impl Source for Config { } } -impl Source for GatewayConfig { - fn clone_into_box(&self) -> Box { - Box::new(self.clone()) - } - - fn collect(&self) -> Result, ConfigError> { - let mut map: Map = Map::new(); - insert_into_config_map(&mut map, "writeable", self.writeable); - insert_into_config_map(&mut map, "fetch", self.fetch); - insert_into_config_map(&mut map, "cache", self.cache); - insert_into_config_map(&mut map, "denylist", self.denylist); - insert_into_config_map(&mut map, "raw_gateway", self.raw_gateway.clone()); - // Some issue between deserializing u64 & u16, converting this to - // an signed int fixes the issue - insert_into_config_map(&mut map, "port", self.port as i32); - insert_into_config_map(&mut map, "headers", collect_headers(&self.headers)?); - Ok(map) - } -} - impl iroh_gateway::handlers::StateConfig for Config { fn rpc_client(&self) -> iroh_rpc_client::Config { self.rpc_client.clone() @@ -311,107 +164,3 @@ impl iroh_gateway::handlers::StateConfig for Config { self.gateway.headers.clone() } } - -fn collect_headers(headers: &HeaderMap) -> Result, ConfigError> { - let mut map = Map::new(); - for (key, value) in headers.iter() { - insert_into_config_map( - &mut map, - key.as_str(), - value.to_str().map_err(|e| ConfigError::Foreign(e.into()))?, - ); - } - Ok(map) -} - -#[cfg(test)] -mod tests { - use super::*; - use config::Config as ConfigBuilder; - - #[test] - fn test_default_headers() { - let headers = default_headers(); - assert_eq!(headers.len(), 5); - let h = headers.get(&ACCESS_CONTROL_ALLOW_ORIGIN).unwrap(); - assert_eq!(h, "*"); - } - - #[test] - fn default_config() { - let config = Config::default(); - assert!(!config.gateway.writeable); - assert!(!config.gateway.fetch); - assert!(!config.gateway.cache); - assert_eq!(config.gateway.port, DEFAULT_PORT); - } - - #[test] - fn test_collect() { - let default = Config::default(); - let mut expect: Map = Map::new(); - expect.insert( - "gateway".to_string(), - Value::new(None, default.gateway.collect().unwrap()), - ); - expect.insert( - "store".to_string(), - Value::new(None, default.store.collect().unwrap()), - ); - expect.insert( - "p2p".to_string(), - Value::new(None, default.p2p.collect().unwrap()), - ); - expect.insert( - "metrics".to_string(), - Value::new(None, default.metrics.collect().unwrap()), - ); - expect.insert( - "rpc_client".to_string(), - Value::new(None, default.rpc_client.collect().unwrap()), - ); - - let got = default.collect().unwrap(); - for key in got.keys() { - let left = expect.get(key).unwrap_or_else(|| panic!("{}", key)); - let right = got.get(key).unwrap(); - assert_eq!(left, right); - } - } - - #[test] - fn test_collect_headers() { - let mut expect = Map::new(); - expect.insert( - "access-control-allow-origin".to_string(), - Value::new(None, "*"), - ); - expect.insert( - "access-control-allow-methods".to_string(), - Value::new(None, "GET, PUT, POST, DELETE, HEAD, OPTIONS"), - ); - expect.insert("access-control-allow-headers".to_string(), Value::new(None, "content-type, content-disposition, last-modified, cache-control, accept-ranges, etag, service-worker, x-ipfs-gateway-prefix, x-trace-id, x-content-type-options, x-ipfs-path, x-ipfs-roots")); - expect.insert( - "cache-control".to_string(), - Value::new(None, "no-cache, no-transform"), - ); - expect.insert("accept-ranges".to_string(), Value::new(None, "none")); - let got = collect_headers(&default_headers()).unwrap(); - assert_eq!(expect, got); - } - - #[test] - fn test_build_config_from_struct() { - let mut expect = Config::default(); - expect.gateway.set_default_headers(); - let source = expect.clone(); - let got: Config = ConfigBuilder::builder() - .add_source(source) - .build() - .unwrap() - .try_deserialize() - .unwrap(); - - assert_eq!(expect, got); - } -} diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs index 443b98428c..e78528905f 100644 --- a/iroh-one/src/core.rs +++ b/iroh-one/src/core.rs @@ -1,4 +1,4 @@ -use crate::{rpc, rpc::Gateway, uds}; +use crate::uds; use axum::{Router, Server}; use iroh_gateway::{ bad_bits::BadBits, @@ -13,11 +13,9 @@ use iroh_rpc_types::gateway::GatewayServerAddr; use prometheus_client::registry::Registry; use std::{collections::HashMap, sync::Arc}; use tokio::{net::UnixListener, sync::RwLock}; - -#[derive(Debug)] -pub struct Core { - state: Arc, -} +use iroh_gateway::{core::State, handlers::get_app_routes}; +use std::sync::Arc; +use tokio::net::UnixListener; impl Core { pub async fn new( diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 767fb58758..41880a392a 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -1,12 +1,13 @@ -use std::sync::Arc; +use std::{path::PathBuf, sync::Arc}; use anyhow::{anyhow, Result}; use clap::Parser; -use iroh_gateway::{bad_bits::BadBits, cli::Args, metrics}; +use iroh_gateway::{bad_bits::BadBits, cli::Args, core::Core, metrics}; use iroh_metrics::gateway::Metrics; use iroh_one::{ config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, - core::Core, + // core::Core, + core, }; use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; @@ -30,14 +31,17 @@ async fn main() -> Result<()> { ) .unwrap(); + config.gateway.raw_gateway = "dweb.link".to_string(); + config.store.path = PathBuf::from("./iroh-store-db"); + let (store_rpc, p2p_rpc) = { let (store_recv, store_sender) = Addr::new_mem(); config.rpc_client.store_addr = Some(store_sender); - let store_rpc = iroh_one::mem_store::start(store_recv, config.clone().into()).await?; + let store_rpc = iroh_one::mem_store::start(store_recv, config.clone().store.into()).await?; let (p2p_recv, p2p_sender) = Addr::new_mem(); config.rpc_client.p2p_addr = Some(p2p_sender); - let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv, config.clone().into()).await?; + let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv, config.clone().p2p.into()).await?; (store_rpc, p2p_rpc) }; @@ -58,27 +62,28 @@ async fn main() -> Result<()> { false => Arc::new(None), }; - let handler = Core::new( + let shared_state = Core::make_state( Arc::new(config), - rpc_addr, gw_metrics, &mut prom_registry, Arc::clone(&bad_bits), ) .await?; + let handler = Core::new_with_state(rpc_addr, Arc::clone(&shared_state)).await?; + let metrics_handle = iroh_metrics::MetricsHandle::from_registry_with_tracer(metrics_config, prom_registry) .await .expect("failed to initialize metrics"); - let server = handler.http_server(); + let server = handler.server(); println!("HTTP endpoint listening on {}", server.local_addr()); let core_task = tokio::spawn(async move { server.await.unwrap(); }); let uds_server_task = { - let uds_server = handler.uds_server(); + let uds_server = core::uds_server(shared_state); tokio::spawn(async move { uds_server.await.unwrap(); }) diff --git a/iroh-one/src/rpc.rs b/iroh-one/src/rpc.rs index 90ca1f7288..a19b7d9830 100644 --- a/iroh-one/src/rpc.rs +++ b/iroh-one/src/rpc.rs @@ -19,6 +19,7 @@ impl iroh_rpc_types::NamedService for Gateway { const NAME: &'static str = "gateway"; } +#[allow(dead_code)] pub async fn new(addr: GatewayServerAddr, gateway: Gateway) -> Result<()> { iroh_rpc_types::gateway::serve(addr, gateway).await } From d209b76796adc566ef8c56706fb278a8649780a4 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Sat, 20 Aug 2022 14:02:49 +0200 Subject: [PATCH 24/29] fix rpc --- iroh-one/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 41880a392a..e35f5772d6 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -46,6 +46,9 @@ async fn main() -> Result<()> { }; config.rpc_client.raw_gateway = Some(config.gateway.raw_gateway.clone()); + config.gateway.rpc_client = config.rpc_client.clone(); + config.p2p.rpc_client = config.rpc_client.clone(); + config.store.rpc_client = config.rpc_client.clone(); config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); println!("{:#?}", config); From bb8d41e4a6cbd61bffef8730fb1ebd5b8c328cff Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 29 Aug 2022 13:56:53 +0200 Subject: [PATCH 25/29] fixup configs --- iroh-gateway/src/bad_bits.rs | 1 + iroh-one/.gitignore | 2 -- iroh-one/Cargo.toml | 3 +- iroh-one/src/cli.rs | 54 +++++++++++++++++++++++++++++++++++ iroh-one/src/config.rs | 55 ++++++++++++------------------------ iroh-one/src/lib.rs | 5 ++-- iroh-one/src/main.rs | 24 ++++++++-------- iroh-one/src/uds.rs | 23 +++++++++++++++ iroh-p2p/src/cli.rs | 23 +++++++++++++++ iroh-p2p/src/lib.rs | 1 + iroh-p2p/src/main.rs | 25 +--------------- iroh-store/src/cli.rs | 30 ++++++++++++++++++++ iroh-store/src/lib.rs | 1 + iroh-store/src/main.rs | 32 ++------------------- 14 files changed, 172 insertions(+), 107 deletions(-) delete mode 100644 iroh-one/.gitignore create mode 100644 iroh-one/src/cli.rs create mode 100644 iroh-p2p/src/cli.rs create mode 100644 iroh-store/src/cli.rs diff --git a/iroh-gateway/src/bad_bits.rs b/iroh-gateway/src/bad_bits.rs index f67d1ea570..f6918c274d 100644 --- a/iroh-gateway/src/bad_bits.rs +++ b/iroh-gateway/src/bad_bits.rs @@ -181,6 +181,7 @@ mod tests { false, false, false, + "", 0, RpcClientConfig { gateway_addr: None, diff --git a/iroh-one/.gitignore b/iroh-one/.gitignore deleted file mode 100644 index 48180fec31..0000000000 --- a/iroh-one/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Default path for the store. -iroh-store-db/ diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index 0049b0cbb1..ceeb6f7df7 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -35,6 +35,7 @@ axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messa http = "0.2" [features] -default = ["rpc-mem", "rpc-grpc"] +default = ["rpc-mem", "rpc-grpc", "uds-gateway"] rpc-grpc = ["iroh-rpc-types/grpc", "iroh-rpc-client/grpc", "iroh-metrics/rpc-grpc"] rpc-mem = ["iroh-rpc-types/mem", "iroh-rpc-client/mem"] +uds-gateway = [] \ No newline at end of file diff --git a/iroh-one/src/cli.rs b/iroh-one/src/cli.rs new file mode 100644 index 0000000000..db1d64e72a --- /dev/null +++ b/iroh-one/src/cli.rs @@ -0,0 +1,54 @@ +/// CLI arguments support. +use clap::Parser; +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Parser, Debug, Clone)] +#[clap(author, version, about, long_about = None)] +pub struct Args { + //Gateway + #[clap(short, long)] + port: Option, + #[clap(short, long)] + writeable: Option, + #[clap(short, long)] + fetch: Option, + #[clap(short, long)] + cache: Option, + #[clap(long)] + metrics: bool, + #[clap(long)] + tracing: bool, + #[clap(long)] + denylist: bool, + /// Path to the store + #[clap(long = "store-path")] + pub store_path: Option, + #[clap(long)] + pub cfg: Option, +} + +impl Args { + pub fn make_overrides_map(&self) -> HashMap<&str, String> { + let mut map: HashMap<&str, String> = HashMap::new(); + if let Some(port) = self.port { + map.insert("gateway.port", port.to_string()); + } + if let Some(writable) = self.writeable { + map.insert("gateway.writable", writable.to_string()); + } + if let Some(fetch) = self.fetch { + map.insert("gateway.fetch", fetch.to_string()); + } + if let Some(cache) = self.cache { + map.insert("gateway.cache", cache.to_string()); + } + map.insert("gateway.denylist", self.denylist.to_string()); + map.insert("metrics.collect", self.metrics.to_string()); + map.insert("metrics.tracing", self.tracing.to_string()); + if let Some(path) = self.store_path.clone() { + map.insert("store.path", path.to_str().unwrap_or("").to_string()); + } + map + } +} diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 47a0978fe2..532288a6b5 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -1,11 +1,11 @@ -use anyhow::{bail, Result}; -use axum::http::{header::*}; +use anyhow::Result; +use axum::http::header::*; use config::{ConfigError, Map, Source, Value}; use iroh_metrics::config::Config as MetricsConfig; -use iroh_p2p::{Libp2pConfig}; +use iroh_p2p::Libp2pConfig; use iroh_rpc_client::Config as RpcClientConfig; -use iroh_rpc_types::{gateway::GatewayServerAddr, Addr}; +use iroh_rpc_types::Addr; use iroh_util::insert_into_config_map; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -51,51 +51,32 @@ impl Config { } } - /// Derive server addr for non memory addrs. - pub fn server_rpc_addr(&self) -> Result> { - self.rpc_client - .gateway_addr - .as_ref() - .map(|addr| { - #[allow(unreachable_patterns)] - match addr { - #[cfg(feature = "rpc-grpc")] - Addr::GrpcHttp2(addr) => Ok(Addr::GrpcHttp2(*addr)), - #[cfg(all(feature = "rpc-grpc", unix))] - Addr::GrpcUds(path) => Ok(Addr::GrpcUds(path.clone())), - #[cfg(feature = "rpc-mem")] - Addr::Mem(_) => bail!("can not derive rpc_addr for mem addr"), - _ => bail!("invalid rpc_addr"), - } - }) - .transpose() - } - // When running in ipfsd mode, the resolver will use memory channels to // communicate with the p2p and store modules. // The gateway itself is exposing a UDS rpc endpoint to be also usable - // as a single entry point for other system services. + // as a single entry point for other system services if feature enabled. pub fn default_ipfsd() -> RpcClientConfig { - let path = { - #[cfg(target_os = "android")] - { - "/dev/socket/ipfsd".into() - } - - #[cfg(not(target_os = "android"))] - { - let path = format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()); - path.into() - } - }; + #[cfg(feature = "uds-gateway")] + let path: PathBuf = + format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()).into(); RpcClientConfig { + #[cfg(feature = "uds-gateway")] gateway_addr: Some(Addr::GrpcUds(path)), + #[cfg(not(feature = "uds-gateway"))] + gateway_addr: None, p2p_addr: None, store_addr: None, raw_gateway: None, } } + + // synchronize the rpc config across subsystems + pub fn sync_rpc_client(&mut self) { + self.gateway.rpc_client = self.rpc_client.clone(); + self.p2p.rpc_client = self.rpc_client.clone(); + self.store.rpc_client = self.rpc_client.clone(); + } } impl Default for Config { diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs index c66279d346..d6c51fe6b4 100644 --- a/iroh-one/src/lib.rs +++ b/iroh-one/src/lib.rs @@ -1,6 +1,7 @@ +pub mod cli; pub mod config; -pub mod core; pub mod mem_p2p; pub mod mem_store; mod rpc; -mod uds; +#[cfg(feature = "uds-gateway")] +pub mod uds; diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index e35f5772d6..bc1a4f9b74 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -1,13 +1,14 @@ -use std::{path::PathBuf, sync::Arc}; +use std::sync::Arc; use anyhow::{anyhow, Result}; use clap::Parser; -use iroh_gateway::{bad_bits::BadBits, cli::Args, core::Core, metrics}; +use iroh_gateway::{bad_bits::BadBits, core::Core, metrics}; use iroh_metrics::gateway::Metrics; +#[cfg(feature = "uds-gateway")] +use iroh_one::uds; use iroh_one::{ + cli::Args, config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}, - // core::Core, - core, }; use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; @@ -32,23 +33,20 @@ async fn main() -> Result<()> { .unwrap(); config.gateway.raw_gateway = "dweb.link".to_string(); - config.store.path = PathBuf::from("./iroh-store-db"); let (store_rpc, p2p_rpc) = { let (store_recv, store_sender) = Addr::new_mem(); config.rpc_client.store_addr = Some(store_sender); - let store_rpc = iroh_one::mem_store::start(store_recv, config.clone().store.into()).await?; + let store_rpc = iroh_one::mem_store::start(store_recv, config.clone().store).await?; let (p2p_recv, p2p_sender) = Addr::new_mem(); config.rpc_client.p2p_addr = Some(p2p_sender); - let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv, config.clone().p2p.into()).await?; + let p2p_rpc = iroh_one::mem_p2p::start(p2p_recv, config.clone().p2p).await?; (store_rpc, p2p_rpc) }; config.rpc_client.raw_gateway = Some(config.gateway.raw_gateway.clone()); - config.gateway.rpc_client = config.rpc_client.clone(); - config.p2p.rpc_client = config.rpc_client.clone(); - config.store.rpc_client = config.rpc_client.clone(); + config.sync_rpc_client(); config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); println!("{:#?}", config); @@ -57,6 +55,7 @@ async fn main() -> Result<()> { let mut prom_registry = Registry::default(); let gw_metrics = Metrics::new(&mut prom_registry); let rpc_addr = config + .gateway .server_rpc_addr()? .ok_or_else(|| anyhow!("missing gateway rpc addr"))?; @@ -85,8 +84,10 @@ async fn main() -> Result<()> { server.await.unwrap(); }); + #[cfg(feature = "uds-gateway")] let uds_server_task = { - let uds_server = core::uds_server(shared_state); + let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); + let uds_server = uds::uds_server(shared_state, path); tokio::spawn(async move { uds_server.await.unwrap(); }) @@ -96,6 +97,7 @@ async fn main() -> Result<()> { store_rpc.abort(); p2p_rpc.abort(); + #[cfg(feature = "uds-gateway")] uds_server_task.abort(); core_task.abort(); diff --git a/iroh-one/src/uds.rs b/iroh-one/src/uds.rs index 713af0c643..7fbf840261 100644 --- a/iroh-one/src/uds.rs +++ b/iroh-one/src/uds.rs @@ -1,11 +1,13 @@ /// HTTP over UDS support /// From https://github.com/tokio-rs/axum/blob/1fe45583626a4c9c890cc01131d38c57f8728686/examples/unix-domain-socket/src/main.rs use axum::extract::connect_info; +use axum::{Router, Server}; use futures::ready; use hyper::{ client::connect::{Connected, Connection}, server::accept::Accept, }; +use iroh_gateway::{core::State, handlers::get_app_routes}; use std::{ io, pin::Pin, @@ -93,3 +95,24 @@ impl connect_info::Connected<&UnixStream> for UdsConnectInfo { } } } + +pub fn uds_server( + state: Arc, + path: String, +) -> Server< + ServerAccept, + axum::extract::connect_info::IntoMakeServiceWithConnectInfo, +> { + // #[cfg(target_os = "android")] + // let path = "/dev/socket/ipfsd.http".to_owned(); + + // #[cfg(not(target_os = "android"))] + // let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); + + let _ = std::fs::remove_file(&path); + let uds = UnixListener::bind(&path).unwrap(); + println!("Binding to UDS at {}", path); + let app = get_app_routes(&state); + Server::builder(ServerAccept { uds }) + .serve(app.into_make_service_with_connect_info::()) +} diff --git a/iroh-p2p/src/cli.rs b/iroh-p2p/src/cli.rs new file mode 100644 index 0000000000..e38baec707 --- /dev/null +++ b/iroh-p2p/src/cli.rs @@ -0,0 +1,23 @@ +use std::{collections::HashMap, path::PathBuf}; + +use clap::Parser; + +#[derive(Parser, Debug, Clone)] +#[clap(author, version, about, long_about = None)] +pub struct Args { + #[clap(long = "metrics")] + metrics: bool, + #[clap(long = "tracing")] + tracing: bool, + #[clap(long)] + pub cfg: Option, +} + +impl Args { + pub fn make_overrides_map(&self) -> HashMap { + let mut map = HashMap::new(); + map.insert("metrics.collect".to_string(), self.metrics.to_string()); + map.insert("metrics.tracing".to_string(), self.tracing.to_string()); + map + } +} diff --git a/iroh-p2p/src/lib.rs b/iroh-p2p/src/lib.rs index d64afc1dfe..aebf973a1e 100644 --- a/iroh-p2p/src/lib.rs +++ b/iroh-p2p/src/lib.rs @@ -1,4 +1,5 @@ mod behaviour; +pub mod cli; pub mod config; mod keys; pub mod metrics; diff --git a/iroh-p2p/src/main.rs b/iroh-p2p/src/main.rs index 608b0d5911..643080ba37 100644 --- a/iroh-p2p/src/main.rs +++ b/iroh-p2p/src/main.rs @@ -1,35 +1,12 @@ -use std::collections::HashMap; -use std::path::PathBuf; - use anyhow::anyhow; use clap::Parser; use iroh_p2p::config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}; -use iroh_p2p::{metrics, DiskStorage, Keychain, Node}; +use iroh_p2p::{cli::Args, metrics, DiskStorage, Keychain, Node}; use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; use tokio::task; use tracing::error; -#[derive(Parser, Debug, Clone)] -#[clap(author, version, about, long_about = None)] -struct Args { - #[clap(long = "metrics")] - metrics: bool, - #[clap(long = "tracing")] - tracing: bool, - #[clap(long)] - cfg: Option, -} - -impl Args { - fn make_overrides_map(&self) -> HashMap { - let mut map = HashMap::new(); - map.insert("metrics.collect".to_string(), self.metrics.to_string()); - map.insert("metrics.tracing".to_string(), self.tracing.to_string()); - map - } -} - /// Starts daemon process #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { diff --git a/iroh-store/src/cli.rs b/iroh-store/src/cli.rs new file mode 100644 index 0000000000..7d328fc0f9 --- /dev/null +++ b/iroh-store/src/cli.rs @@ -0,0 +1,30 @@ +use std::{collections::HashMap, path::PathBuf}; + +use clap::Parser; + +#[derive(Parser, Debug)] +#[clap(author, version, about)] +pub struct Args { + /// Path to the store + #[clap(long, short)] + pub path: Option, + #[clap(long = "metrics")] + metrics: bool, + #[clap(long = "tracing")] + tracing: bool, + /// Path to the config file + #[clap(long)] + pub cfg: Option, +} + +impl Args { + pub fn make_overrides_map(&self) -> HashMap { + let mut map = HashMap::new(); + if let Some(path) = self.path.clone() { + map.insert("path".to_string(), path.to_str().unwrap_or("").to_string()); + } + map.insert("metrics.collect".to_string(), self.metrics.to_string()); + map.insert("metrics.tracing".to_string(), self.tracing.to_string()); + map + } +} diff --git a/iroh-store/src/lib.rs b/iroh-store/src/lib.rs index 72862fabd1..bffdc4af2b 100644 --- a/iroh-store/src/lib.rs +++ b/iroh-store/src/lib.rs @@ -1,4 +1,5 @@ mod cf; +pub mod cli; pub mod config; pub mod metrics; pub mod rpc; diff --git a/iroh-store/src/main.rs b/iroh-store/src/main.rs index 7da0e10aad..89ee6dc07e 100644 --- a/iroh-store/src/main.rs +++ b/iroh-store/src/main.rs @@ -1,44 +1,16 @@ -use std::collections::HashMap; -use std::path::PathBuf; - use anyhow::anyhow; use clap::Parser; use iroh_metrics::store::Metrics; use iroh_store::{ + cli::Args, config::{CONFIG_FILE_NAME, ENV_PREFIX}, metrics, rpc, Config, Store, }; use iroh_util::{block_until_sigint, iroh_home_path, make_config}; use prometheus_client::registry::Registry; +use std::path::PathBuf; use tracing::info; -#[derive(Parser, Debug)] -#[clap(author, version, about)] -struct Args { - /// Path to the store - #[clap(long, short)] - path: Option, - #[clap(long = "metrics")] - metrics: bool, - #[clap(long = "tracing")] - tracing: bool, - /// Path to the config file - #[clap(long)] - cfg: Option, -} - -impl Args { - fn make_overrides_map(&self) -> HashMap { - let mut map = HashMap::new(); - if let Some(path) = self.path.clone() { - map.insert("path".to_string(), path.to_str().unwrap_or("").to_string()); - } - map.insert("metrics.collect".to_string(), self.metrics.to_string()); - map.insert("metrics.tracing".to_string(), self.tracing.to_string()); - map - } -} - #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { let args = Args::parse(); From 50a628f64bda2fccda720a30124f4f0e811705c8 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Mon, 29 Aug 2022 15:47:31 +0200 Subject: [PATCH 26/29] further cleanup --- iroh-gateway/src/bad_bits.rs | 2 - iroh-gateway/src/config.rs | 6 -- iroh-gateway/src/core.rs | 2 - iroh-gateway/src/handlers.rs | 7 +- iroh-one/Cargo.toml | 2 +- iroh-one/src/cli.rs | 7 ++ iroh-one/src/config.rs | 17 ++++- iroh-one/src/core.rs | 131 ---------------------------------- iroh-one/src/main.rs | 20 ++++-- iroh-one/src/uds.rs | 11 +-- iroh-resolver/Cargo.toml | 1 - iroh-resolver/src/resolver.rs | 124 ++++++++------------------------ iroh-rpc-client/src/client.rs | 9 --- iroh-rpc-client/src/config.rs | 6 -- iroh-share/src/p2p_node.rs | 2 - 15 files changed, 78 insertions(+), 269 deletions(-) delete mode 100644 iroh-one/src/core.rs diff --git a/iroh-gateway/src/bad_bits.rs b/iroh-gateway/src/bad_bits.rs index f6918c274d..c945b2b45f 100644 --- a/iroh-gateway/src/bad_bits.rs +++ b/iroh-gateway/src/bad_bits.rs @@ -181,13 +181,11 @@ mod tests { false, false, false, - "", 0, RpcClientConfig { gateway_addr: None, p2p_addr: None, store_addr: None, - raw_gateway: None, }, ); config.set_default_headers(); diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index 1d2e1d0dd0..a03f0942a9 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -29,8 +29,6 @@ pub struct Config { pub cache: bool, /// default port to listen on pub port: u16, - /// Gateway from which to fetch raw CIDs. TODO: move to p2p config? - pub raw_gateway: String, // NOTE: for toml to serialize properly, the "table" values must be serialized at the end, and // so much come at the end of the `Config` struct /// set of user provided headers to attach to all responses @@ -49,7 +47,6 @@ impl Config { writeable: bool, fetch: bool, cache: bool, - raw_gateway: &str, port: u16, rpc_client: RpcClientConfig, ) -> Self { @@ -59,7 +56,6 @@ impl Config { cache, headers: HeaderMap::new(), port, - raw_gateway: raw_gateway.to_owned(), rpc_client, metrics: MetricsConfig::default(), denylist: false, @@ -139,7 +135,6 @@ impl Default for Config { cache: false, headers: HeaderMap::new(), port: DEFAULT_PORT, - raw_gateway: String::new(), rpc_client, metrics: MetricsConfig::default(), denylist: false, @@ -164,7 +159,6 @@ impl Source for Config { // Some issue between deserializing u64 & u16, converting this to // an signed int fixes the issue insert_into_config_map(&mut map, "port", self.port as i32); - insert_into_config_map(&mut map, "raw_gateway", self.raw_gateway.clone()); insert_into_config_map(&mut map, "headers", collect_headers(&self.headers)?); insert_into_config_map(&mut map, "rpc_client", rpc_client); let metrics = self.metrics.collect()?; diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index d56179a210..ff637165e9 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -118,13 +118,11 @@ mod tests { false, false, false, - "", 0, RpcClientConfig { gateway_addr: None, p2p_addr: None, store_addr: None, - raw_gateway: None, }, ); config.set_default_headers(); diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index 7badceb5a6..80a734ab7a 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -12,7 +12,10 @@ use bytes::Bytes; use futures::TryStreamExt; use handlebars::Handlebars; use iroh_metrics::get_current_trace_id; -use iroh_resolver::{resolver::{CidOrDomain, UnixfsType, OutMetrics}, unixfs::Link}; +use iroh_resolver::{ + resolver::{CidOrDomain, OutMetrics, UnixfsType}, + unixfs::Link, +}; use serde::{Deserialize, Serialize}; use serde_json::{ json, @@ -33,7 +36,7 @@ use url::Url; use urlencoding::encode; use crate::{ - client::{Request, FileResult}, + client::{FileResult, Request}, constants::*, core::State, error::GatewayError, diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index ceeb6f7df7..bf938f6fcf 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -35,7 +35,7 @@ axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messa http = "0.2" [features] -default = ["rpc-mem", "rpc-grpc", "uds-gateway"] +default = ["rpc-mem", "rpc-grpc"] rpc-grpc = ["iroh-rpc-types/grpc", "iroh-rpc-client/grpc", "iroh-metrics/rpc-grpc"] rpc-mem = ["iroh-rpc-types/mem", "iroh-rpc-client/mem"] uds-gateway = [] \ No newline at end of file diff --git a/iroh-one/src/cli.rs b/iroh-one/src/cli.rs index db1d64e72a..6fc6f77fca 100644 --- a/iroh-one/src/cli.rs +++ b/iroh-one/src/cli.rs @@ -24,6 +24,9 @@ pub struct Args { /// Path to the store #[clap(long = "store-path")] pub store_path: Option, + #[cfg(feature = "uds-gateway")] + #[clap(long = "uds-path")] + pub uds_path: Option, #[clap(long)] pub cfg: Option, } @@ -49,6 +52,10 @@ impl Args { if let Some(path) = self.store_path.clone() { map.insert("store.path", path.to_str().unwrap_or("").to_string()); } + #[cfg(feature = "uds-gateway")] + if let Some(path) = self.uds_path.clone() { + map.insert("uds_path", path.to_str().unwrap_or("").to_string()); + } map } } diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 532288a6b5..aae004bf5d 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -5,6 +5,7 @@ use config::{ConfigError, Map, Source, Value}; use iroh_metrics::config::Config as MetricsConfig; use iroh_p2p::Libp2pConfig; use iroh_rpc_client::Config as RpcClientConfig; +#[cfg(feature = "uds-gateway")] use iroh_rpc_types::Addr; use iroh_util::insert_into_config_map; use serde::{Deserialize, Serialize}; @@ -33,6 +34,9 @@ pub struct Config { pub rpc_client: RpcClientConfig, /// metrics configuration pub metrics: MetricsConfig, + /// Path for the UDS socket for the gateway. + #[cfg(feature = "uds-gateway")] + pub uds_path: Option, } impl Config { @@ -41,6 +45,7 @@ impl Config { store: iroh_store::config::Config, p2p: iroh_p2p::config::Config, rpc_client: RpcClientConfig, + #[cfg(feature = "uds-gateway")] uds_path: Option, ) -> Self { Self { gateway, @@ -48,6 +53,8 @@ impl Config { p2p, rpc_client, metrics: MetricsConfig::default(), + #[cfg(feature = "uds-gateway")] + uds_path, } } @@ -67,7 +74,6 @@ impl Config { gateway_addr: None, p2p_addr: None, store_addr: None, - raw_gateway: None, } } @@ -81,6 +87,9 @@ impl Config { impl Default for Config { fn default() -> Self { + #[cfg(feature = "uds-gateway")] + let uds_path: PathBuf = + format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()).into(); let ipfsd = Self::default_ipfsd(); let metrics_config = MetricsConfig::default(); Self { @@ -89,6 +98,8 @@ impl Default for Config { gateway: iroh_gateway::config::Config::default(), store: default_store_config(ipfsd.clone(), metrics_config.clone()), p2p: default_p2p_config(ipfsd, metrics_config), + #[cfg(feature = "uds-gateway")] + uds_path: Some(uds_path), } } } @@ -128,6 +139,10 @@ impl Source for Config { insert_into_config_map(&mut map, "p2p", self.p2p.collect()?); insert_into_config_map(&mut map, "rpc_client", self.rpc_client.collect()?); insert_into_config_map(&mut map, "metrics", self.metrics.collect()?); + #[cfg(feature = "uds-gateway")] + if let Some(uds_path) = self.uds_path.as_ref() { + insert_into_config_map(&mut map, "uds_path", uds_path.to_str().unwrap().to_string()); + } Ok(map) } } diff --git a/iroh-one/src/core.rs b/iroh-one/src/core.rs deleted file mode 100644 index e78528905f..0000000000 --- a/iroh-one/src/core.rs +++ /dev/null @@ -1,131 +0,0 @@ -use crate::uds; -use axum::{Router, Server}; -use iroh_gateway::{ - bad_bits::BadBits, - client::Client, - core::State, - handlers::{get_app_routes, StateConfig}, - templates, -}; -use iroh_metrics::gateway::Metrics; -use iroh_rpc_client::Client as RpcClient; -use iroh_rpc_types::gateway::GatewayServerAddr; -use prometheus_client::registry::Registry; -use std::{collections::HashMap, sync::Arc}; -use tokio::{net::UnixListener, sync::RwLock}; -use iroh_gateway::{core::State, handlers::get_app_routes}; -use std::sync::Arc; -use tokio::net::UnixListener; - -impl Core { - pub async fn new( - config: Arc, - rpc_addr: GatewayServerAddr, - metrics: Metrics, - registry: &mut Registry, - bad_bits: Arc>>, - ) -> anyhow::Result { - tokio::spawn(async move { - // TODO: handle error - rpc::new(rpc_addr, Gateway::default()).await - }); - let rpc_client = RpcClient::new(config.rpc_client()).await?; - let mut templates = HashMap::new(); - templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); - templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); - let client = Client::new(&rpc_client, registry); - - Ok(Self { - state: Arc::new(iroh_gateway::core::State { - config, - client, - metrics, - handlebars: templates, - bad_bits, - }), - }) - } - - pub fn http_server( - &self, - ) -> Server> { - let app = get_app_routes(&self.state); - // todo(arqu): make configurable - let addr = format!("0.0.0.0:{}", self.state.config.port()); - - Server::bind(&addr.parse().unwrap()) - .http1_preserve_header_case(true) - .http1_title_case_headers(true) - .serve(app.into_make_service()) - } - - pub fn uds_server( - &self, - ) -> Server< - uds::ServerAccept, - axum::extract::connect_info::IntoMakeServiceWithConnectInfo, - > { - #[cfg(target_os = "android")] - let path = "/dev/socket/ipfsd.http".to_owned(); - - #[cfg(not(target_os = "android"))] - let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); - - let _ = std::fs::remove_file(&path); - let uds = UnixListener::bind(&path).unwrap(); - println!("Binding to UDS at {}", path); - let app = get_app_routes(&self.state); - Server::builder(uds::ServerAccept { uds }) - .serve(app.into_make_service_with_connect_info::()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::{Config, GatewayConfig}; - use prometheus_client::registry::Registry; - - #[tokio::test] - async fn gateway_health() { - let mut gateway = GatewayConfig::default(); - gateway.set_default_headers(); - let config = Config { - gateway, - ..Default::default() - }; - - let mut prom_registry = Registry::default(); - let gw_metrics = Metrics::new(&mut prom_registry); - let rpc_addr = "grpc://0.0.0.0:0".parse().unwrap(); - let handler = Core::new( - Arc::new(config), - rpc_addr, - gw_metrics, - &mut prom_registry, - Arc::new(None), - ) - .await - .unwrap(); - let server = handler.http_server(); - let addr = server.local_addr(); - let core_task = tokio::spawn(async move { - server.await.unwrap(); - }); - - let uri = hyper::Uri::builder() - .scheme("http") - .authority(format!("localhost:{}", addr.port())) - .path_and_query("/health") - .build() - .unwrap(); - let client = hyper::Client::new(); - let res = client.get(uri).await.unwrap(); - - assert_eq!(http::StatusCode::OK, res.status()); - let body = hyper::body::to_bytes(res.into_body()).await.unwrap(); - assert_eq!(b"OK", &body[..]); - core_task.abort(); - core_task.await.unwrap_err(); - } -} diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index bc1a4f9b74..874973b18a 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +#[allow(unused_imports)] use anyhow::{anyhow, Result}; use clap::Parser; use iroh_gateway::{bad_bits::BadBits, core::Core, metrics}; @@ -32,8 +33,6 @@ async fn main() -> Result<()> { ) .unwrap(); - config.gateway.raw_gateway = "dweb.link".to_string(); - let (store_rpc, p2p_rpc) = { let (store_recv, store_sender) = Addr::new_mem(); config.rpc_client.store_addr = Some(store_sender); @@ -45,7 +44,13 @@ async fn main() -> Result<()> { (store_rpc, p2p_rpc) }; - config.rpc_client.raw_gateway = Some(config.gateway.raw_gateway.clone()); + #[cfg(not(feature = "uds-gateway"))] + let (rpc_addr, gw_sender) = Addr::new_mem(); + #[cfg(not(feature = "uds-gateway"))] + { + config.rpc_client.gateway_addr = Some(gw_sender); + } + config.sync_rpc_client(); config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); @@ -54,6 +59,8 @@ async fn main() -> Result<()> { let metrics_config = config.metrics.clone(); let mut prom_registry = Registry::default(); let gw_metrics = Metrics::new(&mut prom_registry); + + #[cfg(feature = "uds-gateway")] let rpc_addr = config .gateway .server_rpc_addr()? @@ -65,7 +72,7 @@ async fn main() -> Result<()> { }; let shared_state = Core::make_state( - Arc::new(config), + Arc::new(config.clone()), gw_metrics, &mut prom_registry, Arc::clone(&bad_bits), @@ -86,7 +93,10 @@ async fn main() -> Result<()> { #[cfg(feature = "uds-gateway")] let uds_server_task = { - let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); + let mut path = std::env::temp_dir().join("ipfsd.http"); + if let Some(uds_path) = config.uds_path { + path = uds_path; + } let uds_server = uds::uds_server(shared_state, path); tokio::spawn(async move { uds_server.await.unwrap(); diff --git a/iroh-one/src/uds.rs b/iroh-one/src/uds.rs index 7fbf840261..89f63abff7 100644 --- a/iroh-one/src/uds.rs +++ b/iroh-one/src/uds.rs @@ -8,6 +8,7 @@ use hyper::{ server::accept::Accept, }; use iroh_gateway::{core::State, handlers::get_app_routes}; +use std::path::PathBuf; use std::{ io, pin::Pin, @@ -98,20 +99,14 @@ impl connect_info::Connected<&UnixStream> for UdsConnectInfo { pub fn uds_server( state: Arc, - path: String, + path: PathBuf, ) -> Server< ServerAccept, axum::extract::connect_info::IntoMakeServiceWithConnectInfo, > { - // #[cfg(target_os = "android")] - // let path = "/dev/socket/ipfsd.http".to_owned(); - - // #[cfg(not(target_os = "android"))] - // let path = format!("{}", std::env::temp_dir().join("ipfsd.http").display()); - let _ = std::fs::remove_file(&path); let uds = UnixListener::bind(&path).unwrap(); - println!("Binding to UDS at {}", path); + println!("Binding to UDS at {}", path.display()); let app = get_app_routes(&state); Server::builder(ServerAccept { uds }) .serve(app.into_make_service_with_connect_info::()) diff --git a/iroh-resolver/Cargo.toml b/iroh-resolver/Cargo.toml index 289b736512..7b62d8e6ee 100644 --- a/iroh-resolver/Cargo.toml +++ b/iroh-resolver/Cargo.toml @@ -28,7 +28,6 @@ async-stream = "0.3.3" fastmurmur3 = "0.1.2" once_cell = "1.13.0" tokio-util = { version = "0.7", features = ["io"] } -reqwest = { version = "0.11.10", features = ["rustls-tls"], default-features = false} [dev-dependencies] criterion = { version = "0.3.5", features = ["async_tokio"] } diff --git a/iroh-resolver/src/resolver.rs b/iroh-resolver/src/resolver.rs index f20417bb6b..e5caa0434c 100644 --- a/iroh-resolver/src/resolver.rs +++ b/iroh-resolver/src/resolver.rs @@ -9,9 +9,8 @@ use std::time::Instant; use anyhow::{anyhow, bail, ensure, Context as _, Result}; use async_trait::async_trait; use bytes::Bytes; -use cid::{multibase, Cid}; +use cid::Cid; use futures::Stream; -use futures::{future::FutureExt, pin_mut, select}; use iroh_metrics::resolver::Metrics; use iroh_rpc_client::Client; use libipld::codec::{Decode, Encode}; @@ -19,7 +18,7 @@ use libipld::prelude::Codec as _; use libipld::{Ipld, IpldCodec}; use prometheus_client::registry::Registry; use tokio::io::AsyncRead; -use tracing::{debug, error, trace, warn}; +use tracing::{debug, trace, warn}; use crate::codecs::Codec; use crate::unixfs::{ @@ -382,7 +381,6 @@ pub struct LoadedCid { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Source { Bitswap, - Http, Store(&'static str), } @@ -405,31 +403,6 @@ impl ContentLoader for Arc { } } -#[async_trait] -trait CidFetcher { - async fn fetch_p2p(&self, cid: &Cid) -> Result; - - async fn fetch_http(&self, cid: &Cid) -> Result; -} - -#[async_trait] -impl CidFetcher for Client { - async fn fetch_p2p(&self, cid: &Cid) -> Result { - let p2p = self.try_p2p()?; - let providers = p2p.fetch_providers(cid).await?; - p2p.fetch_bitswap(*cid, providers).await - } - - async fn fetch_http(&self, cid: &Cid) -> Result { - let gateway = self.try_raw_gateway()?; - let cid_str = multibase::encode(multibase::Base::Base32Lower, cid.to_bytes().as_slice()); - let gateway_url = format!("https://{}.ipfs.{}?format=raw", cid_str, gateway); - debug!("Will fetch {}", gateway_url); - let response = reqwest::get(gateway_url).await?; - response.bytes().await.map_err(|e| e.into()) - } -} - #[async_trait] impl ContentLoader for Client { async fn load_cid(&self, cid: &Cid) -> Result { @@ -449,75 +422,40 @@ impl ContentLoader for Client { warn!("failed to fetch data from store {}: {:?}", cid, err); } } - - let p2p_fut = self.fetch_p2p(&cid).fuse(); - let http_fut = self.fetch_http(&cid).fuse(); - pin_mut!(p2p_fut, http_fut); - - let mut bytes: Option = None; - let mut source = Source::Bitswap; - - // Race the p2p and http fetches. - loop { - select! { - res = http_fut => { - if let Ok(data) = res { - debug!("retrieved from http"); - if let Some(true) = iroh_util::verify_hash(&cid, &data) { - source = Source::Http; - bytes = Some(data); - break; - } else { - error!("Got http data, but CID verification failed!"); - } - } - } - res = p2p_fut => { - if let Ok(data) = res { - debug!("retrieved from p2p"); - bytes = Some(data); - break; + let p2p = self.try_p2p()?; + let providers = p2p.fetch_providers(&cid).await?; + let bytes = p2p.fetch_bitswap(cid, providers).await?; + + // trigger storage in the background + let cloned = bytes.clone(); + let rpc = self.clone(); + tokio::spawn(async move { + let clone2 = cloned.clone(); + let links = + tokio::task::spawn_blocking(move || parse_links(&cid, &clone2).unwrap_or_default()) + .await + .unwrap_or_default(); + + let len = cloned.len(); + let links_len = links.len(); + if let Some(store_rpc) = rpc.store.as_ref() { + match store_rpc.put(cid, cloned, links).await { + Ok(_) => debug!("stored {} ({}bytes, {}links)", cid, len, links_len), + Err(err) => { + warn!("failed to store {}: {:?}", cid, err); } } - complete => { break; } + } else { + warn!("failed to store: missing store rpc conn"); } - } + }); - if let Some(bytes) = bytes { - // trigger storage in the background - let cloned = bytes.clone(); - let rpc = self.clone(); - tokio::spawn(async move { - let clone2 = cloned.clone(); - let links = tokio::task::spawn_blocking(move || { - parse_links(&cid, &clone2).unwrap_or_default() - }) - .await - .unwrap_or_default(); - - if let Some(store_rpc) = rpc.store.as_ref() { - let len = cloned.len(); - let links_len = links.len(); - match store_rpc.put(cid, cloned, links).await { - Ok(_) => { - debug!("stored {} ({} bytes, {} links)", cid, len, links_len) - } - Err(err) => { - error!("failed to store {}: {:?}", cid, err); - } - } - } else { - error!("failed to store: missing store rpc conn"); - } - }); + trace!("retrieved from p2p"); - Ok(LoadedCid { - data: bytes, - source, - }) - } else { - Err(anyhow::anyhow!("Failed to load from p2p & http")) - } + Ok(LoadedCid { + data: bytes, + source: Source::Bitswap, + }) } } diff --git a/iroh-rpc-client/src/client.rs b/iroh-rpc-client/src/client.rs index befc7aa3d3..16e6f530c6 100644 --- a/iroh-rpc-client/src/client.rs +++ b/iroh-rpc-client/src/client.rs @@ -12,7 +12,6 @@ pub struct Client { pub gateway: Option, pub p2p: Option, pub store: Option, - pub raw_gateway: Option, } impl Client { @@ -21,7 +20,6 @@ impl Client { gateway_addr, p2p_addr, store_addr, - raw_gateway, } = cfg; let gateway = if let Some(addr) = gateway_addr { @@ -57,16 +55,9 @@ impl Client { gateway, p2p, store, - raw_gateway, }) } - pub fn try_raw_gateway(&self) -> Result<&String> { - self.raw_gateway - .as_ref() - .ok_or_else(|| anyhow!("no gateway configured to fetch raw CIDs")) - } - pub fn try_p2p(&self) -> Result<&P2pClient> { self.p2p .as_ref() diff --git a/iroh-rpc-client/src/config.rs b/iroh-rpc-client/src/config.rs index 072d2ffa52..352acce228 100644 --- a/iroh-rpc-client/src/config.rs +++ b/iroh-rpc-client/src/config.rs @@ -12,8 +12,6 @@ pub struct Config { pub p2p_addr: Option, // store rpc address pub store_addr: Option, - // Domain name of a gateway to fetch raw CIDs. - pub raw_gateway: Option, } impl Source for Config { @@ -32,9 +30,6 @@ impl Source for Config { if let Some(addr) = &self.store_addr { insert_into_config_map(&mut map, "store_addr", addr.to_string()); } - if let Some(path) = &self.raw_gateway { - insert_into_config_map(&mut map, "raw_gateway", path.clone()); - } Ok(map) } } @@ -45,7 +40,6 @@ impl Config { gateway_addr: Some("grpc://0.0.0.0:4400".parse().unwrap()), p2p_addr: Some("grpc://0.0.0.0:4401".parse().unwrap()), store_addr: Some("grpc://0.0.0.0:4402".parse().unwrap()), - raw_gateway: None, } } } diff --git a/iroh-share/src/p2p_node.rs b/iroh-share/src/p2p_node.rs index 60be79b5bd..1a6581ced6 100644 --- a/iroh-share/src/p2p_node.rs +++ b/iroh-share/src/p2p_node.rs @@ -120,13 +120,11 @@ impl P2pNode { p2p_addr: Some(rpc_p2p_addr_client.clone()), store_addr: Some(rpc_store_addr_client.clone()), gateway_addr: None, - raw_gateway: None, }; let rpc_p2p_client_config = iroh_rpc_client::Config { p2p_addr: Some(rpc_p2p_addr_client.clone()), store_addr: Some(rpc_store_addr_client.clone()), gateway_addr: None, - raw_gateway: None, }; let config = config::Config { libp2p: config::Libp2pConfig { From bed1853f30e1e0c4fd418c3a40a4363092608088 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Tue, 30 Aug 2022 10:32:01 +0200 Subject: [PATCH 27/29] cr --- iroh-gateway/src/config.rs | 8 +++---- iroh-gateway/src/core.rs | 4 ++-- iroh-gateway/src/handlers.rs | 6 ++--- iroh-one/Cargo.toml | 1 + iroh-one/README.md | 24 ++++++++----------- iroh-one/src/cli.rs | 18 +++++++------- iroh-one/src/config.rs | 46 ++++++++++++++++++++---------------- iroh-one/src/lib.rs | 1 - iroh-one/src/main.rs | 6 +++-- iroh-one/src/rpc.rs | 25 -------------------- 10 files changed, 58 insertions(+), 81 deletions(-) delete mode 100644 iroh-one/src/rpc.rs diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index a03f0942a9..b697ecca21 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -168,16 +168,16 @@ impl Source for Config { } impl crate::handlers::StateConfig for Config { - fn rpc_client(&self) -> iroh_rpc_client::Config { - self.rpc_client.clone() + fn rpc_client(&self) -> &iroh_rpc_client::Config { + &self.rpc_client } fn port(&self) -> u16 { self.port } - fn user_headers(&self) -> HeaderMap { - self.headers.clone() + fn user_headers(&self) -> &HeaderMap { + &self.headers } } diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index ff637165e9..8e5b0913cb 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -41,7 +41,7 @@ impl Core { // TODO: handle error rpc::new(rpc_addr, Gateway::default()).await }); - let rpc_client = RpcClient::new(config.rpc_client()).await?; + let rpc_client = RpcClient::new(config.rpc_client().clone()).await?; let mut templates = HashMap::new(); templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); @@ -75,7 +75,7 @@ impl Core { registry: &mut Registry, bad_bits: Arc>>, ) -> anyhow::Result> { - let rpc_client = RpcClient::new(config.rpc_client()).await?; + let rpc_client = RpcClient::new(config.rpc_client().clone()).await?; let mut templates = HashMap::new(); templates.insert("dir_list".to_string(), templates::DIR_LIST.to_string()); templates.insert("not_found".to_string(), templates::NOT_FOUND.to_string()); diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index 80a734ab7a..2458412b12 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -47,11 +47,11 @@ use crate::{ /// Trait describing what needs to be accessed on the configuration /// from the shared state. pub trait StateConfig: std::fmt::Debug + Sync + Send { - fn rpc_client(&self) -> iroh_rpc_client::Config; + fn rpc_client(&self) -> &iroh_rpc_client::Config; fn port(&self) -> u16; - fn user_headers(&self) -> HeaderMap; + fn user_headers(&self) -> &HeaderMap; } pub fn get_app_routes(state: &Arc) -> Router { @@ -183,7 +183,7 @@ pub async fn get_handler( // init headers format.write_headers(&mut headers); - add_user_headers(&mut headers, state.config.user_headers()); + add_user_headers(&mut headers, state.config.user_headers().clone()); headers.insert( &HEADER_X_IPFS_PATH, HeaderValue::from_str(&full_content_path).unwrap(), diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index bf938f6fcf..6a5498eb67 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -29,6 +29,7 @@ prometheus-client = "0.17.0" serde = {version = "1.0", features = ["derive"]} tokio = {version = "1", features = ["macros", "rt-multi-thread", "process"]} tracing = "0.1.33" +tempdir = "0.3.7" [dev-dependencies] axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers diff --git a/iroh-one/README.md b/iroh-one/README.md index a74c03aebc..f9525fe448 100644 --- a/iroh-one/README.md +++ b/iroh-one/README.md @@ -1,28 +1,24 @@ -# Iroh One Gateway +# Iroh One -A rust implementation of an IPFS gateway. +Single binary of iroh services (gateway, p2p, store) communicating via mem channels. ## Running / Building -`cargo run --release -- -p 10000` +`cargo run --release -- -p 10000 --store-path=tmpstore` ### Options - Run with `cargo run --release -- -h` for details - `-wcf` Writeable, Cache, Fetch (options to toggle write enable, caching mechanics and fetching from the network); currently exists but is not implemented - `-p` Port the gateway should listen on +- `--store-path` Path for the iroh-store -## ENV Variables +### Features -- `IROH_INSTANCE_ID` - unique instance identifier, preferably some name than hard id (default: generated lower & snake case name) -- `IROH_ENV` - indicates the service environment (default: `dev`) +- `uds-gateway` - enables the usage and binding of the gateway over UDS. -## Endpoints +### Reference -| Endpoint | Flag | Description | Default | -|-----------------------------------|--------------------------------------------|-----------------------------------------------------------------------------------------|-------------| -| `/ipfs/:cid` & `/ipfs/:cid/:path` | `?format={"", "fs", "raw", "car"}` | Specifies the serving format & content-type | `""/fs` | -| | `?filename=DESIRED_FILE_NAME` | Specifies a filename for the attachment | `{cid}.bin` | -| | `?download={true, false}` | Sets content-disposition to attachment, browser prompts to save file instead of loading | `false` | -| | `?force_dir={true, false}` | Lists unixFS directories even if they contain an `index.html` file | `false` | -| | `?uri=ENCODED_URL` | Query parameter to handle navigator.registerProtocolHandler Web API ie. ipfs:// | `""` | \ No newline at end of file +- [Gateway](../iroh-gateway/README.md) +- [P2P](../iroh-p2p/README.md) +- [Store](../iroh-store/README.md) \ No newline at end of file diff --git a/iroh-one/src/cli.rs b/iroh-one/src/cli.rs index 6fc6f77fca..37dc7637e4 100644 --- a/iroh-one/src/cli.rs +++ b/iroh-one/src/cli.rs @@ -6,9 +6,9 @@ use std::path::PathBuf; #[derive(Parser, Debug, Clone)] #[clap(author, version, about, long_about = None)] pub struct Args { - //Gateway - #[clap(short, long)] - port: Option, + /// Gateway + #[clap(short = 'p', long = "gateway-port")] + gateway_port: Option, #[clap(short, long)] writeable: Option, #[clap(short, long)] @@ -21,12 +21,12 @@ pub struct Args { tracing: bool, #[clap(long)] denylist: bool, + #[cfg(feature = "uds-gateway")] + #[clap(long = "gateway-uds-path")] + pub gateway_uds_path: Option, /// Path to the store #[clap(long = "store-path")] pub store_path: Option, - #[cfg(feature = "uds-gateway")] - #[clap(long = "uds-path")] - pub uds_path: Option, #[clap(long)] pub cfg: Option, } @@ -34,7 +34,7 @@ pub struct Args { impl Args { pub fn make_overrides_map(&self) -> HashMap<&str, String> { let mut map: HashMap<&str, String> = HashMap::new(); - if let Some(port) = self.port { + if let Some(port) = self.gateway_port { map.insert("gateway.port", port.to_string()); } if let Some(writable) = self.writeable { @@ -53,8 +53,8 @@ impl Args { map.insert("store.path", path.to_str().unwrap_or("").to_string()); } #[cfg(feature = "uds-gateway")] - if let Some(path) = self.uds_path.clone() { - map.insert("uds_path", path.to_str().unwrap_or("").to_string()); + if let Some(path) = self.gateway_uds_path.clone() { + map.insert("gateway_uds_path", path.to_str().unwrap_or("").to_string()); } map } diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index aae004bf5d..7fdfaac591 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -10,6 +10,8 @@ use iroh_rpc_types::Addr; use iroh_util::insert_into_config_map; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +#[cfg(feature = "uds-gateway")] +use tempdir::TempDir; /// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory pub const CONFIG_FILE_NAME: &str = "one.config.toml"; @@ -23,11 +25,11 @@ pub const DEFAULT_PORT: u16 = 9050; /// as well as the common rpc & metrics ones. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct Config { - // Gateway specific configuration. + /// Gateway specific configuration. pub gateway: iroh_gateway::config::Config, - // Store specific configuration. + /// Store specific configuration. pub store: iroh_store::config::Config, - // P2P specific configuration. + /// P2P specific configuration. pub p2p: iroh_p2p::config::Config, /// rpc addresses for the gateway & addresses for the rpc client to dial @@ -36,7 +38,7 @@ pub struct Config { pub metrics: MetricsConfig, /// Path for the UDS socket for the gateway. #[cfg(feature = "uds-gateway")] - pub uds_path: Option, + pub gateway_uds_path: Option, } impl Config { @@ -45,7 +47,7 @@ impl Config { store: iroh_store::config::Config, p2p: iroh_p2p::config::Config, rpc_client: RpcClientConfig, - #[cfg(feature = "uds-gateway")] uds_path: Option, + #[cfg(feature = "uds-gateway")] gateway_uds_path: Option, ) -> Self { Self { gateway, @@ -54,18 +56,17 @@ impl Config { rpc_client, metrics: MetricsConfig::default(), #[cfg(feature = "uds-gateway")] - uds_path, + gateway_uds_path, } } - // When running in ipfsd mode, the resolver will use memory channels to - // communicate with the p2p and store modules. - // The gateway itself is exposing a UDS rpc endpoint to be also usable - // as a single entry point for other system services if feature enabled. + /// When running in ipfsd mode, the resolver will use memory channels to + /// communicate with the p2p and store modules. + /// The gateway itself is exposing a UDS rpc endpoint to be also usable + /// as a single entry point for other system services if feature enabled. pub fn default_ipfsd() -> RpcClientConfig { #[cfg(feature = "uds-gateway")] - let path: PathBuf = - format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()).into(); + let path: PathBuf = TempDir::new("iroh").unwrap().path().join("ipfsd.http"); RpcClientConfig { #[cfg(feature = "uds-gateway")] @@ -88,8 +89,7 @@ impl Config { impl Default for Config { fn default() -> Self { #[cfg(feature = "uds-gateway")] - let uds_path: PathBuf = - format!("{}", std::env::temp_dir().join("ipfsd.gateway").display()).into(); + let gateway_uds_path: PathBuf = TempDir::new("iroh").unwrap().path().join("ipfsd.http"); let ipfsd = Self::default_ipfsd(); let metrics_config = MetricsConfig::default(); Self { @@ -99,7 +99,7 @@ impl Default for Config { store: default_store_config(ipfsd.clone(), metrics_config.clone()), p2p: default_p2p_config(ipfsd, metrics_config), #[cfg(feature = "uds-gateway")] - uds_path: Some(uds_path), + gateway_uds_path: Some(gateway_uds_path), } } } @@ -140,23 +140,27 @@ impl Source for Config { insert_into_config_map(&mut map, "rpc_client", self.rpc_client.collect()?); insert_into_config_map(&mut map, "metrics", self.metrics.collect()?); #[cfg(feature = "uds-gateway")] - if let Some(uds_path) = self.uds_path.as_ref() { - insert_into_config_map(&mut map, "uds_path", uds_path.to_str().unwrap().to_string()); + if let Some(uds_path) = self.gateway_uds_path.as_ref() { + insert_into_config_map( + &mut map, + "gateway_uds_path", + uds_path.to_str().unwrap().to_string(), + ); } Ok(map) } } impl iroh_gateway::handlers::StateConfig for Config { - fn rpc_client(&self) -> iroh_rpc_client::Config { - self.rpc_client.clone() + fn rpc_client(&self) -> &iroh_rpc_client::Config { + &self.rpc_client } fn port(&self) -> u16 { self.gateway.port } - fn user_headers(&self) -> HeaderMap { - self.gateway.headers.clone() + fn user_headers(&self) -> &HeaderMap { + &self.gateway.headers } } diff --git a/iroh-one/src/lib.rs b/iroh-one/src/lib.rs index d6c51fe6b4..5cdc083413 100644 --- a/iroh-one/src/lib.rs +++ b/iroh-one/src/lib.rs @@ -2,6 +2,5 @@ pub mod cli; pub mod config; pub mod mem_p2p; pub mod mem_store; -mod rpc; #[cfg(feature = "uds-gateway")] pub mod uds; diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index 874973b18a..a1b42ce0b3 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -14,6 +14,8 @@ use iroh_one::{ use iroh_rpc_types::Addr; use iroh_util::{iroh_home_path, make_config}; use prometheus_client::registry::Registry; +#[cfg(feature = "uds-gateway")] +use tempdir::TempDir; use tokio::sync::RwLock; #[tokio::main(flavor = "multi_thread")] @@ -93,8 +95,8 @@ async fn main() -> Result<()> { #[cfg(feature = "uds-gateway")] let uds_server_task = { - let mut path = std::env::temp_dir().join("ipfsd.http"); - if let Some(uds_path) = config.uds_path { + let mut path = TempDir::new("iroh")?.path().join("ipfsd.http"); + if let Some(uds_path) = config.gateway_uds_path { path = uds_path; } let uds_server = uds::uds_server(shared_state, path); diff --git a/iroh-one/src/rpc.rs b/iroh-one/src/rpc.rs deleted file mode 100644 index a19b7d9830..0000000000 --- a/iroh-one/src/rpc.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Result; -use async_trait::async_trait; -use iroh_rpc_types::gateway::{Gateway as RpcGateway, GatewayServerAddr, VersionResponse}; - -#[derive(Default)] -pub struct Gateway {} - -#[async_trait] -impl RpcGateway for Gateway { - #[tracing::instrument(skip(self))] - async fn version(&self, _: ()) -> Result { - let version = env!("CARGO_PKG_VERSION").to_string(); - Ok(VersionResponse { version }) - } -} - -#[cfg(feature = "grpc")] -impl iroh_rpc_types::NamedService for Gateway { - const NAME: &'static str = "gateway"; -} - -#[allow(dead_code)] -pub async fn new(addr: GatewayServerAddr, gateway: Gateway) -> Result<()> { - iroh_rpc_types::gateway::serve(addr, gateway).await -} From 8e2851def8857498fb67869eda7490e143a943eb Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Wed, 31 Aug 2022 12:47:57 +0200 Subject: [PATCH 28/29] fix configs --- iroh-gateway/src/config.rs | 12 ++++++------ iroh-one/src/config.rs | 14 ++++++++------ iroh-one/src/main.rs | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index b697ecca21..d8ce5efe24 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -29,17 +29,17 @@ pub struct Config { pub cache: bool, /// default port to listen on pub port: u16, + /// flag to toggle whether the gateway should use denylist on requests + pub denylist: bool, + /// rpc addresses for the gateway & addresses for the rpc client to dial + pub rpc_client: RpcClientConfig, + /// metrics configuration + pub metrics: MetricsConfig, // NOTE: for toml to serialize properly, the "table" values must be serialized at the end, and // so much come at the end of the `Config` struct /// set of user provided headers to attach to all responses #[serde(with = "http_serde::header_map")] pub headers: HeaderMap, - /// rpc addresses for the gateway & addresses for the rpc client to dial - pub rpc_client: RpcClientConfig, - /// metrics configuration - pub metrics: MetricsConfig, - /// flag to toggle whether the gateway should use denylist on requests - pub denylist: bool, } impl Config { diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index 7fdfaac591..72ade1f6e0 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -25,20 +25,19 @@ pub const DEFAULT_PORT: u16 = 9050; /// as well as the common rpc & metrics ones. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct Config { + /// Path for the UDS socket for the gateway. + #[cfg(feature = "uds-gateway")] + pub gateway_uds_path: Option, /// Gateway specific configuration. pub gateway: iroh_gateway::config::Config, /// Store specific configuration. pub store: iroh_store::config::Config, /// P2P specific configuration. pub p2p: iroh_p2p::config::Config, - /// rpc addresses for the gateway & addresses for the rpc client to dial pub rpc_client: RpcClientConfig, /// metrics configuration pub metrics: MetricsConfig, - /// Path for the UDS socket for the gateway. - #[cfg(feature = "uds-gateway")] - pub gateway_uds_path: Option, } impl Config { @@ -78,11 +77,14 @@ impl Config { } } - // synchronize the rpc config across subsystems - pub fn sync_rpc_client(&mut self) { + // synchronize the top level configs across subsystems + pub fn synchronize_subconfigs(&mut self) { self.gateway.rpc_client = self.rpc_client.clone(); self.p2p.rpc_client = self.rpc_client.clone(); self.store.rpc_client = self.rpc_client.clone(); + self.gateway.metrics = self.metrics.clone(); + self.p2p.metrics = self.metrics.clone(); + self.store.metrics = self.metrics.clone(); } } diff --git a/iroh-one/src/main.rs b/iroh-one/src/main.rs index a1b42ce0b3..8b7134a24c 100644 --- a/iroh-one/src/main.rs +++ b/iroh-one/src/main.rs @@ -53,7 +53,7 @@ async fn main() -> Result<()> { config.rpc_client.gateway_addr = Some(gw_sender); } - config.sync_rpc_client(); + config.synchronize_subconfigs(); config.metrics = metrics::metrics_config_with_compile_time_info(config.metrics); println!("{:#?}", config); From 7085e17159749e0fb8bd043f320679c3bcde9eea Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Wed, 31 Aug 2022 13:30:38 +0200 Subject: [PATCH 29/29] bump deps --- iroh-one/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh-one/Cargo.toml b/iroh-one/Cargo.toml index 6a5498eb67..e9c60dc2d1 100644 --- a/iroh-one/Cargo.toml +++ b/iroh-one/Cargo.toml @@ -25,7 +25,7 @@ iroh-rpc-client = {path = "../iroh-rpc-client", default-features = false} iroh-rpc-types = {path = "../iroh-rpc-types", default-features = false} iroh-store = {path = "../iroh-store", default-features = false, features = ["rpc-mem"]} iroh-util = {path = "../iroh-util"} -prometheus-client = "0.17.0" +prometheus-client = "0.18.0" serde = {version = "1.0", features = ["derive"]} tokio = {version = "1", features = ["macros", "rt-multi-thread", "process"]} tracing = "0.1.33"