Use user-space DNS resolver (#608)

* use minimal features * use hickory for http * remove public error constructor * get rid of future wrapper * share http client throughout kitsune (where possible and useful) to maximize pooling effectiveness * fix feature flags * up
kitsune-soc · Nov 13, 2024 · fe44a5d · fe44a5d
1 parent 4ddeef8
commit fe44a5d
Show file tree

Hide file tree

Showing 56 changed files with 356 additions and 277 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -109,6 +109,11 @@ garde = { version = "0.20.0", features = [
     "regex",
     "serde",
 ] }
+hickory-resolver = { version = "0.25.0-alpha.3", default-features = false, features = [
+    "dns-over-rustls",
+    "native-certs",
+    "tokio-runtime",
+] }
 iso8601-timestamp = "0.3.0"
 itertools = { version = "0.13.0", default-features = false }
 moka = { version = "=0.12.7", features = ["sync"] }
@@ -199,3 +204,6 @@ license = "AGPL-3.0-or-later"
 [patch.crates-io]
 # SIMD runtime detection and generic I/O wrapper
 sonic-rs = { git = "https://github.com/aumetra/sonic-rs.git", rev = "12df930a57ff9f07eb16111a3da1feff3dc8e5ad" }
+
+# Compatibility with latest opentelemetry crates
+tracing-opentelemetry = { git = "https://github.com/tokio-rs/tracing-opentelemetry.git", rev = "d18214596c97b1243ee955a63bed3875a4f8f5f9" }
diff --git a/crates/kitsune-activitypub/Cargo.toml b/crates/kitsune-activitypub/Cargo.toml
@@ -31,7 +31,7 @@ kitsune-util = { workspace = true }
 kitsune-wasm-mrf = { workspace = true }
 mime = "0.3.17"
 mime_guess = { version = "2.0.5", default-features = false }
-serde = "1.0.214"
+serde = "1.0.215"
 sha2 = "0.10.8"
 sonic-rs = { workspace = true }
 speedy-uuid = { workspace = true }

diff --git a/crates/kitsune-activitypub/src/deliverer/core.rs b/crates/kitsune-activitypub/src/deliverer/core.rs
@@ -1,6 +1,5 @@
 use futures_util::{stream::FuturesUnordered, Stream, StreamExt};
 use http::{Method, Request};
-use kitsune_core::consts::USER_AGENT;
 use kitsune_db::model::{account::Account, user::User};
 use kitsune_error::{Error, Result};
 use kitsune_federation_filter::FederationFilter;
@@ -17,8 +16,7 @@ use url::Url;
 /// Does not need to be Arc wrapped for cheap cloning. It's inherently cheap to clone.
 #[derive(Clone, TypedBuilder)]
 pub struct Deliverer {
-    #[builder(default = Client::builder().user_agent(USER_AGENT).unwrap().build())]
-    client: Client,
+    http_client: Client,
     federation_filter: FederationFilter,
     mrf_service: MrfService,
 }
@@ -55,7 +53,7 @@ impl Deliverer {
             .body(body.into())?;
 
         let response = self
-            .client
+            .http_client
             .execute_signed(request, &account.public_key_id, &user.private_key)
             .await?;
 

diff --git a/crates/kitsune-activitypub/src/fetcher/mod.rs b/crates/kitsune-activitypub/src/fetcher/mod.rs
@@ -1,15 +1,12 @@
 use async_trait::async_trait;
 use headers::{ContentType, HeaderMapExt};
-use http::HeaderValue;
+use http::{header::ACCEPT, HeaderValue, Request};
 use kitsune_cache::ArcCache;
 use kitsune_config::language_detection::Configuration as LanguageDetectionConfig;
-use kitsune_core::{
-    consts::USER_AGENT,
-    traits::{
-        coerce::CoerceResolver,
-        fetcher::{AccountFetchOptions, PostFetchOptions},
-        Fetcher as FetcherTrait, Resolver,
-    },
+use kitsune_core::traits::{
+    coerce::CoerceResolver,
+    fetcher::{AccountFetchOptions, PostFetchOptions},
+    Fetcher as FetcherTrait, Resolver,
 };
 use kitsune_db::{
     model::{account::Account, custom_emoji::CustomEmoji, post::Post},
@@ -32,23 +29,14 @@ mod actor;
 mod emoji;
 mod object;
 
+static ACCEPT_VALUE: HeaderValue = HeaderValue::from_static(
+    "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\", application/activity+json",
+);
+
 #[derive(TypedBuilder)]
 #[builder(build_method(into = Arc<Fetcher>))]
 pub struct Fetcher {
-    #[builder(default =
-        Client::builder()
-            .default_header(
-                "accept",
-                HeaderValue::from_static(
-                    "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\", application/activity+json",
-                ),
-            )
-            .unwrap()
-            .user_agent(USER_AGENT)
-            .unwrap()
-            .build()
-    )]
-    client: Client,
+    http_client: Client,
     db_pool: PgPool,
     embed_client: Option<EmbedClient>,
     federation_filter: FederationFilter,
@@ -74,8 +62,13 @@ impl Fetcher {
             bail!("instance is blocked");
         }
 
-        let response = self.client.get(url.as_str()).await?;
+        let request = Request::builder()
+            .method(http::Method::GET)
+            .uri(url.as_str())
+            .header(ACCEPT, &ACCEPT_VALUE)
+            .body(kitsune_http_client::Body::empty())?;
 
+        let response = self.http_client.execute(request).await?;
         if !response.status().is_success() {
             return Ok(None);
         }

diff --git a/crates/kitsune-activitypub/tests/fetcher/basic.rs b/crates/kitsune-activitypub/tests/fetcher/basic.rs
@@ -25,7 +25,7 @@ async fn fetch_actor() {
         let client = Client::builder().service(service_fn(handle));
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -36,7 +36,7 @@ async fn fetch_actor() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();
@@ -64,7 +64,7 @@ async fn fetch_emoji() {
         let client = Client::builder().service(service_fn(handle));
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool.clone())
             .embed_client(None)
             .federation_filter(
@@ -75,7 +75,7 @@ async fn fetch_emoji() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();
@@ -115,7 +115,7 @@ async fn fetch_note() {
         let client = Client::builder().service(service_fn(handle));
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool.clone())
             .embed_client(None)
             .federation_filter(
@@ -126,7 +126,7 @@ async fn fetch_note() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/filter.rs b/crates/kitsune-activitypub/tests/fetcher/filter.rs
@@ -45,9 +45,9 @@ async fn federation_allow() {
         let client = Client::builder().service(client);
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         assert_blocked!(fetcher
@@ -63,9 +63,9 @@ async fn federation_allow() {
         let client = Client::builder().service(service_fn(handle));
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         assert!(matches!(
@@ -90,7 +90,7 @@ async fn federation_deny() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -101,7 +101,7 @@ async fn federation_deny() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/infinite.rs b/crates/kitsune-activitypub/tests/fetcher/infinite.rs
@@ -91,7 +91,7 @@ async fn fetch_infinitely_long_reply_chain() {
             let client = Client::builder().service(client);
 
             let fetcher = Fetcher::builder()
-                .client(client.clone())
+                .http_client(client.clone())
                 .db_pool(db_pool)
                 .embed_client(None)
                 .federation_filter(
@@ -102,7 +102,7 @@ async fn fetch_infinitely_long_reply_chain() {
                 )
                 .language_detection_config(language_detection_config())
                 .search_backend(NoopSearchService)
-                .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+                .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
                 .account_cache(Arc::new(NoopCache.into()))
                 .post_cache(Arc::new(NoopCache.into()))
                 .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/origin.rs b/crates/kitsune-activitypub/tests/fetcher/origin.rs
@@ -37,9 +37,9 @@ async fn check_ap_id_authority() {
         let client = Client::builder().service(client);
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         // The mock HTTP client ensures that the fetcher doesn't access the correct server
@@ -60,9 +60,9 @@ async fn check_ap_id_authority() {
         let client = Client::builder().service(client);
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         let _ = fetcher
@@ -84,7 +84,7 @@ async fn check_ap_content_type() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -95,7 +95,7 @@ async fn check_ap_content_type() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/webfinger.rs b/crates/kitsune-activitypub/tests/fetcher/webfinger.rs
@@ -47,7 +47,7 @@ async fn fetch_actor_with_custom_acct() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -58,7 +58,7 @@ async fn fetch_actor_with_custom_acct() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();
@@ -125,7 +125,7 @@ async fn ignore_fake_webfinger_acct() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -136,7 +136,7 @@ async fn ignore_fake_webfinger_acct() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-cache/Cargo.toml b/crates/kitsune-cache/Cargo.toml
@@ -10,7 +10,7 @@ enum_dispatch = "0.3.13"
 fred = { workspace = true }
 kitsune-error = { workspace = true }
 moka = { workspace = true }
-serde = "1.0.214"
+serde = "1.0.215"
 sonic-rs = { workspace = true }
 tracing = "0.1.40"
 triomphe = { workspace = true }

diff --git a/crates/kitsune-captcha/Cargo.toml b/crates/kitsune-captcha/Cargo.toml
@@ -10,7 +10,7 @@ enum_dispatch = "0.3.13"
 http = "1.1.0"
 kitsune-error = { workspace = true }
 kitsune-http-client = { workspace = true }
-serde = { version = "1.0.214", features = ["derive"] }
+serde = { version = "1.0.215", features = ["derive"] }
 serde_urlencoded = "0.7.1"
 sonic-rs = { workspace = true }
 strum = { version = "0.26.3", features = ["derive"] }

diff --git a/crates/kitsune-captcha/src/hcaptcha.rs b/crates/kitsune-captcha/src/hcaptcha.rs
@@ -10,8 +10,7 @@ pub struct Captcha {
     pub site_key: String,
     pub secret_key: String,
 
-    #[builder(default)]
-    client: Client,
+    http_client: Client,
 }
 
 #[derive(Serialize, Deserialize, TypedBuilder)]
@@ -46,7 +45,7 @@ impl CaptchaBackend for Captcha {
             .header("Accept", "application/json")
             .body(body.into())?;
 
-        let response = self.client.execute(request).await?;
+        let response = self.http_client.execute(request).await?;
 
         let verification_result = response.json::<HCaptchaResponse>().await?;
         if !verification_result.success {

diff --git a/crates/kitsune-captcha/src/mcaptcha.rs b/crates/kitsune-captcha/src/mcaptcha.rs
@@ -11,8 +11,7 @@ pub struct Captcha {
     pub secret_key: String,
     pub site_key: String,
 
-    #[builder(default)]
-    client: Client,
+    http_client: Client,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -41,7 +40,7 @@ impl CaptchaBackend for Captcha {
             .header("Accept", "application/json")
             .body(body.into())?;
 
-        let response = self.client.execute(request).await?;
+        let response = self.http_client.execute(request).await?;
 
         let verification_result = response.json::<MCaptchaResponse>().await?;
         if !verification_result.valid {

diff --git a/crates/kitsune-config/Cargo.toml b/crates/kitsune-config/Cargo.toml
@@ -9,7 +9,7 @@ license.workspace = true
 eyre = "0.6.12"
 human-size = { version = "0.4.3", features = ["serde"] }
 isolang = { version = "2.4.0", features = ["serde"] }
-serde = { version = "1.0.214", features = ["derive"] }
+serde = { version = "1.0.215", features = ["derive"] }
 smol_str = { version = "0.3.2", features = ["serde"] }
 tokio = { workspace = true, features = ["fs"] }
 toml = { version = "0.8.19", default-features = false, features = ["parse"] }

diff --git a/crates/kitsune-core/Cargo.toml b/crates/kitsune-core/Cargo.toml
@@ -12,7 +12,7 @@ git-version = "0.3.9"
 kitsune-db = { workspace = true }
 kitsune-error = { workspace = true }
 paste = "1.0.15"
-serde = { version = "1.0.214", features = ["derive"] }
+serde = { version = "1.0.215", features = ["derive"] }
 triomphe = { workspace = true }
 typed-builder = "0.20.0"
 unsize = "1.1.0"

diff --git a/crates/kitsune-db/Cargo.toml b/crates/kitsune-db/Cargo.toml
@@ -29,7 +29,7 @@ rustls = { version = "0.23.16", default-features = false, features = [
     "tls12",
 ] }
 rustls-native-certs = "0.8.0"
-serde = { version = "1.0.214", features = ["derive"] }
+serde = { version = "1.0.215", features = ["derive"] }
 sonic-rs = { workspace = true }
 speedy-uuid = { workspace = true, features = ["diesel"] }
 tokio = { workspace = true, features = ["rt"] }