From 3757ca9748df44e1d897511a84cb5ab4958a8ad2 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:24:54 +0800 Subject: [PATCH 1/9] add config for azdls Signed-off-by: Xuanwo --- core/src/services/azblob/backend.rs | 3 +- core/src/services/azdls/backend.rs | 73 +++++++++++++++++------------ 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index 65a25bcf1950..7d6e71e98a74 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -52,6 +52,7 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[ ]; const AZBLOB_BATCH_LIMIT: usize = 256; + /// Azure Storage Blob services support. #[derive(Default, Deserialize, Clone)] pub struct AzblobConfig { @@ -124,7 +125,7 @@ pub struct AzblobBuilder { impl Debug for AzblobBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("AzblobBuilder"); ds.field("config", &self.config); diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index e7243ab45aab..0b345777a5ff 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use async_trait::async_trait; use http::StatusCode; use log::debug; +use madsim::net::rpc::Deserialize; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; @@ -30,9 +31,8 @@ use reqsign::AzureStorageSigner; use super::core::AzdlsCore; use super::error::parse_error; use super::lister::AzdlsLister; -use super::writer::AzdlsWriter; +use super::writer::{AzdlsWriter, AzdlsWriters}; use crate::raw::*; -use crate::services::azdls::writer::AzdlsWriters; use crate::*; /// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax. @@ -46,20 +46,18 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[ ]; /// Azure Data Lake Storage Gen2 Support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct AzdlsBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct AzdlsConfig { root: Option, filesystem: String, endpoint: Option, account_name: Option, account_key: Option, - http_client: Option, } -impl Debug for AzdlsBuilder { +impl Debug for AzdlsConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("AzdlsConfig"); ds.field("root", &self.root); ds.field("filesystem", &self.filesystem); @@ -76,13 +74,31 @@ impl Debug for AzdlsBuilder { } } +/// Azure Data Lake Storage Gen2 Support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct AzdlsBuilder { + config: AzdlsConfig, + http_client: Option, +} + +impl Debug for AzdlsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzdlsBuilder"); + + ds.field("config", &self.config); + + ds.finish() + } +} + impl AzdlsBuilder { /// Set root of this backend. /// /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -90,7 +106,7 @@ impl AzdlsBuilder { /// Set filesystem name of this backend. pub fn filesystem(&mut self, filesystem: &str) -> &mut Self { - self.filesystem = filesystem.to_string(); + self.config.filesystem = filesystem.to_string(); self } @@ -104,7 +120,7 @@ impl AzdlsBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -116,7 +132,7 @@ impl AzdlsBuilder { /// - If not, we will try to load it from environment. pub fn account_name(&mut self, account_name: &str) -> &mut Self { if !account_name.is_empty() { - self.account_name = Some(account_name.to_string()); + self.config.account_name = Some(account_name.to_string()); } self @@ -128,7 +144,7 @@ impl AzdlsBuilder { /// - If not, we will try to load it from environment. pub fn account_key(&mut self, account_key: &str) -> &mut Self { if !account_key.is_empty() { - self.account_key = Some(account_key.to_string()); + self.config.account_key = Some(account_key.to_string()); } self @@ -153,19 +169,19 @@ impl Builder for AzdlsBuilder { fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); // Handle endpoint, region and container name. - let filesystem = match self.filesystem.is_empty() { - false => Ok(&self.filesystem), + let filesystem = match self.config.filesystem.is_empty() { + false => Ok(&self.config.filesystem), true => Err(Error::new(ErrorKind::ConfigInvalid, "filesystem is empty") .with_operation("Builder::build") .with_context("service", Scheme::Azdls)), }?; debug!("backend use filesystem {}", &filesystem); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -184,10 +200,11 @@ impl Builder for AzdlsBuilder { let config_loader = AzureStorageConfig { account_name: self + .config .account_name .clone() .or_else(|| infer_storage_name_from_endpoint(endpoint.as_str())), - account_key: self.account_key.clone(), + account_key: self.config.account_key.clone(), sas_token: None, ..Default::default() }; @@ -198,7 +215,7 @@ impl Builder for AzdlsBuilder { debug!("backend build finished: {:?}", &self); Ok(AzdlsBackend { core: Arc::new(AzdlsCore { - filesystem: self.filesystem.clone(), + filesystem: self.config.filesystem.clone(), root, endpoint, client, @@ -209,15 +226,13 @@ impl Builder for AzdlsBuilder { } fn from_map(map: HashMap) -> Self { - let mut builder = AzdlsBuilder::default(); + let config = AzdlsConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - map.get("root").map(|v| builder.root(v)); - map.get("filesystem").map(|v| builder.filesystem(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("account_name").map(|v| builder.account_name(v)); - map.get("account_key").map(|v| builder.account_key(v)); - - builder + AzdlsBuilder { + config, + http_client: None, + } } } @@ -467,7 +482,7 @@ mod tests { assert_eq!(azdls.core.filesystem, "filesystem".to_string()); assert_eq!( - azdls_builder.account_key.unwrap(), + azdls_builder.config.account_key.unwrap(), "account-key".to_string() ); } @@ -488,6 +503,6 @@ mod tests { assert_eq!(azdls.core.filesystem, "filesystem".to_string()); - assert_eq!(azdls_builder.account_key, None); + assert_eq!(azdls_builder.config.account_key, None); } } From 636b2bc44510d3ea9b9ea3f526a6404896291733 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:29:08 +0800 Subject: [PATCH 2/9] Add config for azfile Signed-off-by: Xuanwo --- core/src/services/azfile/backend.rs | 74 ++++++++++++++++++----------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index 84f02c88bf2e..58eeaa683610 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use async_trait::async_trait; use http::StatusCode; use log::debug; +use madsim::net::rpc::Deserialize; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; @@ -39,31 +40,51 @@ use crate::*; const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net"; /// Azure File services support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct AzfileBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct AzfileConfig { root: Option, endpoint: Option, - account_name: Option, share_name: String, + account_name: Option, account_key: Option, sas_token: Option, - http_client: Option, } -impl Debug for AzfileBuilder { +impl Debug for AzfileConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("AzfileConfig"); ds.field("root", &self.root); - ds.field("endpoint", &self.endpoint); ds.field("share_name", &self.share_name); + ds.field("endpoint", &self.endpoint); + if self.account_name.is_some() { ds.field("account_name", &""); } if self.account_key.is_some() { ds.field("account_key", &""); } + if self.sas_token.is_some() { + ds.field("sas_token", &""); + } + + ds.finish() + } +} + +/// Azure File services support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct AzfileBuilder { + config: AzfileConfig, + http_client: Option, +} + +impl Debug for AzfileBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzfileBuilder"); + + ds.field("config", &self.config); ds.finish() } @@ -75,7 +96,7 @@ impl AzfileBuilder { /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -85,7 +106,7 @@ impl AzfileBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -97,7 +118,7 @@ impl AzfileBuilder { /// - If not, we will try to load it from environment. pub fn account_name(&mut self, account_name: &str) -> &mut Self { if !account_name.is_empty() { - self.account_name = Some(account_name.to_string()); + self.config.account_name = Some(account_name.to_string()); } self @@ -109,7 +130,7 @@ impl AzfileBuilder { /// - If not, we will try to load it from environment. pub fn account_key(&mut self, account_key: &str) -> &mut Self { if !account_key.is_empty() { - self.account_key = Some(account_key.to_string()); + self.config.account_key = Some(account_key.to_string()); } self @@ -121,7 +142,7 @@ impl AzfileBuilder { /// You can find more about from: pub fn share_name(&mut self, share_name: &str) -> &mut Self { if !share_name.is_empty() { - self.share_name = share_name.to_string(); + self.config.share_name = share_name.to_string(); } self @@ -144,24 +165,22 @@ impl Builder for AzfileBuilder { type Accessor = AzfileBackend; fn from_map(map: HashMap) -> Self { - let mut builder = AzfileBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("account_name").map(|v| builder.account_name(v)); - map.get("account_key").map(|v| builder.account_key(v)); - map.get("share_name").map(|v| builder.share_name(v)); + let config = AzfileConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + AzfileBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -179,6 +198,7 @@ impl Builder for AzfileBuilder { }; let account_name_option = self + .config .account_name .clone() .or_else(|| infer_account_name_from_endpoint(endpoint.as_str())); @@ -194,8 +214,8 @@ impl Builder for AzfileBuilder { let config_loader = AzureStorageConfig { account_name: Some(account_name), - account_key: self.account_key.clone(), - sas_token: self.sas_token.clone(), + account_key: self.config.account_key.clone(), + sas_token: self.config.sas_token.clone(), ..Default::default() }; @@ -211,7 +231,7 @@ impl Builder for AzfileBuilder { loader: cred_loader, client, signer, - share_name: self.share_name.clone(), + share_name: self.config.share_name.clone(), }), }) } @@ -435,7 +455,7 @@ mod tests { ); assert_eq!( - azfile_builder.account_key.unwrap(), + azfile_builder.config.account_key.unwrap(), "account-key".to_string() ); } From 958b076870497ad758b54c95b7eabfbc166cd369 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:32:04 +0800 Subject: [PATCH 3/9] Add cacache Signed-off-by: Xuanwo --- core/src/services/cacache/backend.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index 4fc47f15e948..9c13ce6db53c 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -22,26 +22,34 @@ use std::str; use async_trait::async_trait; use cacache; +use madsim::net::rpc::Deserialize; use crate::raw::adapters::kv; +use crate::raw::ConfigDeserializer; use crate::Builder; use crate::Error; use crate::ErrorKind; use crate::Scheme; use crate::*; +/// cacache service support. +#[derive(Default, Deserialize, Clone)] +pub struct CacacheConfig { + /// That path to the cacache data directory. + datadir: Option, +} + /// cacache service support. #[doc = include_str!("docs.md")] #[derive(Default)] pub struct CacacheBuilder { - /// That path to the cacache data directory. - datadir: Option, + config: CacacheConfig, } impl CacacheBuilder { /// Set the path to the cacache data directory. Will create if not exists. pub fn datadir(&mut self, path: &str) -> &mut Self { - self.datadir = Some(path.into()); + self.config.datadir = Some(path.into()); self } } @@ -51,15 +59,14 @@ impl Builder for CacacheBuilder { type Accessor = CacacheBackend; fn from_map(map: HashMap) -> Self { - let mut builder = CacacheBuilder::default(); - - map.get("datadir").map(|v| builder.datadir(v)); + let config = CacacheConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + Self { config } } fn build(&mut self) -> Result { - let datadir_path = self.datadir.take().ok_or_else(|| { + let datadir_path = self.config.datadir.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Cacache) })?; From 33b17b6d09aed4404aced533ba3b8b3fa55e8bb2 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:36:28 +0800 Subject: [PATCH 4/9] Impl config for cloudflare kv Signed-off-by: Xuanwo --- core/src/services/cloudflare_kv/backend.rs | 69 ++++++++++++++-------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs index e05bb3931e22..3050582c842d 100644 --- a/core/src/services/cloudflare_kv/backend.rs +++ b/core/src/services/cloudflare_kv/backend.rs @@ -31,9 +31,9 @@ use crate::raw::*; use crate::ErrorKind; use crate::*; -#[doc = include_str!("docs.md")] -#[derive(Default)] -pub struct CloudflareKvBuilder { +/// Cloudflare Kv Service Support. +#[derive(Default, Deserialize, Clone)] +pub struct CloudflareKvConfig { /// The token used to authenticate with CloudFlare. token: Option, /// The account ID used to authenticate with CloudFlare. Used as URI path parameter. @@ -41,18 +41,39 @@ pub struct CloudflareKvBuilder { /// The namespace ID. Used as URI path parameter. namespace_id: Option, - /// The HTTP client used to communicate with CloudFlare. - http_client: Option, /// Root within this backend. root: Option, } +impl Debug for CloudflareKvConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("CloudflareKvConfig"); + + ds.field("root", &self.root); + ds.field("account_id", &self.account_id); + ds.field("namespace_id", &self.namespace_id); + + if self.token.is_some() { + ds.field("token", &""); + } + + ds.finish() + } +} + +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct CloudflareKvBuilder { + config: CloudflareKvConfig, + + /// The HTTP client used to communicate with CloudFlare. + http_client: Option, +} + impl Debug for CloudflareKvBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("CloudFlareKvBuilder") - .field("account_id", &self.account_id) - .field("namespace_id", &self.namespace_id) - .field("root", &self.root) + .field("config", &self.config) .finish() } } @@ -61,7 +82,7 @@ impl CloudflareKvBuilder { /// Set the token used to authenticate with CloudFlare. pub fn token(&mut self, token: &str) -> &mut Self { if !token.is_empty() { - self.token = Some(token.to_string()) + self.config.token = Some(token.to_string()) } self } @@ -69,7 +90,7 @@ impl CloudflareKvBuilder { /// Set the account ID used to authenticate with CloudFlare. pub fn account_id(&mut self, account_id: &str) -> &mut Self { if !account_id.is_empty() { - self.account_id = Some(account_id.to_string()) + self.config.account_id = Some(account_id.to_string()) } self } @@ -77,7 +98,7 @@ impl CloudflareKvBuilder { /// Set the namespace ID. pub fn namespace_id(&mut self, namespace_id: &str) -> &mut Self { if !namespace_id.is_empty() { - self.namespace_id = Some(namespace_id.to_string()) + self.config.namespace_id = Some(namespace_id.to_string()) } self } @@ -85,7 +106,7 @@ impl CloudflareKvBuilder { /// Set the root within this backend. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self } @@ -97,28 +118,29 @@ impl Builder for CloudflareKvBuilder { type Accessor = CloudflareKvBackend; fn from_map(map: HashMap) -> Self { - let mut builder = Self::default(); - map.get("token").map(|v| builder.token(v)); - map.get("account_id").map(|v| builder.account_id(v)); - map.get("namespace_id").map(|v| builder.namespace_id(v)); - map.get("root").map(|v| builder.root(v)); - builder + let config = CloudflareKvConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); + + Self { + config, + http_client: None, + } } fn build(&mut self) -> Result { - let authorization = match &self.token { + let authorization = match &self.config.token { Some(token) => format_authorization_by_bearer(token)?, None => return Err(Error::new(ErrorKind::ConfigInvalid, "token is required")), }; - let Some(account_id) = self.account_id.clone() else { + let Some(account_id) = self.config.account_id.clone() else { return Err(Error::new( ErrorKind::ConfigInvalid, "account_id is required", )); }; - let Some(namespace_id) = self.namespace_id.clone() else { + let Some(namespace_id) = self.config.namespace_id.clone() else { return Err(Error::new( ErrorKind::ConfigInvalid, "namespace_id is required", @@ -135,7 +157,8 @@ impl Builder for CloudflareKvBuilder { }; let root = normalize_root( - self.root + self.config + .root .clone() .unwrap_or_else(|| "/".to_string()) .as_str(), @@ -271,7 +294,7 @@ impl kv::Adapter for Adapter { let body = resp.into_body().bytes().await?; let response: CfKvScanResponse = serde_json::from_slice(&body).map_err(|e| { Error::new( - crate::ErrorKind::Unexpected, + ErrorKind::Unexpected, &format!("failed to parse error response: {}", e), ) })?; From f1676bb40af0e96157a4110a7ed2b45d93c8d148 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:40:17 +0800 Subject: [PATCH 5/9] Migrate cos Signed-off-by: Xuanwo --- core/src/services/cos/backend.rs | 66 +++++++++++++++++++------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index decb37bd1da2..7dcc32ad21d2 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -26,6 +26,7 @@ use log::debug; use reqsign::TencentCosConfig; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; +use serde::Deserialize; use super::core::*; use super::error::parse_error; @@ -36,27 +37,40 @@ use crate::services::cos::writer::CosWriters; use crate::*; /// Tencent-Cloud COS services support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct CosBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct CosConfig { root: Option, endpoint: Option, secret_id: Option, secret_key: Option, bucket: Option, - http_client: Option, - disable_config_load: bool, } -impl Debug for CosBuilder { +impl Debug for CosConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Builder") + f.debug_struct("CosConfig") .field("root", &self.root) .field("endpoint", &self.endpoint) .field("secret_id", &"") .field("secret_key", &"") .field("bucket", &self.bucket) + .finish_non_exhaustive() + } +} + +/// Tencent-Cloud COS services support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct CosBuilder { + config: CosConfig, + http_client: Option, +} + +impl Debug for CosBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CosBuilder") + .field("config", &self.config) .finish() } } @@ -67,7 +81,7 @@ impl CosBuilder { /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -82,7 +96,7 @@ impl CosBuilder { /// - `https://cos.ap-singapore.myqcloud.com` pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -93,7 +107,7 @@ impl CosBuilder { /// - If not, we will try to load it from environment. pub fn secret_id(&mut self, secret_id: &str) -> &mut Self { if !secret_id.is_empty() { - self.secret_id = Some(secret_id.to_string()); + self.config.secret_id = Some(secret_id.to_string()); } self @@ -104,7 +118,7 @@ impl CosBuilder { /// - If not, we will try to load it from environment. pub fn secret_key(&mut self, secret_key: &str) -> &mut Self { if !secret_key.is_empty() { - self.secret_key = Some(secret_key.to_string()); + self.config.secret_key = Some(secret_key.to_string()); } self @@ -114,7 +128,7 @@ impl CosBuilder { /// The param is required. pub fn bucket(&mut self, bucket: &str) -> &mut Self { if !bucket.is_empty() { - self.bucket = Some(bucket.to_string()); + self.config.bucket = Some(bucket.to_string()); } self @@ -127,7 +141,7 @@ impl CosBuilder { /// /// - envs like `TENCENTCLOUD_SECRET_ID` pub fn disable_config_load(&mut self) -> &mut Self { - self.disable_config_load = true; + self.config.disable_config_load = true; self } @@ -148,24 +162,22 @@ impl Builder for CosBuilder { type Accessor = CosBackend; fn from_map(map: HashMap) -> Self { - let mut builder = CosBuilder::default(); + let config = CosConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - map.get("root").map(|v| builder.root(v)); - map.get("bucket").map(|v| builder.bucket(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("secret_id").map(|v| builder.secret_id(v)); - map.get("secret_key").map(|v| builder.secret_key(v)); - - builder + Self { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let bucket = match &self.bucket { + let bucket = match &self.config.bucket { Some(bucket) => Ok(bucket.to_string()), None => Err( Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured") @@ -174,7 +186,7 @@ impl Builder for CosBuilder { }?; debug!("backend use bucket {}", &bucket); - let uri = match &self.endpoint { + let uri = match &self.config.endpoint { Some(endpoint) => endpoint.parse::().map_err(|err| { Error::new(ErrorKind::ConfigInvalid, "endpoint is invalid") .with_context("service", Scheme::Cos) @@ -204,14 +216,14 @@ impl Builder for CosBuilder { }; let mut cfg = TencentCosConfig::default(); - if !self.disable_config_load { + if !self.config.disable_config_load { cfg = cfg.from_env(); } - if let Some(v) = self.secret_id.take() { + if let Some(v) = self.config.secret_id.take() { cfg.secret_id = Some(v); } - if let Some(v) = self.secret_key.take() { + if let Some(v) = self.config.secret_key.take() { cfg.secret_key = Some(v); } From d0c65bd08bce1d55180543249f15574758491e8c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:43:12 +0800 Subject: [PATCH 6/9] Impl config for dashmap Signed-off-by: Xuanwo --- core/src/services/dashmap/backend.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs index f626bcae28ff..ae6a22952bb3 100644 --- a/core/src/services/dashmap/backend.rs +++ b/core/src/services/dashmap/backend.rs @@ -21,21 +21,29 @@ use std::fmt::Formatter; use async_trait::async_trait; use dashmap::DashMap; +use serde::Deserialize; use crate::raw::adapters::typed_kv; +use crate::raw::ConfigDeserializer; use crate::*; +/// [dashmap](https://github.com/xacrimon/dashmap) backend support. +#[derive(Default, Deserialize, Clone, Debug)] +pub struct DashmapConfig { + root: Option, +} + /// [dashmap](https://github.com/xacrimon/dashmap) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] pub struct DashmapBuilder { - root: Option, + config: DashmapConfig, } impl DashmapBuilder { /// Set the root for dashmap. pub fn root(&mut self, path: &str) -> &mut Self { - self.root = Some(path.into()); + self.config.root = Some(path.into()); self } } @@ -45,18 +53,17 @@ impl Builder for DashmapBuilder { type Accessor = DashmapBackend; fn from_map(map: HashMap) -> Self { - let mut builder = Self::default(); - - map.get("root").map(|v| builder.root(v)); + let config = DashmapConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + Self { config } } fn build(&mut self) -> Result { Ok(DashmapBackend::new(Adapter { inner: DashMap::default(), }) - .with_root(self.root.as_deref().unwrap_or_default())) + .with_root(self.config.root.as_deref().unwrap_or_default())) } } From 380f83a33a1cdd363cc8a06d2396e421c13d99ff Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:46:41 +0800 Subject: [PATCH 7/9] Add config for dbfs Signed-off-by: Xuanwo --- core/src/services/cacache/backend.rs | 2 +- core/src/services/dbfs/backend.rs | 46 ++++++++++++++++++---------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index 9c13ce6db53c..153a7d853b9d 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -22,7 +22,7 @@ use std::str; use async_trait::async_trait; use cacache; -use madsim::net::rpc::Deserialize; +use serde::Deserialize; use crate::raw::adapters::kv; use crate::raw::ConfigDeserializer; diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs index 260a5abec364..1028c8b9dea3 100644 --- a/core/src/services/dbfs/backend.rs +++ b/core/src/services/dbfs/backend.rs @@ -34,17 +34,16 @@ use crate::raw::*; use crate::*; /// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct DbfsBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct DbfsConfig { root: Option, endpoint: Option, token: Option, } -impl Debug for DbfsBuilder { +impl Debug for DbfsConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("DbfsConfig"); ds.field("root", &self.root); ds.field("endpoint", &self.endpoint); @@ -57,13 +56,30 @@ impl Debug for DbfsBuilder { } } +/// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct DbfsBuilder { + config: DbfsConfig, +} + +impl Debug for DbfsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("DbfsBuilder"); + + ds.field("config", &self.config); + + ds.finish() + } +} + impl DbfsBuilder { /// Set root of this backend. /// /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -76,7 +92,7 @@ impl DbfsBuilder { /// - Azure: `https://adb-1234567890123456.78.azuredatabricks.net` /// - Aws: `https://dbc-123a5678-90bc.cloud.databricks.com` pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { - self.endpoint = if endpoint.is_empty() { + self.config.endpoint = if endpoint.is_empty() { None } else { Some(endpoint.trim_end_matches('/').to_string()) @@ -87,7 +103,7 @@ impl DbfsBuilder { /// Set the token of this backend. pub fn token(&mut self, token: &str) -> &mut Self { if !token.is_empty() { - self.token = Some(token.to_string()); + self.config.token = Some(token.to_string()); } self } @@ -98,22 +114,20 @@ impl Builder for DbfsBuilder { type Accessor = DbfsBackend; fn from_map(map: HashMap) -> Self { - let mut builder = DbfsBuilder::default(); - - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("token").map(|v| builder.token(v)); + let config = DbfsConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + Self { config } } /// Build a DbfsBackend. fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -121,7 +135,7 @@ impl Builder for DbfsBuilder { }?; debug!("backend use endpoint: {}", &endpoint); - let token = match self.token.take() { + let token = match self.config.token.take() { Some(token) => token, None => { return Err(Error::new( From 5ec0f5f192a1479d349e52e7fa70bdc8199155c8 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:50:17 +0800 Subject: [PATCH 8/9] Fix build Signed-off-by: Xuanwo --- core/src/services/azdls/backend.rs | 2 +- core/src/services/azfile/backend.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index 0b345777a5ff..7e128524cee4 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -23,10 +23,10 @@ use std::sync::Arc; use async_trait::async_trait; use http::StatusCode; use log::debug; -use madsim::net::rpc::Deserialize; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; +use serde::Deserialize; use super::core::AzdlsCore; use super::error::parse_error; diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index 58eeaa683610..d11d7a700b8d 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -23,10 +23,10 @@ use std::sync::Arc; use async_trait::async_trait; use http::StatusCode; use log::debug; -use madsim::net::rpc::Deserialize; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; +use serde::Deserialize; use super::core::AzfileCore; use super::error::parse_error; From 7d7f21e6947be4128455ad77fee7c9443a3d214a Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 27 Feb 2024 07:54:48 +0800 Subject: [PATCH 9/9] Fix cos Signed-off-by: Xuanwo --- core/src/services/cos/backend.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 7dcc32ad21d2..48f0ca444d9d 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -38,6 +38,7 @@ use crate::*; /// Tencent-Cloud COS services support. #[derive(Default, Deserialize, Clone)] +#[serde(default)] pub struct CosConfig { root: Option, endpoint: Option,