Skip to content

Commit

Permalink
feat: rust 2024 edition, moved unity to its own crate
Browse files Browse the repository at this point in the history 
Signed-off-by: Stephen Carman <[email protected]>
  • Loading branch information
@hntd187 @rtyler
hntd187 authored and rtyler committed Dec 16, 2024
1 parent a51d75a commit 3ce734b
Show file tree
Hide file tree
Showing 24 changed files with 523 additions and 442 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ resolver = "2"

[workspace.package]
authors = ["Qingping Hou <[email protected]>"]
rust-version = "1.80"
rust-version = "1.85"
keywords = ["deltalake", "delta", "datalake"]
readme = "README.md"
edition = "2021"
Expand Down
3 changes: 1 addition & 2 deletions crates/aws/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -728,7 +728,6 @@ fn extract_version_from_filename(name: &str) -> Option<i64> {
mod tests {
use super::*;
use aws_sdk_sts::config::ProvideCredentials;

use object_store::memory::InMemory;
use serial_test::serial;

Expand Down Expand Up @@ -771,7 +770,7 @@ mod tests {
let factory = S3LogStoreFactory::default();
let store = InMemory::new();
let url = Url::parse("s3://test-bucket").unwrap();
std::env::remove_var(crate::constants::AWS_S3_LOCKING_PROVIDER);
unsafe { std::env::remove_var(crate::constants::AWS_S3_LOCKING_PROVIDER); }
let logstore = factory
.with_options(Arc::new(store), &url, &StorageOptions::from(HashMap::new()))
.unwrap();
Expand Down
116 changes: 59 additions & 57 deletions crates/aws/src/storage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -529,30 +529,30 @@ mod tests {
fn storage_options_default_test() {
ScopedEnv::run(|| {
clear_env_of_aws_keys();

std::env::set_var(constants::AWS_ENDPOINT_URL, "http://localhost");
std::env::set_var(constants::AWS_REGION, "us-west-1");
std::env::set_var(constants::AWS_PROFILE, "default");
std::env::set_var(constants::AWS_ACCESS_KEY_ID, "default_key_id");
std::env::set_var(constants::AWS_SECRET_ACCESS_KEY, "default_secret_key");
std::env::set_var(constants::AWS_S3_LOCKING_PROVIDER, "dynamodb");
std::env::set_var(
constants::AWS_IAM_ROLE_ARN,
"arn:aws:iam::123456789012:role/some_role",
);
std::env::set_var(constants::AWS_IAM_ROLE_SESSION_NAME, "session_name");
std::env::set_var(
#[allow(deprecated)]
constants::AWS_S3_ASSUME_ROLE_ARN,
"arn:aws:iam::123456789012:role/some_role",
);
std::env::set_var(
#[allow(deprecated)]
constants::AWS_S3_ROLE_SESSION_NAME,
"session_name",
);
std::env::set_var(constants::AWS_WEB_IDENTITY_TOKEN_FILE, "token_file");

unsafe {
std::env::set_var(constants::AWS_ENDPOINT_URL, "http://localhost");
std::env::set_var(constants::AWS_REGION, "us-west-1");
std::env::set_var(constants::AWS_PROFILE, "default");
std::env::set_var(constants::AWS_ACCESS_KEY_ID, "default_key_id");
std::env::set_var(constants::AWS_SECRET_ACCESS_KEY, "default_secret_key");
std::env::set_var(constants::AWS_S3_LOCKING_PROVIDER, "dynamodb");
std::env::set_var(
constants::AWS_IAM_ROLE_ARN,
"arn:aws:iam::123456789012:role/some_role",
);
std::env::set_var(constants::AWS_IAM_ROLE_SESSION_NAME, "session_name");
std::env::set_var(
#[allow(deprecated)]
constants::AWS_S3_ASSUME_ROLE_ARN,
"arn:aws:iam::123456789012:role/some_role",
);
std::env::set_var(
#[allow(deprecated)]
constants::AWS_S3_ROLE_SESSION_NAME,
"session_name",
);
std::env::set_var(constants::AWS_WEB_IDENTITY_TOKEN_FILE, "token_file");
}
let options = S3StorageOptions::try_default().unwrap();
assert_eq!(
S3StorageOptions {
Expand Down Expand Up @@ -585,7 +585,7 @@ mod tests {
fn storage_options_with_only_region_and_credentials() {
ScopedEnv::run(|| {
clear_env_of_aws_keys();
std::env::remove_var(constants::AWS_ENDPOINT_URL);
unsafe { std::env::remove_var(constants::AWS_ENDPOINT_URL); }
let options = S3StorageOptions::from_map(&hashmap! {
constants::AWS_REGION.to_string() => "eu-west-1".to_string(),
constants::AWS_ACCESS_KEY_ID.to_string() => "test".to_string(),
Expand Down Expand Up @@ -676,26 +676,28 @@ mod tests {
fn storage_options_mixed_test() {
ScopedEnv::run(|| {
clear_env_of_aws_keys();
std::env::set_var(constants::AWS_ENDPOINT_URL, "http://localhost");
std::env::set_var(
constants::AWS_ENDPOINT_URL_DYNAMODB,
"http://localhost:dynamodb",
);
std::env::set_var(constants::AWS_REGION, "us-west-1");
std::env::set_var(constants::AWS_PROFILE, "default");
std::env::set_var(constants::AWS_ACCESS_KEY_ID, "wrong_key_id");
std::env::set_var(constants::AWS_SECRET_ACCESS_KEY, "wrong_secret_key");
std::env::set_var(constants::AWS_S3_LOCKING_PROVIDER, "dynamodb");
std::env::set_var(
constants::AWS_IAM_ROLE_ARN,
"arn:aws:iam::123456789012:role/some_role",
);
std::env::set_var(constants::AWS_IAM_ROLE_SESSION_NAME, "session_name");
std::env::set_var(constants::AWS_WEB_IDENTITY_TOKEN_FILE, "token_file");

std::env::set_var(constants::AWS_S3_POOL_IDLE_TIMEOUT_SECONDS, "1");
std::env::set_var(constants::AWS_STS_POOL_IDLE_TIMEOUT_SECONDS, "2");
std::env::set_var(constants::AWS_S3_GET_INTERNAL_SERVER_ERROR_RETRIES, "3");
unsafe {
std::env::set_var(constants::AWS_ENDPOINT_URL, "http://localhost");
std::env::set_var(
constants::AWS_ENDPOINT_URL_DYNAMODB,
"http://localhost:dynamodb",
);
std::env::set_var(constants::AWS_REGION, "us-west-1");
std::env::set_var(constants::AWS_PROFILE, "default");
std::env::set_var(constants::AWS_ACCESS_KEY_ID, "wrong_key_id");
std::env::set_var(constants::AWS_SECRET_ACCESS_KEY, "wrong_secret_key");
std::env::set_var(constants::AWS_S3_LOCKING_PROVIDER, "dynamodb");
std::env::set_var(
constants::AWS_IAM_ROLE_ARN,
"arn:aws:iam::123456789012:role/some_role",
);
std::env::set_var(constants::AWS_IAM_ROLE_SESSION_NAME, "session_name");
std::env::set_var(constants::AWS_WEB_IDENTITY_TOKEN_FILE, "token_file");

std::env::set_var(constants::AWS_S3_POOL_IDLE_TIMEOUT_SECONDS, "1");
std::env::set_var(constants::AWS_STS_POOL_IDLE_TIMEOUT_SECONDS, "2");
std::env::set_var(constants::AWS_S3_GET_INTERNAL_SERVER_ERROR_RETRIES, "3");
}
let options = S3StorageOptions::from_map(&hashmap! {
constants::AWS_ACCESS_KEY_ID.to_string() => "test_id_mixed".to_string(),
constants::AWS_SECRET_ACCESS_KEY.to_string() => "test_secret_mixed".to_string(),
Expand Down Expand Up @@ -767,12 +769,12 @@ mod tests {
ScopedEnv::run(|| {
clear_env_of_aws_keys();
let raw_options = hashmap! {};

std::env::set_var(constants::AWS_ACCESS_KEY_ID, "env_key");
std::env::set_var(constants::AWS_ENDPOINT_URL, "env_key");
std::env::set_var(constants::AWS_SECRET_ACCESS_KEY, "env_key");
std::env::set_var(constants::AWS_REGION, "env_key");

unsafe {
std::env::set_var(constants::AWS_ACCESS_KEY_ID, "env_key");
std::env::set_var(constants::AWS_ENDPOINT_URL, "env_key");
std::env::set_var(constants::AWS_SECRET_ACCESS_KEY, "env_key");
std::env::set_var(constants::AWS_REGION, "env_key");
}
let combined_options =
S3ObjectStoreFactory {}.with_env_s3(&StorageOptions(raw_options));

Expand All @@ -795,12 +797,12 @@ mod tests {
"AWS_SECRET_ACCESS_KEY".to_string() => "options_key".to_string(),
"AWS_REGION".to_string() => "options_key".to_string()
};

std::env::set_var("aws_access_key_id", "env_key");
std::env::set_var("aws_endpoint", "env_key");
std::env::set_var("aws_secret_access_key", "env_key");
std::env::set_var("aws_region", "env_key");

unsafe {
std::env::set_var("aws_access_key_id", "env_key");
std::env::set_var("aws_endpoint", "env_key");
std::env::set_var("aws_secret_access_key", "env_key");
std::env::set_var("aws_region", "env_key");
}
let combined_options =
S3ObjectStoreFactory {}.with_env_s3(&StorageOptions(raw_options));

Expand Down
8 changes: 4 additions & 4 deletions crates/aws/tests/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use deltalake_aws::constants;
use deltalake_aws::register_handlers;
use deltalake_aws::storage::*;
use deltalake_test::utils::*;
use rand::Rng;
use rand::{random, Rng};
use std::process::{Command, ExitStatus, Stdio};

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -43,14 +43,14 @@ impl StorageIntegration for S3Integration {
fn prepare_env(&self) {
set_env_if_not_set(
constants::LOCK_TABLE_KEY_NAME,
format!("delta_log_it_{}", rand::thread_rng().gen::<u16>()),
format!("delta_log_it_{}", random::<u16>()),
);
match std::env::var(s3_constants::AWS_ENDPOINT_URL).ok() {
Some(endpoint_url) if endpoint_url.to_lowercase() == "none" => {
std::env::remove_var(s3_constants::AWS_ENDPOINT_URL)
unsafe { std::env::remove_var(s3_constants::AWS_ENDPOINT_URL) }
}
Some(_) => (),
None => std::env::set_var(s3_constants::AWS_ENDPOINT_URL, "http://localhost:4566"),
None => unsafe { std::env::set_var(s3_constants::AWS_ENDPOINT_URL, "http://localhost:4566") },
}
set_env_if_not_set(s3_constants::AWS_ACCESS_KEY_ID, "deltalake");
set_env_if_not_set(s3_constants::AWS_SECRET_ACCESS_KEY, "weloverust");
Expand Down
38 changes: 38 additions & 0 deletions crates/catalog-unity/Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
[package]
name = "deltalake-catalog-unity"
version = "0.6.0"
authors.workspace = true
keywords.workspace = true
readme.workspace = true
edition.workspace = true
homepage.workspace = true
description.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true

[dependencies]
async-trait = { workspace = true }
deltalake-core = { version = "0.22", path = "../core", features = ["unity-experimental"] }
thiserror = { workspace = true }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "http2"] }
rand = "0.8"
futures = "0.3"
chrono = "0.4"
tokio.workspace = true
serde.workspace = true
serde_json.workspace = true
dashmap = "6"
tracing = "0.1"
datafusion = { version = "43", optional = true }
datafusion-common = { version = "43", optional = true }

[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
tempfile = "3"
httpmock = { version = "0.8.0-alpha.1", features = [] }

[features]
default = []
datafusion = ["dep:datafusion", "datafusion-common"]

0 ...s/core/src/data_catalog/client/backoff.rs → crates/catalog-unity/src/client/backoff.rs
View file
File renamed without changes.
0 ...re/src/data_catalog/client/mock_server.rs → ...s/catalog-unity/src/client/mock_server.rs
View file
File renamed without changes.
4 changes: 2 additions & 2 deletions crates/core/src/data_catalog/client/mod.rs → crates/catalog-unity/src/client/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
//! Generic utilities reqwest based Catalog implementations
pub mod backoff;
#[cfg(test)]
pub mod mock_server;
// #[cfg(test)]
// pub mod mock_server;
#[allow(unused)]
pub mod pagination;
pub mod retry;
Expand Down
2 changes: 1 addition & 1 deletion ...ore/src/data_catalog/client/pagination.rs → ...es/catalog-unity/src/client/pagination.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::future::Future;

use futures::Stream;

use crate::data_catalog::DataCatalogResult;
use deltalake_core::data_catalog::DataCatalogResult;

/// Takes a paginated operation `op` that when called with:
///
Expand Down
Loading

0 comments on commit 3ce734b

Please sign in to comment.