diff --git a/Cargo.lock b/Cargo.lock index 512d28c5fd..dabb96dc93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "anyhow" version = "1.0.69" @@ -105,12 +120,11 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" dependencies = [ + "brotli", "futures-core", "memchr", "pin-project-lite", "tokio", - "zstd", - "zstd-safe", ] [[package]] @@ -226,6 +240,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.12.0" @@ -280,9 +315,6 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" -dependencies = [ - "jobserver", -] [[package]] name = "cfg-if" @@ -1009,15 +1041,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.61" @@ -1433,12 +1456,6 @@ dependencies = [ "spki", ] -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - [[package]] name = "portable-atomic" version = "0.3.19" @@ -2824,33 +2841,3 @@ dependencies = [ "syn", "synstructure", ] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" -dependencies = [ - "cc", - "libc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index d971d0f897..89ce62966a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ rust-version = "1.63" [dependencies] abao = { version = "0.2.0", features = ["group_size_16k", "tokio_io"], default-features = false } anyhow = { version = "1", features = ["backtrace"] } -async-compression = { version = "0.3.15", features = ["tokio", "zstd"] } +async-compression = { version = "0.3.15", features = ["tokio", "brotli"] } base64 = "0.21.0" blake3 = "1.3.3" bytes = "1" diff --git a/src/get.rs b/src/get.rs index 4564ea8a77..29f4cb35a7 100644 --- a/src/get.rs +++ b/src/get.rs @@ -106,7 +106,7 @@ impl Stats { pub struct DataStream(AsyncSliceDecoder); type RecvStream = - async_compression::tokio::bufread::ZstdDecoder>; + async_compression::tokio::bufread::BrotliDecoder>; impl DataStream { fn new(inner: RecvStream, hash: Hash) -> Self { @@ -284,7 +284,7 @@ async fn handle_blob_response( assert!(buffer.is_empty()); // Decompress data let decompress_reader = - async_compression::tokio::bufread::ZstdDecoder::new(reader); + async_compression::tokio::bufread::BrotliDecoder::new(reader); let decoder = DataStream::new(decompress_reader, hash); Ok(decoder) } diff --git a/src/provider/mod.rs b/src/provider/mod.rs index 777155dd78..c029688f09 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -838,14 +838,10 @@ async fn send_blob( // taking a reference does not work. spawn_blocking requires // 'static lifetime. - writer = tokio::task::spawn_blocking(move || { - // Compress data - let mut compressed_writer = - async_compression::tokio::write::ZstdEncoder::with_quality( - writer, - async_compression::Level::Fastest, - ); + // Compress data + let mut compressed_writer = async_compression::tokio::write::BrotliEncoder::new(writer); + compressed_writer = tokio::task::spawn_blocking(move || { let file_reader = std::fs::File::open(&path)?; let outboard_reader = std::io::Cursor::new(outboard); let mut wrapper = SyncIoBridge::new(&mut compressed_writer); @@ -856,10 +852,13 @@ async fn send_blob( size, ); let _copied = std::io::copy(&mut slice_extractor, &mut wrapper)?; - std::io::Result::Ok(compressed_writer.into_inner()) + std::io::Result::Ok(compressed_writer) }) .await??; + compressed_writer.flush().await?; + let writer = compressed_writer.into_inner(); + Ok((SentStatus::Sent, writer, size)) } _ => {