From 759ffefccc5666aae18649fefac3cf03de90f3f7 Mon Sep 17 00:00:00 2001 From: WenyXu Date: Wed, 7 Aug 2024 17:33:39 +0000 Subject: [PATCH 1/3] chore(integrations/parquet): add README --- integrations/parquet/README.md | 104 ++++++++++++++++++++++++++++++++ integrations/parquet/src/lib.rs | 4 +- 2 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 integrations/parquet/README.md diff --git a/integrations/parquet/README.md b/integrations/parquet/README.md new file mode 100644 index 000000000000..56b7dd0ba4ca --- /dev/null +++ b/integrations/parquet/README.md @@ -0,0 +1,104 @@ +# Apache OpenDAL™ parquet integration + +[![Build Status]][actions] [![Latest Version]][crates.io] [![Crate Downloads]][crates.io] [![chat]][discord] + +[build status]: https://img.shields.io/github/actions/workflow/status/apache/opendal/ci_integration_parquet.yml?branch=main +[actions]: https://github.com/apache/opendal/actions?query=branch%3Amain +[latest version]: https://img.shields.io/crates/v/parquet_opendal.svg +[crates.io]: https://crates.io/crates/parquet_opendal +[crate downloads]: https://img.shields.io/crates/d/parquet_opendal.svg +[chat]: https://img.shields.io/discord/1081052318650339399 +[discord]: https://opendal.apache.org/discord + +`parquet_opendal` provides [`parquet`](https://crates.io/crates/parquet) efficient IO utilities. + +## Useful Links + +- Documentation: [release](https://docs.rs/parquet_opendal/) | [dev](https://opendal.apache.org/docs/object-store-opendal/parquet_opendal/) + +## Examples + +Add the following dependencies to your `Cargo.toml` with correct version: + +```toml +[dependencies] +parquet_opendal = "0.0.1" +opendal = { version = "0.48.0", features = ["services-s3"] } +``` + + +```rust +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int64Array, RecordBatch}; + +use futures::StreamExt; +use opendal::{services::S3Config, Operator}; +use parquet::arrow::{AsyncArrowWriter, ParquetRecordBatchStreamBuilder}; +use parquet_opendal::{AsyncReader, AsyncWriter}; + +#[tokio::main] +async fn main() { + let mut cfg = S3Config::default(); + cfg.access_key_id = Some("my_access_key".to_string()); + cfg.secret_access_key = Some("my_secret_key".to_string()); + cfg.endpoint = Some("my_endpoint".to_string()); + cfg.region = Some("my_region".to_string()); + cfg.bucket = "my_bucket".to_string(); + + // Create a new operator + let operator = Operator::from_config(cfg).unwrap().finish(); + let path = "/path/to/file.parquet"; + + // Create an async writer + let writer = AsyncWriter::new( + operator + .writer_with(path) + .chunk(32 * 1024 * 1024) + .concurrent(8) + .await + .unwrap(), + ); + + let col = Arc::new(Int64Array::from_iter_values([1, 2, 3])) as ArrayRef; + let to_write = RecordBatch::try_from_iter([("col", col)]).unwrap(); + let mut writer = AsyncArrowWriter::try_new(writer, to_write.schema(), None).unwrap(); + writer.write(&to_write).await.unwrap(); + writer.close().await.unwrap(); + + /// `gap(512 * 1024)` - Sets the maximum gap size (in bytes) to merge small byte ranges + /// to 512 KB. + /// `chunk(16 * 1024 * 1024)` - Sets the chunk size (in bytes) for reading data to 16 MB. + /// `concurrent(16)` - Sets the number of concurrent fetch operations to 16. + let reader = operator + .reader_with(path) + .gap(512 * 1024) + .chunk(16 * 1024 * 1024) + .concurrent(16) + .await + .unwrap(); + + let content_len = operator.stat(path).await.unwrap().content_length(); + // `with_prefetch_footer_size(512 * 1024)` - Sets the prefetch footer size to 512 KB. + let reader = AsyncReader::new(reader, content_len).with_prefetch_footer_size(512 * 1024); + let mut stream = ParquetRecordBatchStreamBuilder::new(reader) + .await + .unwrap() + .build() + .unwrap(); + let read = stream.next().await.unwrap().unwrap(); + assert_eq!(to_write, read); +} +``` + +## Branding + +The first and most prominent mentions must use the full form: **Apache OpenDAL™** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization. + +For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide). + +## License and Trademarks + +Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + +Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. diff --git a/integrations/parquet/src/lib.rs b/integrations/parquet/src/lib.rs index ed5d932b10f9..54239e60b4b4 100644 --- a/integrations/parquet/src/lib.rs +++ b/integrations/parquet/src/lib.rs @@ -15,9 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! parquet_opendal provides parquet IO utils. -//! -//! AsyncWriter implements AsyncFileWriter trait by using opendal. +//! parquet_opendal provides parquet IO utilities. //! //! ```no_run //! use std::sync::Arc; From 0c2428cf7eae28ffa6f9f1b680ed32f469c24a39 Mon Sep 17 00:00:00 2001 From: WenyXu Date: Thu, 8 Aug 2024 02:57:49 +0000 Subject: [PATCH 2/3] docs: add parquet_opendal to README.md --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 474625cf70e8..895be4103c76 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,6 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e [Swift Binding]: bindings/swift/README.md [Zig Binding]: bindings/zig/README.md - ## For *ANY* methods | Name | Description | Release | @@ -89,6 +88,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e | [fuse3_opendal] | Access data via integrations to [fuse3] | [![fuse3 image]][fuse3 crate] | [![Docs Release]][fuse3 release docs] [![Docs Dev]][fuse3 dev docs] | | [virtiofs_opendal] | Access data via integrations to [vhost-user-backend] | [![virtiofs image]][virtiofs crate] | [![Docs Release]][virtiofs release docs] [![Docs Dev]][virtiofs dev docs] | | [unftp-sbe-opendal] | an [unftp] storage backend implementation using opendal. | [![unftp-sbe image]][unftp-sbe crate] | [![Docs Release]][unftp-sbe release docs] [![Docs Dev]][unftp-sbe dev docs] | +| [parquet_opendal] | Provides [`parquet`](https://crates.io/crates/parquet) efficient IO utilities | [![parquet image]][parquet crate] | [![Docs Release]][parquet release docs] [![Docs Dev]][parquet dev docs] | [dav-server-opendalfs]: integrations/dav-server/README.md [dav-server-rs]: https://github.com/messense/dav-server-rs @@ -125,6 +125,12 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e [unftp-sbe release docs]: https://docs.rs/unftp-sbe-opendal/ [unftp-sbe dev docs]: https://opendal.apache.org/docs/unftp-sbe-opendal/unftp_sbe_opendal/ +[parquet_opendal]: integrations/parquet/README.md +[parquet image]: https://img.shields.io/crates/v/parquet-opendal.svg +[parquet crate]: https://crates.io/crates/parquet-opendal +[parquet release docs]: https://docs.rs/parquet-opendal/ +[parquet dev docs]: https://opendal.apache.org/docs/parquet-opendal/parquet_opendal/ + ## For *ANY* services | Type | Services | @@ -252,6 +258,6 @@ For more details, see the [Apache Product Name Usage Guide](https://www.apache.o ## License and Trademarks -Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 +Licensed under the Apache License, Version 2.0: Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. From ed22cceda2e2e61bd920856c90fa8eb348ec8cea Mon Sep 17 00:00:00 2001 From: WenyXu Date: Thu, 8 Aug 2024 02:58:11 +0000 Subject: [PATCH 3/3] chore: lint --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 895be4103c76..b478cadde8bd 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e ## For *ANY* languages | Name | Release | Docs | -|-------------------|--------------------------------------------------|-----------------------------------------------------------------------------------| +| ----------------- | ------------------------------------------------ | --------------------------------------------------------------------------------- | | [Rust Core] | [![Rust Core Image]][Rust Core Link] | [![Docs Release]][Rust Core Release Docs] [![Docs Dev]][Rust Core Dev Docs] | | [C Binding] | - | [![Docs Dev]][C Binding Dev Docs] | | [Cpp Binding] | - | [![Docs Dev]][Cpp Binding Dev Docs] | @@ -64,7 +64,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e ## For *ANY* methods | Name | Description | Release | -|-------|--------------------------------------------------------------------|---------------------------| +| ----- | ------------------------------------------------------------------ | ------------------------- | | [oay] | Access data via API Gateway | [![oay image]][oay crate] | | [oli] | Access data via Command Line (alternative to s3cmd, s3cli, azcopy) | [![oli image]][oli crate] | | [ofs] | Access data via POSIX file system API (alternative to s3fs) | [![ofs image]][ofs crate] | @@ -81,14 +81,14 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e ## For *ANY* integrations -| Name | Description | Release | Docs | -|------------------------|----------------------------------------------------------|---------------------------------------------|-----------------------------------------------------------------------------------| -| [dav-server-opendalfs] | a [dav-server-rs] implementation using opendal. | [![dav-server image]][dav-server crate] | [![Docs Release]][dav-server release docs] [![Docs Dev]][dav-server dev docs] | -| [object_store_opendal] | an [object_store] implementation using opendal. | [![object_store image]][object_store crate] | [![Docs Release]][object_store release docs] [![Docs Dev]][object_store dev docs] | -| [fuse3_opendal] | Access data via integrations to [fuse3] | [![fuse3 image]][fuse3 crate] | [![Docs Release]][fuse3 release docs] [![Docs Dev]][fuse3 dev docs] | -| [virtiofs_opendal] | Access data via integrations to [vhost-user-backend] | [![virtiofs image]][virtiofs crate] | [![Docs Release]][virtiofs release docs] [![Docs Dev]][virtiofs dev docs] | -| [unftp-sbe-opendal] | an [unftp] storage backend implementation using opendal. | [![unftp-sbe image]][unftp-sbe crate] | [![Docs Release]][unftp-sbe release docs] [![Docs Dev]][unftp-sbe dev docs] | -| [parquet_opendal] | Provides [`parquet`](https://crates.io/crates/parquet) efficient IO utilities | [![parquet image]][parquet crate] | [![Docs Release]][parquet release docs] [![Docs Dev]][parquet dev docs] | +| Name | Description | Release | Docs | +| ---------------------- | ----------------------------------------------------------------------------- | ------------------------------------------- | --------------------------------------------------------------------------------- | +| [dav-server-opendalfs] | a [dav-server-rs] implementation using opendal. | [![dav-server image]][dav-server crate] | [![Docs Release]][dav-server release docs] [![Docs Dev]][dav-server dev docs] | +| [object_store_opendal] | an [object_store] implementation using opendal. | [![object_store image]][object_store crate] | [![Docs Release]][object_store release docs] [![Docs Dev]][object_store dev docs] | +| [fuse3_opendal] | Access data via integrations to [fuse3] | [![fuse3 image]][fuse3 crate] | [![Docs Release]][fuse3 release docs] [![Docs Dev]][fuse3 dev docs] | +| [virtiofs_opendal] | Access data via integrations to [vhost-user-backend] | [![virtiofs image]][virtiofs crate] | [![Docs Release]][virtiofs release docs] [![Docs Dev]][virtiofs dev docs] | +| [unftp-sbe-opendal] | an [unftp] storage backend implementation using opendal. | [![unftp-sbe image]][unftp-sbe crate] | [![Docs Release]][unftp-sbe release docs] [![Docs Dev]][unftp-sbe dev docs] | +| [parquet_opendal] | Provides [`parquet`](https://crates.io/crates/parquet) efficient IO utilities | [![parquet image]][parquet crate] | [![Docs Release]][parquet release docs] [![Docs Dev]][parquet dev docs] | [dav-server-opendalfs]: integrations/dav-server/README.md [dav-server-rs]: https://github.com/messense/dav-server-rs @@ -134,7 +134,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e ## For *ANY* services | Type | Services | -|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | | Standard Storage Protocols | ftp http [sftp] [webdav] | | Object Storage Services | [azblob] [cos] [gcs] [obs] [oss] [s3]
[b2] [openstack_swift] [upyun] [vercel_blob] | | File Storage Services | fs [alluxio] [azdls] [azfile] [chainsafe] [compfs]
[dbfs] [gridfs] [hdfs] [hdfs_native] [ipfs] [webhdfs] |