Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARROW-9453: [Rust] Wasm32 compilation support #7767

Closed
wants to merge 1 commit into from
Closed

ARROW-9453: [Rust] Wasm32 compilation support #7767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions rust/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,7 @@ members = [
"integration-testing",
"benchmarks",
]
default-members= [
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why add this? is this related to the change?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this just so that wasm builds by default at the top level? If so, I'd rather not do this either. i.e. it's enough to just say that we can build the sub-crates on wasm instead of changing this global default.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. This was add as I did not try and port the Flight and Parquet submodules as I believe their core functionality uses file and sockets which don't have stable support in WASM yet.

I'll try out the individual submodule build, add details to readme and revert the Cargo.toml change.

"arrow",
"datafusion"
]
2 changes: 1 addition & 1 deletion rust/arrow-flight/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ bytes = "0.5"
prost = "0.6"
prost-derive = "0.6"
tokio = {version = "0.2", features = ["macros"]}
futures = { version = "0.3", default-features = false, features = ["alloc"]}
futures = { version = "0.3.2", default-features = false, features = ["alloc"]}

[build-dependencies]
tonic-build = "0.2"
Expand Down
14 changes: 8 additions & 6 deletions rust/arrow/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,19 @@ path = "src/lib.rs"
serde = { version = "1.0", features = ["rc"] }
serde_derive = "1.0"
serde_json = { version = "1.0", features = ["preserve_order"] }
indexmap = "1.4"
rand = "0.7"
csv = "1.1"
num = "0.3"
regex = "1.3"
lazy_static = "1.4"
packed_simd = { version = "0.3", optional = true }
chrono = "0.4"
flatbuffers = "0.6"
hex = "0.4"
arrow-flight = { path = "../arrow-flight", optional = true, version = "1.0.0-SNAPSHOT" }

[target.'cfg(not(target_arch="wasm32"))'.dependencies]
indexmap = "1.3"
rand = "0.7"
csv = "1.1"
lazy_static = "1.4"
arrow-flight = { path = "../arrow-flight", optional = true }
prettytable-rs = { version = "0.8.0", optional = true }

[features]
Expand All @@ -58,7 +60,7 @@ flight = ["arrow-flight"]
prettyprint = ["prettytable-rs"]
default = ["flight", "prettyprint"]

[dev-dependencies]
[target.'cfg(not(target_wasm="wasm32"))'.dev-dependencies]
criterion = "0.3"
lazy_static = "1"
flate2 = "1"
Expand Down
7 changes: 6 additions & 1 deletion rust/arrow/examples/read_csv.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,17 @@ extern crate arrow;
use std::fs::File;
use std::sync::Arc;

#[cfg(not(target_arch="wasm32"))]
use arrow::csv;
use arrow::datatypes::{DataType, Field, Schema};
use arrow::error::Result;
#[cfg(feature = "prettyprint")]
#[cfg(all(feature="prettyprint"), not(target_arch="wasm32"))]
use arrow::util::pretty::print_batches;

#[cfg(target_arch="wasm32")]
fn main() {}

#[cfg(not(target_arch="wasm32"))]
fn main() -> Result<()> {
let schema = Schema::new(vec![
Field::new("city", DataType::Utf8, false),
Expand Down
9 changes: 8 additions & 1 deletion rust/arrow/examples/read_csv_infer_schema.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,19 @@

extern crate arrow;

#[cfg(not(target_arch="wasm32"))]
use arrow::csv;
use arrow::error::Result;
#[cfg(feature = "prettyprint")]
#[cfg(all(feature="prettyprint", not(target_arch="wasm32")))]
use arrow::util::pretty::print_batches;
use std::fs::File;

#[cfg(target_arch="wasm32")]
fn main() {
}


#[cfg(not(target_arch="wasm32"))]
fn main() -> Result<()> {
let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();
let builder = csv::ReaderBuilder::new()
Expand Down
8 changes: 8 additions & 0 deletions rust/arrow/src/array/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use std::iter::{FromIterator, IntoIterator};
use std::mem;
use std::sync::Arc;

#[cfg(not(target_arch="wasm32"))]
use chrono::prelude::*;

use super::*;
Expand Down Expand Up @@ -502,6 +503,7 @@ impl<T: ArrowNumericType> PrimitiveArray<T> {
}
}

#[cfg(not(target_arch="wasm32"))]
impl<T: ArrowTemporalType + ArrowNumericType> PrimitiveArray<T>
where
i64: std::convert::From<T::Native>,
Expand Down Expand Up @@ -554,13 +556,15 @@ where
/// Returns value as a chrono `NaiveDate` by using `Self::datetime()`
///
/// If a data type cannot be converted to `NaiveDate`, a `None` is returned
#[cfg(not(target_arch="wasm32"))]
pub fn value_as_date(&self, i: usize) -> Option<NaiveDate> {
self.value_as_datetime(i).map(|datetime| datetime.date())
}

/// Returns a value as a chrono `NaiveTime`
///
/// `Date32` and `Date64` return UTC midnight as they do not have time resolution
#[cfg(not(target_arch="wasm32"))]
pub fn value_as_time(&self, i: usize) -> Option<NaiveTime> {
match self.data_type() {
DataType::Time32(unit) => {
Expand Down Expand Up @@ -637,6 +641,7 @@ impl<T: ArrowNumericType> fmt::Debug for PrimitiveArray<T> {
}
}

#[cfg(not(target_arch="wasm32"))]
impl<T: ArrowNumericType + ArrowTemporalType> fmt::Debug for PrimitiveArray<T>
where
i64: std::convert::From<T::Native>,
Expand Down Expand Up @@ -2401,6 +2406,7 @@ mod tests {
}

#[test]
#[cfg(not(target_arch="wasm32"))]
fn test_date64_array_from_vec_option() {
// Test building a primitive array with null values
// we use Int32 and Int64 as a backing array, so all Int32 and Int64 conventions
Expand Down Expand Up @@ -2428,6 +2434,7 @@ mod tests {
}

#[test]
#[cfg(not(target_arch="wasm32"))]
fn test_time32_millisecond_array_from_vec() {
// 1: 00:00:00.001
// 37800005: 10:30:00.005
Expand All @@ -2448,6 +2455,7 @@ mod tests {
}

#[test]
#[cfg(not(target_arch="wasm32"))]
fn test_time64_nanosecond_array_from_vec() {
// Test building a primitive array with null values
// we use Int32 and Int64 as a backing array, so all Int32 and Int64 conventions
Expand Down
8 changes: 4 additions & 4 deletions rust/arrow/src/buffer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ impl<T: AsRef<[u8]>> From<T> for Buffer {
}

/// Helper function for SIMD `BitAnd` and `BitOr` implementations
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
fn bitwise_bin_op_simd_helper<F>(left: &Buffer, right: &Buffer, op: F) -> Buffer
where
F: Fn(u8x64, u8x64) -> u8x64,
Expand Down Expand Up @@ -300,7 +300,7 @@ impl<'a, 'b> BitAnd<&'b Buffer> for &'a Buffer {
}

// SIMD implementation if available
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
{
return Ok(bitwise_bin_op_simd_helper(&self, &rhs, |a, b| a & b));
}
Expand Down Expand Up @@ -334,7 +334,7 @@ impl<'a, 'b> BitOr<&'b Buffer> for &'a Buffer {
}

// SIMD implementation if available
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
{
return Ok(bitwise_bin_op_simd_helper(&self, &rhs, |a, b| a | b));
}
Expand Down Expand Up @@ -362,7 +362,7 @@ impl Not for &Buffer {

fn not(self) -> Buffer {
// SIMD implementation if available
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
{
let mut result =
MutableBuffer::new(self.len()).with_bitset(self.len(), false);
Expand Down
16 changes: 8 additions & 8 deletions rust/arrow/src/compute/kernels/arithmetic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ use crate::buffer::Buffer;
#[cfg(feature = "simd")]
use crate::buffer::MutableBuffer;
use crate::compute::util::apply_bin_op_to_option_bitmap;
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
use crate::compute::util::simd_load_set_invalid;
use crate::datatypes;
use crate::datatypes::ToByteSlice;
Expand Down Expand Up @@ -99,8 +99,8 @@ where
}

/// SIMD vectorized version of `math_op` above.
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
fn simd_math_op<T, F>(
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
pub fn simd_math_op<T, F>(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These don't need to be public, they are internally selected based on feature gates.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. I made this modification to explicitly test that simd op call was not failing in WASM. Forgot to revert while pushing

left: &PrimitiveArray<T>,
right: &PrimitiveArray<T>,
op: F,
Expand Down Expand Up @@ -158,7 +158,7 @@ where
/// SIMD vectorized version of `divide`, the divide kernel needs it's own implementation as there
/// is a need to handle situations where a divide by `0` occurs. This is complicated by `NULL`
/// slots and padding.
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
fn simd_divide<T>(
left: &PrimitiveArray<T>,
right: &PrimitiveArray<T>,
Expand Down Expand Up @@ -234,7 +234,7 @@ where
+ Div<Output = T::Native>
+ Zero,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_math_op(&left, &right, |a, b| a + b);

#[allow(unreachable_code)]
Expand All @@ -255,7 +255,7 @@ where
+ Div<Output = T::Native>
+ Zero,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_math_op(&left, &right, |a, b| a - b);

#[allow(unreachable_code)]
Expand All @@ -276,7 +276,7 @@ where
+ Div<Output = T::Native>
+ Zero,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_math_op(&left, &right, |a, b| a * b);

#[allow(unreachable_code)]
Expand All @@ -299,7 +299,7 @@ where
+ Zero
+ One,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_divide(&left, &right);

#[allow(unreachable_code)]
Expand Down
26 changes: 13 additions & 13 deletions rust/arrow/src/compute/kernels/comparison.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ pub fn gt_eq_utf8_scalar(left: &StringArray, right: &str) -> Result<BooleanArray

/// Helper function to perform boolean lambda function on values from two arrays using
/// SIMD.
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
fn simd_compare_op<T, F>(
left: &PrimitiveArray<T>,
right: &PrimitiveArray<T>,
Expand Down Expand Up @@ -379,11 +379,11 @@ pub fn eq<T>(left: &PrimitiveArray<T>, right: &PrimitiveArray<T>) -> Result<Bool
where
T: ArrowNumericType,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_compare_op(left, right, T::eq);

#[cfg(any(
not(any(target_arch = "x86", target_arch = "x86_64")),
not(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32")),
not(feature = "simd")
))]
compare_op!(left, right, |a, b| a == b)
Expand All @@ -409,11 +409,11 @@ pub fn neq<T>(left: &PrimitiveArray<T>, right: &PrimitiveArray<T>) -> Result<Boo
where
T: ArrowNumericType,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_compare_op(left, right, T::ne);

#[cfg(any(
not(any(target_arch = "x86", target_arch = "x86_64")),
not(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32")),
not(feature = "simd")
))]
compare_op!(left, right, |a, b| a != b)
Expand All @@ -440,11 +440,11 @@ pub fn lt<T>(left: &PrimitiveArray<T>, right: &PrimitiveArray<T>) -> Result<Bool
where
T: ArrowNumericType,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_compare_op(left, right, T::lt);

#[cfg(any(
not(any(target_arch = "x86", target_arch = "x86_64")),
not(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32")),
not(feature = "simd")
))]
compare_op!(left, right, |a, b| a < b)
Expand Down Expand Up @@ -475,11 +475,11 @@ pub fn lt_eq<T>(
where
T: ArrowNumericType,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_compare_op(left, right, T::le);

#[cfg(any(
not(any(target_arch = "x86", target_arch = "x86_64")),
not(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32")),
not(feature = "simd")
))]
compare_op!(left, right, |a, b| a <= b)
Expand Down Expand Up @@ -507,11 +507,11 @@ pub fn gt<T>(left: &PrimitiveArray<T>, right: &PrimitiveArray<T>) -> Result<Bool
where
T: ArrowNumericType,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_compare_op(left, right, T::gt);

#[cfg(any(
not(any(target_arch = "x86", target_arch = "x86_64")),
not(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32")),
not(feature = "simd")
))]
compare_op!(left, right, |a, b| a > b)
Expand Down Expand Up @@ -542,11 +542,11 @@ pub fn gt_eq<T>(
where
T: ArrowNumericType,
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32"), feature = "simd"))]
return simd_compare_op(left, right, T::ge);

#[cfg(any(
not(any(target_arch = "x86", target_arch = "x86_64")),
not(any(target_arch = "x86", target_arch = "x86_64", target_arch="wasm32")),
not(feature = "simd")
))]
compare_op!(left, right, |a, b| a >= b)
Expand Down
4 changes: 4 additions & 0 deletions rust/arrow/src/compute/kernels/temporal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@

//! Defines temporal kernels for time and date related functions.

#[cfg(not(target_arch="wasm32"))]
use chrono::Timelike;

use crate::array::*;
use crate::datatypes::*;
use crate::error::Result;

/// Extracts the hours of a given temporal array as an array of integers
#[cfg(not(target_arch="wasm32"))]
pub fn hour<T>(array: &PrimitiveArray<T>) -> Result<Int32Array>
where
T: ArrowTemporalType + ArrowNumericType,
Expand Down Expand Up @@ -57,6 +59,7 @@ mod tests {
use super::*;

#[test]
#[cfg(not(target_arch="wasm32"))]
fn test_temporal_array_date64_hour() {
let a: PrimitiveArray<Date64Type> =
vec![Some(1514764800000), None, Some(1550636625000)].into();
Expand All @@ -69,6 +72,7 @@ mod tests {
}

#[test]
#[cfg(not(target_arch="wasm32"))]
fn test_temporal_array_time32_second_hour() {
let a: PrimitiveArray<Time32SecondType> = vec![37800, 86339].into();

Expand Down
Loading