Update to arrow 36 (#5685)

* Update for arrow 36

* Update pin

* Update timestamp test

* Escape .slt

* Remove pin

* Update test

Cargo.toml

@@ -46,12 +46,12 @@ repository = "https://github.com/apache/arrow-datafusion"
 rust-version = "1.64"
 
 [workspace.dependencies]
-arrow = { version = "34.0.0", features = ["prettyprint"] }
-arrow-flight = { version = "34.0.0", features = ["flight-sql-experimental"] }
-arrow-buffer = { version = "34.0.0", default-features = false }
-arrow-schema = { version = "34.0.0", default-features = false }
-arrow-array = { version = "34.0.0", default-features = false, features = ["chrono-tz"] }
-parquet = { version = "34.0.0", features = ["arrow", "async"] }
+arrow = { version = "36.0.0", features = ["prettyprint"] }
+arrow-flight = { version = "36.0.0", features = ["flight-sql-experimental"] }
+arrow-buffer = { version = "36.0.0", default-features = false }
+arrow-schema = { version = "36.0.0", default-features = false }
+arrow-array = { version = "36.0.0", default-features = false, features = ["chrono-tz"] }
+parquet = { version = "36.0.0", features = ["arrow", "async"] }
 
 [profile.release]
 codegen-units = 1

benchmarks/src/bin/tpch.rs

@@ -150,11 +150,11 @@ async fn main() -> Result<()> {
             let compression = match opt.compression.as_str() {
                 "none" => Compression::UNCOMPRESSED,
                 "snappy" => Compression::SNAPPY,
-                "brotli" => Compression::BROTLI,
-                "gzip" => Compression::GZIP,
+                "brotli" => Compression::BROTLI(Default::default()),
+                "gzip" => Compression::GZIP(Default::default()),
                 "lz4" => Compression::LZ4,
                 "lz0" => Compression::LZO,
-                "zstd" => Compression::ZSTD,
+                "zstd" => Compression::ZSTD(Default::default()),
                 other => {
                     return Err(DataFusionError::NotImplemented(format!(
                         "Invalid compression format: {other}"

datafusion-cli/Cargo.toml

@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = "34.0.0"
+arrow = "36.0.0"
 async-trait = "0.1.41"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "21.0.0" }

datafusion-cli/src/object_storage.rs

@@ -175,9 +175,7 @@ mod tests {
     fn s3_region_validation() {
         let s3 = "s3://bucket_name/path";
         let registry = DatafusionCliObjectStoreRegistry::new();
-        let err = registry
-            .get_store(&Url::from_str(s3).unwrap())
-            .unwrap_err();
+        let err = registry.get_store(&Url::from_str(s3).unwrap()).unwrap_err();
         assert!(err.to_string().contains("Generic S3 error: Missing region"));
 
         env::set_var("AWS_REGION", "us-east-1");

datafusion/common/src/scalar.rs

@@ -3437,10 +3437,6 @@ mod tests {
             ScalarValue::Decimal128(None, 10, 2),
             ScalarValue::try_from_array(&array, 3).unwrap()
         );
-        assert_eq!(
-            ScalarValue::Decimal128(None, 10, 2),
-            ScalarValue::try_from_array(&array, 4).unwrap()
-        );
 
         Ok(())
     }

datafusion/core/tests/dataframe_functions.rs

@@ -398,10 +398,10 @@ async fn test_fn_regexp_match() -> Result<()> {
         "+-----------------------------------+",
         "| regexpmatch(test.a,Utf8(\"[a-z]\")) |",
         "+-----------------------------------+",
-        "| []                                |",
-        "| []                                |",
-        "| []                                |",
-        "| []                                |",
+        "| [a]                               |",
+        "| [a]                               |",
+        "| [d]                               |",
+        "| [b]                               |",
         "+-----------------------------------+",
     ];
 

datafusion/core/tests/sql/mod.rs

@@ -1301,15 +1301,11 @@ where
     A: ArrowTimestampType<Native = i64>,
 {
     let schema = Arc::new(Schema::new(vec![
-        Field::new(
-            "ts",
-            DataType::Timestamp(A::get_time_unit(), tz.clone()),
-            false,
-        ),
+        Field::new("ts", DataType::Timestamp(A::UNIT, tz.clone()), false),
         Field::new("value", DataType::Int32, true),
     ]));
 
-    let divisor = match A::get_time_unit() {
+    let divisor = match A::UNIT {
         TimeUnit::Nanosecond => 1,
         TimeUnit::Microsecond => 1000,
         TimeUnit::Millisecond => 1_000_000,

datafusion/core/tests/sql/set_variable.rs

@@ -414,7 +414,7 @@ async fn set_time_zone_bad_time_zone_format() {
             .await
             .unwrap();
     let err = pretty_format_batches(&result).err().unwrap().to_string();
-    assert_eq!(err, "Parser error: Invalid timezone \"+08:00:00\": Expected format [+-]XX:XX, [+-]XX, or [+-]XXXX");
+    assert_eq!(err, "Parser error: Invalid timezone \"+08:00:00\": '+08:00:00' is not a valid timezone");
 
     plan_and_collect(&ctx, "SET TIME ZONE = '08:00'")
         .await

datafusion/core/tests/sqllogictests/test_files/aggregate.slt

@@ -1387,7 +1387,7 @@ as values
  ('2018-11-13T17:11:10.011375885995', 'Row 0'),
  ('2011-12-13T11:13:10.12345', 'Row 1'),
  (null, 'Row 2'),
- ('2021-1-1T05:11:10.432', 'Row 3');
+ ('2021-01-01T05:11:10.432', 'Row 3');
 
 
 statement ok

datafusion/core/tests/sqllogictests/test_files/arrow_typeof.slt

@@ -279,7 +279,7 @@ query error Cannot automatically convert Interval\(DayTime\) to Interval\(MonthD
 ---
 select arrow_cast(interval '30 minutes', 'Interval(MonthDayNano)');
 
-query error DataFusion error: Error during planning: Cannot automatically convert Utf8 to Interval\(MonthDayNano\)
+query error DataFusion error: This feature is not implemented: Can't create a scalar from array of type "Interval\(MonthDayNano\)"
 select arrow_cast('30 minutes', 'Interval(MonthDayNano)');
 
 

datafusion/core/tests/sqllogictests/test_files/dates.slt

@@ -87,4 +87,4 @@ h
 ## Plan error when compare Utf8 and timestamp in where clause
 statement error DataFusion error: Error during planning: The type of Timestamp\(Nanosecond, Some\("\+00:00"\)\) Plus Utf8 of binary physical should be same
 select i_item_desc from test
-where d3_date > now() + '5 days';
+where d3_date > now() + '5 days';

datafusion/core/tests/sqllogictests/test_files/ddl.slt

@@ -564,4 +564,4 @@ insert into foo values (null);
 query I
 select * from foo;
 ----
-NULL
+NULL

datafusion/core/tests/sqllogictests/test_files/timestamps.slt

@@ -259,4 +259,4 @@ SELECT INTERVAL '8' YEAR + '2000-01-01T00:00:00'::timestamp;
 query P
 SELECT INTERVAL '8' MONTH + '2000-01-01T00:00:00'::timestamp;
 ----
-2000-09-01T00:00:00
+2000-09-01T00:00:00

datafusion/core/tests/sqllogictests/test_files/window.slt

@@ -2073,4 +2073,3 @@ SELECT SUM(c12) OVER(ORDER BY c1, c2 GROUPS BETWEEN 1 PRECEDING AND 1 FOLLOWING)
 2.994840293343 NULL
 9.674390599321 NULL
 7.728066219895 NULL
-

datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs

@@ -431,7 +431,8 @@ mod tests {
             .project(proj)?
             .build()?;
 
-        let expected = "Error parsing 'I'M NOT A TIMESTAMP' as timestamp";
+        let expected =
+            "Error parsing timestamp from 'I'M NOT A TIMESTAMP': error parsing date";
         let actual = get_optimized_plan_err(&plan, &Utc::now());
         assert_contains!(actual, expected);
         Ok(())

datafusion/physical-expr/src/expressions/in_list.rs

@@ -144,11 +144,9 @@ where
     T: ArrayAccessor,
     T::Item: PartialEq + HashValue,
 {
-    let data = array.data();
-
     let state = RandomState::new();
     let mut map: HashMap<usize, (), ()> =
-        HashMap::with_capacity_and_hasher(data.len(), ());
+        HashMap::with_capacity_and_hasher(array.len(), ());
 
     let insert_value = |idx| {
         let value = array.value(idx);
@@ -161,10 +159,12 @@ where
         }
     };
 
-    match data.null_buffer() {
-        Some(buffer) => BitIndexIterator::new(buffer.as_ref(), data.offset(), data.len())
-            .for_each(insert_value),
-        None => (0..data.len()).for_each(insert_value),
+    match array.nulls() {
+        Some(nulls) => {
+            BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len())
+                .for_each(insert_value)
+        }
+        None => (0..array.len()).for_each(insert_value),
     }
 
     ArrayHashSet { state, map }