From 728e9cf98a76cbd23922d2dfb15acc43cab8f1e0 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 15 Nov 2023 09:01:27 -0800 Subject: [PATCH 1/9] timestamp fixes --- datafusion/common/src/scalar.rs | 3 ++ .../physical-expr/src/expressions/negative.rs | 7 +++- datafusion/sql/src/expr/mod.rs | 42 +++++++++++++------ datafusion/sql/tests/sql_integration.rs | 2 +- .../sqllogictest/test_files/timestamps.slt | 18 ++++++++ datafusion/sqllogictest/test_files/window.slt | 16 +++---- 6 files changed, 64 insertions(+), 24 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index ffa8ab50f862..10042b225f08 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -983,6 +983,9 @@ impl ScalarValue { ScalarValue::Decimal256(Some(v), precision, scale) => Ok( ScalarValue::Decimal256(Some(v.neg_wrapping()), *precision, *scale), ), + ScalarValue::TimestampSecond(Some(v), tz) => { + Ok(ScalarValue::TimestampSecond(Some(-v), tz.clone())) + } value => _internal_err!( "Can not run arithmetic negative on scalar value {value:?}" ), diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs index a59fd1ae3f20..b64b4a0c86de 100644 --- a/datafusion/physical-expr/src/expressions/negative.rs +++ b/datafusion/physical-expr/src/expressions/negative.rs @@ -33,7 +33,7 @@ use arrow::{ use datafusion_common::{internal_err, DataFusionError, Result}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::{ - type_coercion::{is_interval, is_null, is_signed_numeric}, + type_coercion::{is_interval, is_null, is_signed_numeric, is_timestamp}, ColumnarValue, }; @@ -160,7 +160,10 @@ pub fn negative( let data_type = arg.data_type(input_schema)?; if is_null(&data_type) { Ok(arg) - } else if !is_signed_numeric(&data_type) && !is_interval(&data_type) { + } else if !is_signed_numeric(&data_type) + && !is_interval(&data_type) + && !is_timestamp(&data_type) + { internal_err!( "Can't create negative physical expr for (- '{arg:?}'), the type of child expr is {data_type}, not signed numeric" ) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 7fa16ced39da..324c027f9704 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -29,6 +29,7 @@ mod value; use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use arrow_schema::DataType; +use arrow_schema::TimeUnit; use datafusion_common::{ internal_err, not_impl_err, plan_err, Column, DFSchema, DataFusionError, Result, ScalarValue, @@ -39,7 +40,10 @@ use datafusion_expr::{ col, expr, lit, AggregateFunction, Between, BinaryExpr, BuiltinScalarFunction, Cast, Expr, ExprSchemable, GetFieldAccess, GetIndexedField, Like, Operator, TryCast, }; -use sqlparser::ast::{ArrayAgg, Expr as SQLExpr, JsonOperator, TrimWhereField, Value}; +use sqlparser::ast::{ + ArrayAgg, Expr as SQLExpr, JsonOperator, TrimWhereField, + Value, +}; use sqlparser::parser::ParserError::ParserError; impl<'a, S: ContextProvider> SqlToRel<'a, S> { @@ -224,14 +228,23 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { SQLExpr::Cast { expr, data_type, .. - } => Ok(Expr::Cast(Cast::new( - Box::new(self.sql_expr_to_logical_expr( - *expr, - schema, - planner_context, - )?), - self.convert_data_type(&data_type)?, - ))), + } => { + let mut dt = self.convert_data_type(&data_type)?; + let expr = + self.sql_expr_to_logical_expr(*expr, schema, planner_context)?; + + // int/floats should come as seconds rather as nanoseconds + dt = match &dt { + DataType::Timestamp(TimeUnit::Nanosecond, tz) + if expr.get_type(schema)? == DataType::Int64 => + { + DataType::Timestamp(TimeUnit::Second, tz.clone()) + } + _ => dt, + }; + + Ok(Expr::Cast(Cast::new(Box::new(expr), dt))) + } SQLExpr::TryCast { expr, data_type, .. @@ -244,10 +257,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { self.convert_data_type(&data_type)?, ))), - SQLExpr::TypedString { data_type, value } => Ok(Expr::Cast(Cast::new( - Box::new(lit(value)), - self.convert_data_type(&data_type)?, - ))), + SQLExpr::TypedString { data_type, value } => { + dbg!("TypedString"); + Ok(Expr::Cast(Cast::new( + Box::new(lit(value)), + self.convert_data_type(&data_type)?, + ))) + } SQLExpr::IsNull(expr) => Ok(Expr::IsNull(Box::new( self.sql_expr_to_logical_expr(*expr, schema, planner_context)?, diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index a56e9a50f054..506c64349173 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -608,7 +608,7 @@ fn test_timestamp_filter() { let sql = "SELECT state FROM person WHERE birth_date < CAST (158412331400600000 as timestamp)"; let expected = "Projection: person.state\ - \n Filter: person.birth_date < CAST(Int64(158412331400600000) AS Timestamp(Nanosecond, None))\ + \n Filter: person.birth_date < CAST(Int64(158412331400600000) AS Timestamp(Second, None))\ \n TableScan: person"; quick_test(sql, expected); diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index e186aa12f7a9..9874afc024e9 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1793,3 +1793,21 @@ query PPPPP SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005) ---- NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 + +# verify to_timestamp edge cases to be in sync with postgresql +query PPPPPPPP +SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1) +---- +NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 + +# verify timestamp cast from i64 is in sync with to_timestamp(i64) +query PPPPPPPP +SELECT null::timestamp, -62125747200::timestamp, 0::timestamp, 1926632005177::timestamp, 1926632005::timestamp, 1::timestamp, -1::timestamp, (0-1)::timestamp +---- +NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 + +# verify timestamp cast from i64 is in sync with to_timestamp(i64) using CAST syntax +query PPPPPPP +SELECT cast(null as timestamp), cast(-62125747200 as timestamp), cast(0 as timestamp), cast(1926632005177 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp) +---- +NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index 1ef0ba0d10e3..319c08407661 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -895,14 +895,14 @@ SELECT statement ok create table temp as values -(1664264591000000000), -(1664264592000000000), -(1664264592000000000), -(1664264593000000000), -(1664264594000000000), -(1664364594000000000), -(1664464594000000000), -(1664564594000000000); +(1664264591), +(1664264592), +(1664264592), +(1664264593), +(1664264594), +(1664364594), +(1664464594), +(1664564594); statement ok create table t as select cast(column1 as timestamp) as ts from temp; From ecb86255e23ace38a02152e3a0eefc887217ba0a Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 15 Nov 2023 09:08:09 -0800 Subject: [PATCH 2/9] fmt --- datafusion/sql/src/expr/mod.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 324c027f9704..44d1647a4f2d 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -40,10 +40,7 @@ use datafusion_expr::{ col, expr, lit, AggregateFunction, Between, BinaryExpr, BuiltinScalarFunction, Cast, Expr, ExprSchemable, GetFieldAccess, GetIndexedField, Like, Operator, TryCast, }; -use sqlparser::ast::{ - ArrayAgg, Expr as SQLExpr, JsonOperator, TrimWhereField, - Value, -}; +use sqlparser::ast::{ArrayAgg, Expr as SQLExpr, JsonOperator, TrimWhereField, Value}; use sqlparser::parser::ParserError::ParserError; impl<'a, S: ContextProvider> SqlToRel<'a, S> { @@ -233,7 +230,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let expr = self.sql_expr_to_logical_expr(*expr, schema, planner_context)?; - // int/floats should come as seconds rather as nanoseconds + // int/floats should come as seconds rather as nanoseconds dt = match &dt { DataType::Timestamp(TimeUnit::Nanosecond, tz) if expr.get_type(schema)? == DataType::Int64 => From 5fc8524156c86a1a3ce3ba5e540129d2ba560ce5 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 15 Nov 2023 10:41:26 -0800 Subject: [PATCH 3/9] fix test --- datafusion/sqllogictest/test_files/timestamps.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index 9874afc024e9..9593e3ef8c47 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1807,7 +1807,7 @@ SELECT null::timestamp, -62125747200::timestamp, 0::timestamp, 1926632005177::ti NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 # verify timestamp cast from i64 is in sync with to_timestamp(i64) using CAST syntax -query PPPPPPP +query PPPPPPPP SELECT cast(null as timestamp), cast(-62125747200 as timestamp), cast(0 as timestamp), cast(1926632005177 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp) ---- NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 From a0f1c952f1cbfc7b331b8b7d850c4c821e4a85c2 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 15 Nov 2023 11:02:40 -0800 Subject: [PATCH 4/9] fix test --- datafusion/sqllogictest/test_files/timestamps.slt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index 9593e3ef8c47..22400fe1771d 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1788,12 +1788,6 @@ SELECT TIMESTAMPTZ '2020-01-01 00:00:00Z' = TIMESTAMP '2020-01-01' ---- true -# verify to_timestamp edge cases to be in sync with postgresql -query PPPPP -SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005) ----- -NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 - # verify to_timestamp edge cases to be in sync with postgresql query PPPPPPPP SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1) From 71a8bfc2e6112df963e3e212192a5b4fe2587484 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 15 Nov 2023 13:25:40 -0800 Subject: [PATCH 5/9] rm dbg --- datafusion/sql/src/expr/mod.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 44d1647a4f2d..63572fe4e46d 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -254,13 +254,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { self.convert_data_type(&data_type)?, ))), - SQLExpr::TypedString { data_type, value } => { - dbg!("TypedString"); - Ok(Expr::Cast(Cast::new( - Box::new(lit(value)), - self.convert_data_type(&data_type)?, - ))) - } + SQLExpr::TypedString { data_type, value } => Ok(Expr::Cast(Cast::new( + Box::new(lit(value)), + self.convert_data_type(&data_type)?, + ))), SQLExpr::IsNull(expr) => Ok(Expr::IsNull(Box::new( self.sql_expr_to_logical_expr(*expr, schema, planner_context)?, From 88139158b8a4ffbd86834df141e7ee8519ee1e01 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 Nov 2023 10:51:03 -0800 Subject: [PATCH 6/9] sync signature --- datafusion/common/src/scalar.rs | 9 ++++ datafusion/core/tests/sql/timestamp.rs | 2 +- datafusion/expr/src/built_in_function.rs | 7 +-- .../physical-expr/src/datetime_expressions.rs | 8 +-- datafusion/sql/src/expr/mod.rs | 13 +++-- datafusion/sql/tests/sql_integration.rs | 10 +--- .../sqllogictest/test_files/timestamps.slt | 49 ++++++++++++++----- 7 files changed, 64 insertions(+), 34 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index 10042b225f08..3431d71468ea 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -986,6 +986,15 @@ impl ScalarValue { ScalarValue::TimestampSecond(Some(v), tz) => { Ok(ScalarValue::TimestampSecond(Some(-v), tz.clone())) } + ScalarValue::TimestampNanosecond(Some(v), tz) => { + Ok(ScalarValue::TimestampNanosecond(Some(-v), tz.clone())) + } + ScalarValue::TimestampMicrosecond(Some(v), tz) => { + Ok(ScalarValue::TimestampMicrosecond(Some(-v), tz.clone())) + } + ScalarValue::TimestampMillisecond(Some(v), tz) => { + Ok(ScalarValue::TimestampMillisecond(Some(-v), tz.clone())) + } value => _internal_err!( "Can not run arithmetic negative on scalar value {value:?}" ), diff --git a/datafusion/core/tests/sql/timestamp.rs b/datafusion/core/tests/sql/timestamp.rs index a18e6831b615..ada66503a181 100644 --- a/datafusion/core/tests/sql/timestamp.rs +++ b/datafusion/core/tests/sql/timestamp.rs @@ -742,7 +742,7 @@ async fn test_arrow_typeof() -> Result<()> { "+-----------------------------------------------------------------------+", "| arrow_typeof(date_trunc(Utf8(\"microsecond\"),to_timestamp(Int64(61)))) |", "+-----------------------------------------------------------------------+", - "| Timestamp(Second, None) |", + "| Timestamp(Nanosecond, None) |", "+-----------------------------------------------------------------------+", ]; assert_batches_eq!(expected, &actual); diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index d92067501657..c511c752b4d7 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -779,13 +779,10 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::SubstrIndex => { utf8_to_str_type(&input_expr_types[0], "substr_index") } - BuiltinScalarFunction::ToTimestamp => Ok(match &input_expr_types[0] { - Int64 => Timestamp(Second, None), - _ => Timestamp(Nanosecond, None), - }), + BuiltinScalarFunction::ToTimestamp + | BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)), BuiltinScalarFunction::ToTimestampMillis => Ok(Timestamp(Millisecond, None)), BuiltinScalarFunction::ToTimestampMicros => Ok(Timestamp(Microsecond, None)), - BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)), BuiltinScalarFunction::ToTimestampSeconds => Ok(Timestamp(Second, None)), BuiltinScalarFunction::FromUnixtime => Ok(Timestamp(Second, None)), BuiltinScalarFunction::Now => { diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs index 5b597de78ac9..0d42708c97ec 100644 --- a/datafusion/physical-expr/src/datetime_expressions.rs +++ b/datafusion/physical-expr/src/datetime_expressions.rs @@ -966,9 +966,11 @@ pub fn to_timestamp_invoke(args: &[ColumnarValue]) -> Result { } match args[0].data_type() { - DataType::Int64 => { - cast_column(&args[0], &DataType::Timestamp(TimeUnit::Second, None), None) - } + DataType::Int64 => cast_column( + &cast_column(&args[0], &DataType::Timestamp(TimeUnit::Second, None), None)?, + &DataType::Timestamp(TimeUnit::Nanosecond, None), + None, + ), DataType::Timestamp(_, None) => cast_column( &args[0], &DataType::Timestamp(TimeUnit::Nanosecond, None), diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 63572fe4e46d..1307d5766d61 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -226,18 +226,21 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { SQLExpr::Cast { expr, data_type, .. } => { - let mut dt = self.convert_data_type(&data_type)?; + let dt = self.convert_data_type(&data_type)?; let expr = self.sql_expr_to_logical_expr(*expr, schema, planner_context)?; - // int/floats should come as seconds rather as nanoseconds - dt = match &dt { + // int/floats input should be treated as seconds rather as nanoseconds + let expr = match &dt { DataType::Timestamp(TimeUnit::Nanosecond, tz) if expr.get_type(schema)? == DataType::Int64 => { - DataType::Timestamp(TimeUnit::Second, tz.clone()) + Expr::Cast(Cast::new( + Box::new(expr), + DataType::Timestamp(TimeUnit::Second, tz.clone()), + )) } - _ => dt, + _ => expr, }; Ok(Expr::Cast(Cast::new(Box::new(expr), dt))) diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 506c64349173..f28c9589246e 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -597,20 +597,14 @@ fn select_neg_filter() { fn select_compound_filter() { let sql = "SELECT id, first_name, last_name \ FROM person WHERE state = 'CO' AND age >= 21 AND age <= 65"; - let expected = "Projection: person.id, person.first_name, person.last_name\ - \n Filter: person.state = Utf8(\"CO\") AND person.age >= Int64(21) AND person.age <= Int64(65)\ - \n TableScan: person"; + let expected = "Projection: person.id, person.first_name, person.last_name\n Filter: person.state = Utf8(\"CO\") AND person.age >= Int64(21) AND person.age <= Int64(65)\n TableScan: person"; quick_test(sql, expected); } #[test] fn test_timestamp_filter() { let sql = "SELECT state FROM person WHERE birth_date < CAST (158412331400600000 as timestamp)"; - - let expected = "Projection: person.state\ - \n Filter: person.birth_date < CAST(Int64(158412331400600000) AS Timestamp(Second, None))\ - \n TableScan: person"; - + let expected = "Projection: person.state\n Filter: person.birth_date < CAST(CAST(Int64(158412331400600000) AS Timestamp(Second, None)) AS Timestamp(Nanosecond, None))\n TableScan: person"; quick_test(sql, expected); } diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index 22400fe1771d..d5f16a88f8ff 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1788,20 +1788,45 @@ SELECT TIMESTAMPTZ '2020-01-01 00:00:00Z' = TIMESTAMP '2020-01-01' ---- true -# verify to_timestamp edge cases to be in sync with postgresql -query PPPPPPPP -SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1) +# verify timestamp cast with integer input +query PPPPPP +SELECT to_timestamp(null), to_timestamp(0), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1) ---- -NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 +NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 -# verify timestamp cast from i64 is in sync with to_timestamp(i64) -query PPPPPPPP -SELECT null::timestamp, -62125747200::timestamp, 0::timestamp, 1926632005177::timestamp, 1926632005::timestamp, 1::timestamp, -1::timestamp, (0-1)::timestamp +# verify timestamp cast with integer input timestamp literal syntax +query PPPPPP +SELECT null::timestamp, 0::timestamp, 1926632005::timestamp, 1::timestamp, -1::timestamp, (0-1)::timestamp ---- -NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 +NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 -# verify timestamp cast from i64 is in sync with to_timestamp(i64) using CAST syntax -query PPPPPPPP -SELECT cast(null as timestamp), cast(-62125747200 as timestamp), cast(0 as timestamp), cast(1926632005177 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp) +# verify timestamp cast with integer input timestamp literal syntax using CAST syntax +query PPPPPP +SELECT cast(null as timestamp), cast(0 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp) ---- -NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 +NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 + +# verify timestamp output types +query TTT +SELECT arrow_typeof(to_timestamp(1)), arrow_typeof(to_timestamp(null)), arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) +---- +Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) + +# verify timestamp output types using timestamp literal syntax +query TTT +SELECT arrow_typeof(1::timestamp), arrow_typeof(null::timestamp), arrow_typeof('2023-01-10 12:34:56.000'::timestamp) +---- +Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) + +# verify timestamp output types using CAST syntax +query TTT +SELECT arrow_typeof(cast(1 as timestamp)), arrow_typeof(cast(null as timestamp)), arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp)) +---- +Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) + + +# verify extreme values (expects default precision to be microsecond instead of nanoseconds. Work pending) +#query PPPPPPPP +#SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) +#---- +#0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 From 7fe27df6757cec02d7a65649b5543f04b97b4740 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 Nov 2023 14:41:26 -0800 Subject: [PATCH 7/9] add descr --- datafusion/sqllogictest/test_files/timestamps.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index d5f16a88f8ff..6774421a3a9e 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1825,7 +1825,7 @@ SELECT arrow_typeof(cast(1 as timestamp)), arrow_typeof(cast(null as timestamp)) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) -# verify extreme values (expects default precision to be microsecond instead of nanoseconds. Work pending) +# known issues. verify extreme values (expects default precision to be microsecond instead of nanoseconds. Work pending) #query PPPPPPPP #SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) #---- From ccb276d961e959702585b80ab8eb8d886b61a129 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 Nov 2023 15:08:57 -0800 Subject: [PATCH 8/9] add descr --- datafusion/sqllogictest/test_files/timestamps.slt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index 6774421a3a9e..8fbae638f966 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1825,7 +1825,8 @@ SELECT arrow_typeof(cast(1 as timestamp)), arrow_typeof(cast(null as timestamp)) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) -# known issues. verify extreme values (expects default precision to be microsecond instead of nanoseconds. Work pending) +# known issues. currently overflows (expects default precision to be microsecond instead of nanoseconds. Work pending) +#verify extreme values #query PPPPPPPP #SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) #---- From bfea80c8c416922ba67c0fba59ca59c86a01a10c Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 22 Nov 2023 14:45:26 -0800 Subject: [PATCH 9/9] minors --- datafusion/sql/src/expr/mod.rs | 3 +- datafusion/sql/tests/sql_integration.rs | 8 +++- .../sqllogictest/test_files/timestamps.slt | 48 ++++++++++--------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 1307d5766d61..25fe6b6633c2 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -230,7 +230,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let expr = self.sql_expr_to_logical_expr(*expr, schema, planner_context)?; - // int/floats input should be treated as seconds rather as nanoseconds + // numeric constants are treated as seconds (rather as nanoseconds) + // to align with postgres / duckdb semantics let expr = match &dt { DataType::Timestamp(TimeUnit::Nanosecond, tz) if expr.get_type(schema)? == DataType::Int64 => diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index f28c9589246e..d5b06bcf815f 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -597,14 +597,18 @@ fn select_neg_filter() { fn select_compound_filter() { let sql = "SELECT id, first_name, last_name \ FROM person WHERE state = 'CO' AND age >= 21 AND age <= 65"; - let expected = "Projection: person.id, person.first_name, person.last_name\n Filter: person.state = Utf8(\"CO\") AND person.age >= Int64(21) AND person.age <= Int64(65)\n TableScan: person"; + let expected = "Projection: person.id, person.first_name, person.last_name\ + \n Filter: person.state = Utf8(\"CO\") AND person.age >= Int64(21) AND person.age <= Int64(65)\ + \n TableScan: person"; quick_test(sql, expected); } #[test] fn test_timestamp_filter() { let sql = "SELECT state FROM person WHERE birth_date < CAST (158412331400600000 as timestamp)"; - let expected = "Projection: person.state\n Filter: person.birth_date < CAST(CAST(Int64(158412331400600000) AS Timestamp(Second, None)) AS Timestamp(Nanosecond, None))\n TableScan: person"; + let expected = "Projection: person.state\ + \n Filter: person.birth_date < CAST(CAST(Int64(158412331400600000) AS Timestamp(Second, None)) AS Timestamp(Nanosecond, None))\ + \n TableScan: person"; quick_test(sql, expected); } diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index 8fbae638f966..3830d8f86812 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -1794,17 +1794,23 @@ SELECT to_timestamp(null), to_timestamp(0), to_timestamp(1926632005), to_timesta ---- NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 -# verify timestamp cast with integer input timestamp literal syntax -query PPPPPP -SELECT null::timestamp, 0::timestamp, 1926632005::timestamp, 1::timestamp, -1::timestamp, (0-1)::timestamp ----- -NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 - -# verify timestamp cast with integer input timestamp literal syntax using CAST syntax -query PPPPPP -SELECT cast(null as timestamp), cast(0 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp) ----- -NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59 +# verify timestamp syntax stlyes are consistent +query BBBBBBBBBBBBB +SELECT to_timestamp(null) is null as c1, + null::timestamp is null as c2, + cast(null as timestamp) is null as c3, + to_timestamp(0) = 0::timestamp as c4, + to_timestamp(1926632005) = 1926632005::timestamp as c5, + to_timestamp(1) = 1::timestamp as c6, + to_timestamp(-1) = -1::timestamp as c7, + to_timestamp(0-1) = (0-1)::timestamp as c8, + to_timestamp(0) = cast(0 as timestamp) as c9, + to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10, + to_timestamp(1) = cast(1 as timestamp) as c11, + to_timestamp(-1) = cast(-1 as timestamp) as c12, + to_timestamp(0-1) = cast(0-1 as timestamp) as c13 +---- +true true true true true true true true true true true true true # verify timestamp output types query TTT @@ -1813,17 +1819,15 @@ SELECT arrow_typeof(to_timestamp(1)), arrow_typeof(to_timestamp(null)), arrow_ty Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) # verify timestamp output types using timestamp literal syntax -query TTT -SELECT arrow_typeof(1::timestamp), arrow_typeof(null::timestamp), arrow_typeof('2023-01-10 12:34:56.000'::timestamp) ----- -Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) - -# verify timestamp output types using CAST syntax -query TTT -SELECT arrow_typeof(cast(1 as timestamp)), arrow_typeof(cast(null as timestamp)), arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp)) ----- -Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) - +query BBBBBB +SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1, + arrow_typeof(to_timestamp(null)) = arrow_typeof(null::timestamp) as c2, + arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof('2023-01-10 12:34:56.000'::timestamp) as c3, + arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4, + arrow_typeof(to_timestamp(null)) = arrow_typeof(cast(null as timestamp)) as c5, + arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp)) as c6 +---- +true true true true true true # known issues. currently overflows (expects default precision to be microsecond instead of nanoseconds. Work pending) #verify extreme values