Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Hotfix][SQL Transform] Fix cast to timestamp, date, time bug #5812

Merged
merged 3 commits into from
Nov 27, 2023
Merged

[Hotfix][SQL Transform] Fix cast to timestamp, date, time bug #5812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8823e04
Fix cast to timestamp, date, time bug
EricJoy2048 Nov 8, 2023
60978b9
update test config add decimal scala
EricJoy2048 Nov 10, 2023
ced2b57
update error message
EricJoy2048 Nov 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .../main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertRuleParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ private List<AssertFieldRule.AssertRule> assembleFieldValueRules(
}

private SeaTunnelDataType<?> getFieldType(String fieldTypeStr) {
if (fieldTypeStr.toLowerCase().startsWith("decimal(")) {
String lengthAndScale =
fieldTypeStr.toLowerCase().replace("decimal(", "").replace(")", "");
String[] split = lengthAndScale.split(",");
return new DecimalType(Integer.valueOf(split[0]), Integer.valueOf(split[1]));
}
return TYPES.get(fieldTypeStr.toLowerCase());
}

Expand All @@ -110,6 +116,5 @@ private SeaTunnelDataType<?> getFieldType(String fieldTypeStr) {
TYPES.put("datetime", LocalTimeType.LOCAL_DATE_TIME_TYPE);
TYPES.put("date", LocalTimeType.LOCAL_DATE_TYPE);
TYPES.put("time", LocalTimeType.LOCAL_TIME_TYPE);
TYPES.put("decimal", new DecimalType(38, 18));
}
}
4 changes: 2 additions & 2 deletions ...or-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/resources/jdbc_hive_source_and_assert.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,12 @@ sink{
},
{
field_name = hive_e2e_source_table.decimal_column
field_type = decimal
field_type = "decimal(10,2)"
field_value = [{equals_to = 42.12}]
},
{
field_name = hive_e2e_source_table.numeric_column
field_type = decimal
field_type = "decimal(10,2)"
field_value = [{equals_to = 42.12}]
},
]
Expand Down
2 changes: 1 addition & 1 deletion ...forms-v2-e2e-part-2/src/test/resources/json_path_transform/json_path_basic_type_test.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ sink {
},
{
field_name = c1_decimal
field_type = decimal
field_type = "decimal(4,2)"
field_value = [
{
rule_type = NOT_NULL
Expand Down
42 changes: 40 additions & 2 deletions ...-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_system.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,13 @@ source {
c1 = "string"
c2 = "timestamp"
c3 = "string"
c4 = "bigint"
c5 = "int"
c6 = "int"
}
}
rows = [
{fields = [1, "Joy Ding", "12.4", "2012-12-21T12:34:56", null], kind = INSERT}
{fields = [1, "Joy Ding", "12.4", "2012-12-21T12:34:56", null, 1687747869032, 20230625, 235109], kind = INSERT}
]
}
}
Expand All @@ -46,7 +49,7 @@ transform {
Sql {
source_table_name = "fake"
result_table_name = "fake1"
query = "select cast(id as STRING) as id, cast(id as INT) as id2, cast(id as DOUBLE) as id3 , cast(c1 as double) as c1_1, cast(c1 as DECIMAL(10,2)) as c1_2, cast(c2 as DATE) as c2_1, coalesce(c3,'Unknown') c3_1, ifnull(c3,'Unknown') c3_2, ifnull(nullif(name,'Joy Ding'),'NULL') name1, nullif(name,'Joy Ding_') name2 from fake"
query = "select cast(id as STRING) as id, cast(id as INT) as id2, cast(id as DOUBLE) as id3 , cast(c1 as double) as c1_1, cast(c1 as DECIMAL(10,2)) as c1_2, cast(c2 as DATE) as c2_1, coalesce(c3,'Unknown') c3_1, ifnull(c3,'Unknown') c3_2, ifnull(nullif(name,'Joy Ding'),'NULL') name1, nullif(name,'Joy Ding_') name2, cast(c4 as timestamp) as c4_1, cast(c4 as decimal(17,4)) as c4_2, cast(c5 as date) as c5, cast(c6 as time) as c6 from fake"
}
}

Expand Down Expand Up @@ -86,6 +89,13 @@ sink {
{equals_to = 12.4}
]
},
{
field_name = "c1_2"
field_type = "decimal(10,2)"
field_value = [
{equals_to = "12.40"}
]
},
{
field_name = "c2_1"
field_type = "date"
Expand Down Expand Up @@ -120,6 +130,34 @@ sink {
field_value = [
{equals_to = "Joy Ding"}
]
},
{
field_name = "c4_1"
field_type = "timestamp"
field_value = [
{equals_to = "2023-06-26T02:51:09.032"}
]
},
{
field_name = "c4_2"
field_type = "decimal(17,4)"
field_value = [
{equals_to = "1687747869032.0000"}
]
},
{
field_name = "c5"
field_type = "date"
field_value = [
{equals_to = "2023-06-25"}
]
},
{
field_name = "c6"
field_type = "time"
field_value = [
{equals_to = "23:51:09"}
]
}
]
}
Expand Down
34 changes: 32 additions & 2 deletions ...ms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.util.List;

public class SystemFunction {
Expand Down Expand Up @@ -96,23 +98,51 @@ public static Object castAs(List<Object> args) {
if (v1 instanceof LocalTime) {
return LocalDateTime.of(LocalDate.now(), (LocalTime) v1);
}
if (v1 instanceof Long) {
Instant instant = Instant.ofEpochMilli(((Long) v1).longValue());
ZoneId zone = ZoneId.systemDefault();
return LocalDateTime.ofInstant(instant, zone);
}
throw new TransformException(
CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
String.format("Unsupported CAST AS type: %s", v2));
case "DATE":
if (v1 instanceof LocalDateTime) {
return ((LocalDateTime) v1).toLocalDate();
}
if (v1 instanceof LocalDate) {
return v1;
}
if (v1 instanceof Integer) {
int dateValue = ((Integer) v1).intValue();
int year = dateValue / 10000;
int month = (dateValue / 100) % 100;
int day = dateValue % 100;
return LocalDate.of(year, month, day);
}
throw new TransformException(
CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
String.format("Unsupported CAST AS type: %s", v2));
case "TIME":
if (v1 instanceof LocalDateTime) {
return ((LocalDateTime) v1).toLocalTime();
}
if (v1 instanceof LocalDate) {
return LocalDateTime.of((LocalDate) v1, LocalTime.of(0, 0, 0));
return LocalTime.of(0, 0, 0);
}
if (v1 instanceof LocalTime) {
return LocalDateTime.of(LocalDate.now(), (LocalTime) v1);
return v1;
}
if (v1 instanceof Integer) {
int intTime = ((Integer) v1).intValue();
int hour = intTime / 10000;
int minute = (intTime / 100) % 100;
int second = intTime % 100;
return LocalTime.of(hour, minute, second);
}
throw new TransformException(
CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION,
String.format("Unsupported CAST AS type: %s", v2));
case "DECIMAL":
BigDecimal bigDecimal = new BigDecimal(v1.toString());
Integer scale = (Integer) args.get(3);
Expand Down