From 828a296f102eac47d1396aaffe18ff82a336b2bd Mon Sep 17 00:00:00 2001 From: Laglangyue <35491928+laglangyue@users.noreply.github.com> Date: Sun, 12 Feb 2023 21:58:49 +0800 Subject: [PATCH 1/4] [Hotfix][] fix split path failed in win10 (#4109) * [e2e] fix split path failed in win10 * [e2e] fix style --- .../org/apache/seatunnel/e2e/common/util/ContainerUtil.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java index f33e882d8ec..ff3f5bcf1f3 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java @@ -25,6 +25,8 @@ import org.apache.seatunnel.api.table.factory.FactoryException; import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.commons.lang3.StringUtils; + import org.junit.jupiter.api.Assertions; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.MountableFile; @@ -123,7 +125,7 @@ public static void copySeaTunnelStarterToContainer( String seatunnelHomeInContainer) { // solve the problem of multi modules such as // seatunnel-flink-starter/seatunnel-flink-13-starter - final String[] splits = startModuleName.split(File.separator); + final String[] splits = StringUtils.split(startModuleName, File.separator); final String startJarName = splits[splits.length - 1] + ".jar"; // copy starter final String startJarPath = From dc182d07589e5a156c1369c1a9b67bef8a9160ec Mon Sep 17 00:00:00 2001 From: Hisoka Date: Mon, 13 Feb 2023 14:17:14 +0800 Subject: [PATCH 2/4] [Hotfix][Zeta][Client] Fix Client Not Throw Exception When Error Msg Is Null (#4107) * [Improve] [Zeta] [Client] Fix Client Not Throw Exception When Error Msg Is Null * [Improve][Zeta][Client] Reformat code with spotless --------- Co-authored-by: tyrantlucifer --- .../starter/seatunnel/command/ClientExecuteCommand.java | 3 ++- .../seatunnel/engine/client/job/ClientJobProxy.java | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java index b94c6d9bd04..245bf6bfe9c 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java @@ -211,7 +211,8 @@ private String creatRandomClusterName(String namePrefix) { private void shutdownHook(ClientJobProxy clientJobProxy) { if (clientCommandArgs.isCloseJob()) { - if (jobStatus == null || !jobStatus.isEndState()) { + if (clientJobProxy.getJobResultCache() == null + && (jobStatus == null || !jobStatus.isEndState())) { log.warn("Task will be closed due to client shutdown."); clientJobProxy.cancelJob(); } diff --git a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java index 3bfbbdd5c52..f42e856e1a3 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java +++ b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java @@ -43,6 +43,7 @@ public class ClientJobProxy implements Job { private static final ILogger LOGGER = Logger.getLogger(ClientJobProxy.class); private final SeaTunnelHazelcastClient seaTunnelHazelcastClient; private final JobImmutableInformation jobImmutableInformation; + private JobResult jobResult; public ClientJobProxy( @NonNull SeaTunnelHazelcastClient seaTunnelHazelcastClient, @@ -81,7 +82,6 @@ private void submitJob() { */ @Override public JobStatus waitForJobComplete() { - JobResult jobResult; try { jobResult = RetryUtils.retryWithException( @@ -113,12 +113,17 @@ public JobStatus waitForJobComplete() { jobImmutableInformation.getJobConfig().getName(), jobImmutableInformation.getJobId(), jobResult.getStatus())); - if (StringUtils.isNotEmpty(jobResult.getError())) { + if (StringUtils.isNotEmpty(jobResult.getError()) + || jobResult.getStatus().equals(JobStatus.FAILED)) { throw new SeaTunnelEngineException(jobResult.getError()); } return jobResult.getStatus(); } + public JobResult getJobResultCache() { + return jobResult; + } + @Override public PassiveCompletableFuture doWaitForJobComplete() { return new PassiveCompletableFuture<>( From 096724a60096783008cc5bad5635d8dd82414086 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 14 Feb 2023 19:34:58 +0800 Subject: [PATCH 3/4] [Feature][Transform] Add SimpleSQL transform plugin (#4148) add feature to release-note --- LICENSE | 4 + docs/en/faq.md | 2 - docs/en/transform-v2/simple-sql-functions.md | 893 ++++++++++++++++++ docs/en/transform-v2/simple-sql.md | 100 ++ pom.xml | 1 + release-note.md | 1 + .../api/transform/SeaTunnelTransform.java | 6 + .../assertion/excecutor/AssertExecutor.java | 33 + .../assertion/rule/AssertFieldRule.java | 1 + .../assertion/rule/AssertRuleParser.java | 9 + .../assertion/sink/AssertConfig.java | 2 + .../e2e/transform/TestSimpleSQLIT.java | 313 ++++++ .../binary_expression.conf | 104 ++ .../simple_sql_functions/filter_and_or.conf | 73 ++ .../filter_equals_to.conf | 72 ++ .../simple_sql_functions/filter_function.conf | 72 ++ .../filter_great_than.conf | 72 ++ .../filter_great_than_equals.conf | 72 ++ .../simple_sql_functions/filter_in.conf | 73 ++ .../filter_is_not_null.conf | 72 ++ .../simple_sql_functions/filter_is_null.conf | 72 ++ .../filter_minor_than.conf | 72 ++ .../filter_minor_than_equals.conf | 72 ++ .../filter_not_equals_to.conf | 72 ++ .../simple_sql_functions/filter_not_in.conf | 73 ++ .../simple_sql_functions/func_abs.conf | 76 ++ .../simple_sql_functions/func_acos.conf | 69 ++ .../simple_sql_functions/func_ascii.conf | 84 ++ .../simple_sql_functions/func_asin.conf | 69 ++ .../simple_sql_functions/func_atan.conf | 69 ++ .../simple_sql_functions/func_bit_length.conf | 84 ++ .../simple_sql_functions/func_cast.conf | 104 ++ .../simple_sql_functions/func_ceil.conf | 76 ++ .../func_char_length.conf | 84 ++ .../simple_sql_functions/func_chr.conf | 84 ++ .../simple_sql_functions/func_coalesce.conf | 69 ++ .../simple_sql_functions/func_concat.conf | 76 ++ .../simple_sql_functions/func_concat_ws.conf | 76 ++ .../simple_sql_functions/func_cos.conf | 69 ++ .../simple_sql_functions/func_cosh.conf | 69 ++ .../simple_sql_functions/func_cot.conf | 69 ++ .../func_current_date.conf | 76 ++ .../func_current_time.conf | 76 ++ .../func_current_timestamp.conf | 83 ++ .../simple_sql_functions/func_date_trunc.conf | 104 ++ .../simple_sql_functions/func_dateadd.conf | 111 +++ .../simple_sql_functions/func_datediff.conf | 112 +++ .../func_day_of_month.conf | 85 ++ .../func_day_of_week.conf | 88 ++ .../func_day_of_year.conf | 85 ++ .../simple_sql_functions/func_dayname.conf | 116 +++ .../simple_sql_functions/func_exp.conf | 69 ++ .../simple_sql_functions/func_extract.conf | 127 +++ .../simple_sql_functions/func_floor.conf | 76 ++ .../func_formatdatetime.conf | 83 ++ .../simple_sql_functions/func_hextoraw.conf | 76 ++ .../simple_sql_functions/func_hour.conf | 69 ++ .../simple_sql_functions/func_ifnull.conf | 69 ++ .../simple_sql_functions/func_insert.conf | 76 ++ .../simple_sql_functions/func_left.conf | 69 ++ .../simple_sql_functions/func_ln.conf | 76 ++ .../simple_sql_functions/func_locate.conf | 83 ++ .../simple_sql_functions/func_log.conf | 76 ++ .../simple_sql_functions/func_log10.conf | 76 ++ .../simple_sql_functions/func_lower.conf | 76 ++ .../simple_sql_functions/func_lpad.conf | 76 ++ .../simple_sql_functions/func_ltrim.conf | 76 ++ .../simple_sql_functions/func_minute.conf | 69 ++ .../simple_sql_functions/func_mod.conf | 83 ++ .../simple_sql_functions/func_month.conf | 69 ++ .../simple_sql_functions/func_monthname.conf | 69 ++ .../simple_sql_functions/func_nullif.conf | 76 ++ .../func_octet_length.conf | 76 ++ .../func_parsedatetime.conf | 78 ++ .../simple_sql_functions/func_pi.conf | 69 ++ .../simple_sql_functions/func_power.conf | 76 ++ .../simple_sql_functions/func_quarter.conf | 69 ++ .../simple_sql_functions/func_radians.conf | 69 ++ .../simple_sql_functions/func_rand.conf | 73 ++ .../simple_sql_functions/func_rawtohex.conf | 76 ++ .../func_regexp_like.conf | 76 ++ .../func_regexp_replace.conf | 76 ++ .../func_regexp_substr.conf | 76 ++ .../simple_sql_functions/func_repeat.conf | 69 ++ .../simple_sql_functions/func_replace.conf | 76 ++ .../simple_sql_functions/func_right.conf | 69 ++ .../simple_sql_functions/func_round.conf | 76 ++ .../simple_sql_functions/func_rpad.conf | 76 ++ .../simple_sql_functions/func_rtrim.conf | 76 ++ .../simple_sql_functions/func_second.conf | 69 ++ .../simple_sql_functions/func_sign.conf | 92 ++ .../simple_sql_functions/func_sin.conf | 69 ++ .../simple_sql_functions/func_sinh.conf | 69 ++ .../simple_sql_functions/func_soundex.conf | 69 ++ .../simple_sql_functions/func_space.conf | 69 ++ .../simple_sql_functions/func_sqrt.conf | 76 ++ .../simple_sql_functions/func_substr.conf | 76 ++ .../simple_sql_functions/func_tan.conf | 69 ++ .../simple_sql_functions/func_tanh.conf | 69 ++ .../simple_sql_functions/func_to_char.conf | 69 ++ .../simple_sql_functions/func_translate.conf | 69 ++ .../simple_sql_functions/func_trim.conf | 76 ++ .../simple_sql_functions/func_trunc.conf | 76 ++ .../simple_sql_functions/func_upper.conf | 76 ++ .../simple_sql_functions/func_week.conf | 69 ++ .../simple_sql_functions/func_year.conf | 69 ++ .../test/resources/simple_sql_transform.conf | 112 +++ .../task/flow/TransformFlowLifeCycle.java | 33 + seatunnel-transforms-v2/pom.xml | 5 + .../transform/SimpleSQLTransform.java | 106 +++ .../transform/SimpleSQLTransformFactory.java | 39 + .../common/AbstractSeaTunnelTransform.java | 8 +- .../exception/TransformException.java | 36 + .../transform/sqlengine/SimpleSQLEngine.java | 31 + .../sqlengine/SimpleSQLEngineFactory.java | 40 + .../sqlengine/zeta/ZetaSQLFilter.java | 330 +++++++ .../sqlengine/zeta/ZetaSQLFunction.java | 545 +++++++++++ .../transform/sqlengine/zeta/ZetaSQLType.java | 328 +++++++ .../sqlengine/zeta/ZetaSimpleSQLEngine.java | 202 ++++ .../zeta/functions/DateTimeFunction.java | 544 +++++++++++ .../zeta/functions/NumericFunction.java | 448 +++++++++ .../zeta/functions/StringFunction.java | 667 +++++++++++++ .../zeta/functions/SystemFunction.java | 125 +++ tools/dependencies/known-dependencies.txt | 3 +- 124 files changed, 12200 insertions(+), 7 deletions(-) create mode 100644 docs/en/transform-v2/simple-sql-functions.md create mode 100644 docs/en/transform-v2/simple-sql.md create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/exception/TransformException.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngine.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngineFactory.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/DateTimeFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/StringFunction.java create mode 100644 seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java diff --git a/LICENSE b/LICENSE index 8d74dcda44f..af47dc7d901 100644 --- a/LICENSE +++ b/LICENSE @@ -236,3 +236,7 @@ seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/StandaloneCheckpointIDCounter.java from https://github.com/apache/flink seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/metrics from https://github.com/hazelcast/hazelcast seatunnel-api/src/main/java/org/apache/seatunnel/api/common/metrics from https://github.com/hazelcast/hazelcast +seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java from https://github.com/JSQLParser/JSqlParser +seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java from https://github.com/JSQLParser/JSqlParser +seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java from https://github.com/JSQLParser/JSqlParser +seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java from https://github.com/JSQLParser/JSqlParser \ No newline at end of file diff --git a/docs/en/faq.md b/docs/en/faq.md index f68fb61c806..b3f698e19b2 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -303,8 +303,6 @@ http://spark.apache.org/docs/latest/configuration.html#configuring-logging https://medium.com/@iacomini.riccardo/spark-logging-configuration-in-yarn-faf5ba5fdb01 -https://stackoverflow.com/questions/27781187/how-to-stop-info-messages-displaying-on-spark-console - ## Error when writing to ClickHouse: ClassCastException In SeaTunnel, the data type will not be actively converted. After the Input reads the data, the corresponding diff --git a/docs/en/transform-v2/simple-sql-functions.md b/docs/en/transform-v2/simple-sql-functions.md new file mode 100644 index 00000000000..bdb783fe67d --- /dev/null +++ b/docs/en/transform-v2/simple-sql-functions.md @@ -0,0 +1,893 @@ +# SimpleSQL Functions + +> The Functions of SimpleSQL transform plugin + +## String Functions + +### ASCII + +```ASCII(string)``` + +Returns the ```ASCII``` value of the first character in the string. This method returns an int. + +Example: + +ASCII('Hi') + +### BIT_LENGTH + +```BIT_LENGTH(bytes)``` + +Returns the number of bits in a binary string. This method returns a long. + +Example: + +BIT_LENGTH(NAME) + +### CHAR_LENGTH / LENGTH + +```CHAR_LENGTH | LENGTH (string)``` + +Returns the number of characters in a character string. This method returns a long. + +Example: + +CHAR_LENGTH(NAME) + +### OCTET_LENGTH + +```OCTET_LENGTH(bytes)``` + +Returns the number of bytes in a binary string. This method returns a long. + +Example: + +OCTET_LENGTH(NAME) + +### CHAR / CHR + +```CHAR | CHR (int)``` + +Returns the character that represents the ASCII value. This method returns a string. + +Example: + +CHAR(65) + +### CONCAT + +```CONCAT(string, string[, string ...] )``` + +Combines strings. Unlike with the operator ```||```, **NULL** parameters are ignored, and do not cause the result to become **NULL**. If all parameters are NULL the result is an empty string. This method returns a string. + +Example: + +CONCAT(NAME, '_') + +### CONCAT_WS + +```CONCAT_WS(separatorString, string, string[, string ...] )``` + +Combines strings with separator. If separator is **NULL** it is treated like an empty string. Other **NULL** parameters are ignored. Remaining **non-NULL** parameters, if any, are concatenated with the specified separator. If there are no remaining parameters the result is an empty string. This method returns a string. + +Example: + +CONCAT_WS(',', NAME, '_') + +### HEXTORAW + +```HEXTORAW(string)``` + +Converts a hex representation of a string to a string. 4 hex characters per string character are used. + +Example: + +HEXTORAW(DATA) + +### RAWTOHEX + +```RAWTOHEX(string)``` + +```RAWTOHEX(bytes)``` + +Converts a string or bytes to the hex representation. 4 hex characters per string character are used. This method returns a string. + +Example: + +RAWTOHEX(DATA) + +### INSERT + +```INSERT(originalString, startInt, lengthInt, addString)``` + +Inserts a additional string into the original string at a specified start position. The length specifies the number of characters that are removed at the start position in the original string. This method returns a string. + +Example: + +INSERT(NAME, 1, 1, ' ') + +### LOWER / LCASE + +```LOWER | LCASE (string)``` + +Converts a string to lowercase. + +Example: + +LOWER(NAME) + +### UPPER / UCASE + +```UPPER | UCASE (string)``` + +Converts a string to uppercase. + +Example: + +UPPER(NAME) + +### LEFT + +```LEFT(string, int)``` + +Returns the leftmost number of characters. + +Example: + +LEFT(NAME, 3) + +### RIGHT + +```RIGHT(string, int)``` + +Returns the rightmost number of characters. + +Example: + +RIGHT(NAME, 3) + +### LOCATE / INSTR / POSITION + +```LOCATE(searchString, string[, startInit])``` + +```INSTR(string, searchString[, startInit])``` + +```POSITION(searchString, string)``` + +Returns the location of a search string in a string. If a start position is used, the characters before it are ignored. If position is negative, the rightmost location is returned. 0 is returned if the search string is not found. Please note this function is case sensitive, even if the parameters are not. + +Example: + +LOCATE('.', NAME) + +### LPAD + +```LPAD(string ,int[, string])``` + +Left pad the string to the specified length. If the length is shorter than the string, it will be truncated at the end. If the padding string is not set, spaces will be used. + +Example: + +LPAD(AMOUNT, 10, '*') + +### RPAD + +```RPAD(string, int[, string])``` + +Right pad the string to the specified length. If the length is shorter than the string, it will be truncated. If the padding string is not set, spaces will be used. + +Example: + +RPAD(TEXT, 10, '-') + +### LTRIM + +```LTRIM(string[, characterToTrimString])``` + +Removes all leading spaces or other specified characters from a string. + +This function is deprecated, use TRIM instead of it. + +Example: + +LTRIM(NAME) + +### RTRIM + +```RTRIM(string[, characterToTrimString])``` + +Removes all trailing spaces or other specified characters from a string. + +This function is deprecated, use TRIM instead of it. + +Example: + +RTRIM(NAME) + +### TRIM + +```TRIM(string[, characterToTrimString])``` + +Removes all leading spaces or other specified characters from a string. + +This function is deprecated, use TRIM instead of it. + +Example: + +LTRIM(NAME) + +### REGEXP_REPLACE + +```REGEXP_REPLACE(inputString, regexString, replacementString[, flagsString])``` + +Replaces each substring that matches a regular expression. For details, see the Java String.replaceAll() method. If any parameter is null (except optional flagsString parameter), the result is null. + +Flags values are limited to 'i', 'c', 'n', 'm'. Other symbols cause exception. Multiple symbols could be used in one flagsString parameter (like 'im'). Later flags override first ones, for example 'ic' is equivalent to case sensitive matching 'c'. + +'i' enables case insensitive matching (Pattern.CASE_INSENSITIVE) + +'c' disables case insensitive matching (Pattern.CASE_INSENSITIVE) + +'n' allows the period to match the newline character (Pattern.DOTALL) + +'m' enables multiline mode (Pattern.MULTILINE) + +Example: + +REGEXP_REPLACE('Hello World', ' +', ' ') +REGEXP_REPLACE('Hello WWWWorld', 'w+', 'W', 'i') + +### REGEXP_LIKE + +```REGEXP_LIKE(inputString, regexString[, flagsString])``` + +Matches string to a regular expression. For details, see the Java Matcher.find() method. If any parameter is null (except optional flagsString parameter), the result is null. + +Flags values are limited to 'i', 'c', 'n', 'm'. Other symbols cause exception. Multiple symbols could be used in one flagsString parameter (like 'im'). Later flags override first ones, for example 'ic' is equivalent to case sensitive matching 'c'. + +'i' enables case insensitive matching (Pattern.CASE_INSENSITIVE) + +'c' disables case insensitive matching (Pattern.CASE_INSENSITIVE) + +'n' allows the period to match the newline character (Pattern.DOTALL) + +'m' enables multiline mode (Pattern.MULTILINE) + +Example: + +REGEXP_LIKE('Hello World', '[A-Z ]*', 'i') + +### REGEXP_SUBSTR + +```REGEXP_SUBSTR(inputString, regexString[, positionInt, occurrenceInt, flagsString, groupInt])``` + +Matches string to a regular expression and returns the matched substring. For details, see the java.util.regex.Pattern and related functionality. + +The parameter position specifies where in inputString the match should start. Occurrence indicates which occurrence of pattern in inputString to search for. + +Flags values are limited to 'i', 'c', 'n', 'm'. Other symbols cause exception. Multiple symbols could be used in one flagsString parameter (like 'im'). Later flags override first ones, for example 'ic' is equivalent to case sensitive matching 'c'. + +'i' enables case insensitive matching (Pattern.CASE_INSENSITIVE) + +'c' disables case insensitive matching (Pattern.CASE_INSENSITIVE) + +'n' allows the period to match the newline character (Pattern.DOTALL) + +'m' enables multiline mode (Pattern.MULTILINE) + +If the pattern has groups, the group parameter can be used to specify which group to return. + +Example: + +REGEXP_SUBSTR('2020-10-01', '\d{4}') +REGEXP_SUBSTR('2020-10-01', '(\d{4})-(\d{2})-(\d{2})', 1, 1, NULL, 2) + +### REPEAT + +```REPEAT(string, int)``` + +Returns a string repeated some number of times. + +Example: + +REPEAT(NAME || ' ', 10) + +### REPLACE + +```REPLACE(string, searchString[, replacementString])``` + +Replaces all occurrences of a search string in a text with another string. If no replacement is specified, the search string is removed from the original string. If any parameter is null, the result is null. + +Example: + +REPLACE(NAME, ' ') + +### SOUNDEX + +```SOUNDEX(string)``` + +Returns a four character code representing the sound of a string. This method returns a string, or null if parameter is null. See https://en.wikipedia.org/wiki/Soundex for more information. + +Example: + +SOUNDEX(NAME) + +### SPACE + +```SPACE(int)``` + +Returns a string consisting of a number of spaces. + +Example: + +SPACE(80) + +### SUBSTRING / SUBSTR + +```SUBSTRING | SUBSTR (string, startInt[, lengthInt ])``` + +Returns a substring of a string starting at a position. If the start index is negative, then the start index is relative to the end of the string. The length is optional. + +Example: + +CALL SUBSTRING('[Hello]', 2); +CALL SUBSTRING('hour', 3, 2); + +### TO_CHAR + +```TO_CHAR(value[, formatString])``` + +Oracle-compatible TO_CHAR function that can format a timestamp, a number, or text. + +Example: + +CALL TO_CHAR(SYS_TIME, 'yyyy-MM-dd HH:mm:ss') + +### TRANSLATE + +```TRANSLATE(value, searchString, replacementString)``` + +Oracle-compatible TRANSLATE function that replaces a sequence of characters in a string with another set of characters. + +Example: + +CALL TRANSLATE('Hello world', 'eo', 'EO') + +## Numeric Functions + +### ABS + +```ABS(numeric)``` + +Returns the absolute value of a specified value. The returned value is of the same data type as the parameter. + +Note that TINYINT, SMALLINT, INT, and BIGINT data types cannot represent absolute values of their minimum negative values, because they have more negative values than positive. For example, for INT data type allowed values are from -2147483648 to 2147483647. ABS(-2147483648) should be 2147483648, but this value is not allowed for this data type. It leads to an exception. To avoid it cast argument of this function to a higher data type. + +Example: + +ABS(I) + +### ACOS + +```ACOS(numeric)``` + +Calculate the arc cosine. See also Java Math.acos. This method returns a double. + +Example: + +ACOS(D) + +### ASIN + +```ASIN(numeric)``` + +Calculate the arc sine. See also Java Math.asin. This method returns a double. + +Example: + +ASIN(D) + +### ATAN + +```ATAN(numeric)``` + +Calculate the arc tangent. See also Java Math.atan. This method returns a double. + +Example: + +ATAN(D) + +### COS + +```COS(numeric)``` + +Calculate the trigonometric cosine. See also Java Math.cos. This method returns a double. + +Example: + +COS(ANGLE) + +### COSH + +```COSH(numeric)``` + +Calculate the hyperbolic cosine. See also Java Math.cosh. This method returns a double. + +Example: + +COSH(X) + +### COT + +```COT(numeric)``` + +Calculate the trigonometric cotangent (1/TAN(ANGLE)). See also Java Math.* functions. This method returns a double. + +Example: + +COT(ANGLE) + +### SIN + +```SIN(numeric)``` + +Calculate the trigonometric sine. See also Java Math.sin. This method returns a double. + +Example: + +SIN(ANGLE) + +### SINH + +```SINH(numeric)``` + +Calculate the hyperbolic sine. See also Java Math.sinh. This method returns a double. + +Example: + +SINH(ANGLE) + +### TAN + +```TAN(numeric)``` + +Calculate the trigonometric tangent. See also Java Math.tan. This method returns a double. + +Example: + +TAN(ANGLE) + +### TANH + +```TANH(numeric)``` + +Calculate the hyperbolic tangent. See also Java Math.tanh. This method returns a double. + +Example: + +TANH(X) + +### MOD + +```MOD(dividendNumeric, divisorNumeric )``` + +The modulus expression. + +Result has the same type as divisor. Result is NULL if either of arguments is NULL. If divisor is 0, an exception is raised. Result has the same sign as dividend or is equal to 0. + +Usually arguments should have scale 0, but it isn't required by H2. + +Example: + +MOD(A, B) + +### CEIL / CEILING + +```CEIL | CEILING (numeric)``` + +Returns the smallest integer value that is greater than or equal to the argument. This method returns value of the same type as argument, but with scale set to 0 and adjusted precision, if applicable. + +Example: + +CEIL(A) + +### EXP + +```EXP(numeric)``` + +See also Java Math.exp. This method returns a double. + +Example: + +EXP(A) + +### FLOOR + +```FLOOR(numeric)``` + +Returns the largest integer value that is less than or equal to the argument. This method returns value of the same type as argument, but with scale set to 0 and adjusted precision, if applicable. + +Example: + +FLOOR(A) + +### LN + +```LN(numeric)``` + +Calculates the natural (base e) logarithm as a double value. Argument must be a positive numeric value. + +Example: + +LN(A) + +### LOG + +```LOG(baseNumeric, numeric)``` + +Calculates the logarithm with specified base as a double value. Argument and base must be positive numeric values. Base cannot be equal to 1. + +The default base is e (natural logarithm), in the PostgreSQL mode the default base is base 10. In MSSQLServer mode the optional base is specified after the argument. + +Single-argument variant of LOG function is deprecated, use LN or LOG10 instead. + +Example: + +LOG(2, A) + +### LOG10 + +```LOG10(numeric)``` + +Calculates the base 10 logarithm as a double value. Argument must be a positive numeric value. + +Example: + +LOG10(A) + +### RADIANS + +```RADIANS(numeric)``` + +See also Java Math.toRadians. This method returns a double. + +Example: + +RADIANS(A) + +### SQRT + +```SQRT(numeric)``` + +See also Java Math.sqrt. This method returns a double. + +Example: + +SQRT(A) + +### PI + +```PI()``` + +See also Java Math.PI. This method returns a double. + +Example: + +PI() + +### POWER + +```POWER(numeric, numeric)``` + +See also Java Math.pow. This method returns a double. + +Example: + +POWER(A, B) + +### RAND / RANDOM + +```RAND | RANDOM([ int ])``` + +Calling the function without parameter returns the next a pseudo random number. Calling it with an parameter seeds the session's random number generator. This method returns a double between 0 (including) and 1 (excluding). + +Example: + +RAND() + +### ROUND + +```ROUND(numeric[, digitsInt])``` + +Rounds to a number of fractional digits. This method returns value of the same type as argument, but with adjusted precision and scale, if applicable. + +Example: + +ROUND(N, 2) + +### SIGN + +```SIGN(numeric)``` + +Returns -1 if the value is smaller than 0, 0 if zero or NaN, and otherwise 1. + +Example: + +SIGN(N) + +### TRUNC + +```TRUNC | TRUNCATE(numeric[, digitsInt])``` + +When a numeric argument is specified, truncates it to a number of digits (to the next value closer to 0) and returns value of the same type as argument, but with adjusted precision and scale, if applicable. + +Example: + +TRUNC(N, 2) + +## Time and Date Functions + +### CURRENT_DATE + +```CURRENT_DATE [()]``` + +Returns the current date. + +These functions return the same value within a transaction (default) or within a command depending on database mode. + +Example: + +CURRENT_DATE + +### CURRENT_TIME + +```CURRENT_TIME [()]``` + +Returns the current time with system time zone. The actual maximum available precision depends on operating system and JVM and can be 3 (milliseconds) or higher. Higher precision is not available before Java 9. + +Example: + +CURRENT_TIME + +### CURRENT_TIMESTAMP / NOW + +```CURRENT_TIMESTAMP[()] | NOW()``` + +Returns the current timestamp with system time zone. The actual maximum available precision depends on operating system and JVM and can be 3 (milliseconds) or higher. Higher precision is not available before Java 9. + +Example: + +CURRENT_TIMESTAMP + +### DATEADD / TIMESTAMPADD + +```DATEADD| TIMESTAMPADD(dateAndTime, addIntLong, datetimeFieldString)``` + +Adds units to a date-time value. The datetimeFieldString indicates the unit. Use negative values to subtract units. addIntLong may be a long value when manipulating milliseconds, microseconds, or nanoseconds otherwise its range is restricted to int. This method returns a value with the same type as specified value if unit is compatible with this value. If specified field is a HOUR, MINUTE, SECOND, MILLISECOND, etc and value is a DATE value DATEADD returns combined TIMESTAMP. Fields DAY, MONTH, YEAR, WEEK, etc are not allowed for TIME values. + +Example: + +DATEADD(CREATED, 1, 'MONTH') + +### DATEDIFF + +```DATEDIFF(aDateAndTime, bDateAndTime, datetimeFieldString)``` + +Returns the number of crossed unit boundaries between two date-time values. This method returns a long. The datetimeField indicates the unit. + +Example: + +DATEDIFF(T1.CREATED, T2.CREATED, 'MONTH') + +### DATE_TRUNC + +```DATE_TRUNC (dateAndTime, datetimeFieldString)``` + +Truncates the specified date-time value to the specified field. + +Example: + +DATE_TRUNC(CREATED, 'DAY'); + +### DAYNAME + +```DAYNAME(dateAndTime)``` + +Returns the name of the day (in English). + +Example: + +DAYNAME(CREATED) + +### DAY_OF_MONTH + +```DAY_OF_MONTH(dateAndTime)``` + +Returns the day of the month (1-31). + +Example: + +DAY_OF_MONTH(CREATED) + +### DAY_OF_WEEK + +```DAY_OF_WEEK(dateAndTime)``` + +Returns the day of the week (1-7) (Monday-Sunday), locale-specific. + +Example: + +DAY_OF_WEEK(CREATED) + +### DAY_OF_YEAR + +```DAY_OF_YEAR(dateAndTime)``` + +Returns the day of the year (1-366). + +Example: + +DAY_OF_YEAR(CREATED) + +### EXTRACT + +```EXTRACT ( datetimeField FROM dateAndTime)``` + +Returns a value of the specific time unit from a date/time value. This method returns a numeric value with EPOCH field and an int for all other fields. + +Example: + +EXTRACT(SECOND FROM CURRENT_TIMESTAMP) + +### FORMATDATETIME + +```FORMATDATETIME (dateAndTime, formatString)``` + +Formats a date, time or timestamp as a string. The most important format characters are: y year, M month, d day, H hour, m minute, s second. For details of the format, see java.time.format.DateTimeFormatter. + +This method returns a string. + +Example: + +CALL FORMATDATETIME(CREATED, 'yyyy-MM-dd HH:mm:ss') + +### HOUR + +```HOUR(dateAndTime)``` + +Returns the hour (0-23) from a date/time value. + +Example: + +HOUR(CREATED) + +### MINUTE + +```MINUTE(dateAndTime)``` + +Returns the minute (0-59) from a date/time value. + +This function is deprecated, use EXTRACT instead of it. + +Example: + +MINUTE(CREATED) + +### MONTH + +```MONTH(dateAndTime)``` + +Returns the month (1-12) from a date/time value. + +This function is deprecated, use EXTRACT instead of it. + +Example: + +MONTH(CREATED) + +### MONTHNAME + +```MONTHNAME(dateAndTime)``` + +Returns the name of the month (in English). + +Example: + +MONTHNAME(CREATED) + +### PARSEDATETIME / TO_DATE + +```PARSEDATETIME | TO_DATE(string, formatString)``` +Parses a string and returns a TIMESTAMP WITH TIME ZONE value. The most important format characters are: y year, M month, d day, H hour, m minute, s second. For details of the format, see java.time.format.DateTimeFormatter. + +Example: + +CALL PARSEDATETIME('2021-04-08 13:34:45','yyyy-MM-dd HH:mm:ss') + +### QUARTER + +```QUARTER(dateAndTime)``` + +Returns the quarter (1-4) from a date/time value. + +Example: + +QUARTER(CREATED) + +### SECOND + +```SECOND(dateAndTime)``` + +Returns the second (0-59) from a date/time value. + +This function is deprecated, use EXTRACT instead of it. + +Example: + +SECOND(CREATED) + +### WEEK + +```WEEK(dateAndTime)``` + +Returns the week (1-53) from a date/time value. + +This function uses the current system locale. + +Example: + +WEEK(CREATED) + +### YEAR + +```YEAR(dateAndTime)``` + +Returns the year from a date/time value. + +Example: + +YEAR(CREATED) + +## System Functions + +### CAST + +```CAST(value as dataType)``` + +Converts a value to another data type. + +Supported data types: STRING | VARCHAR, INT | INTEGER, LONG | BIGINT, BYTE, FLOAT, DOUBLE, DECIMAL(p,s), TIMESTAMP, DATE, TIME + +Example: + +CONVERT(NAME AS INT) + +### COALESCE + +```COALESCE(aValue, bValue [,...])``` + +Returns the first value that is not null. + +Example: + +COALESCE(A, B, C) + +### IFNULL + +```IFNULL(aValue, bValue)``` + +Returns the first value that is not null. + +Example: + +IFNULL(A, B) + +### NULLIF + +```NULLIF(aValue, bValue)``` + +Returns NULL if 'a' is equal to 'b', otherwise 'a'. + +Example: + +NULLIF(A, B) diff --git a/docs/en/transform-v2/simple-sql.md b/docs/en/transform-v2/simple-sql.md new file mode 100644 index 00000000000..d7a527e039e --- /dev/null +++ b/docs/en/transform-v2/simple-sql.md @@ -0,0 +1,100 @@ +# SimpleSQL + +> SimpleSQL transform plugin + +## Description + +Use simple SQL to transform given input row. + +SimpleSQL transform use memory SQL engine, we can via SQL functions and ability of SQL engine to implement the transform task. + +## Options + +| name | type | required | default value | +|-------------------|--------|----------|---------------| +| source_table_name | string | yes | - | +| result_table_name | string | yes | - | +| query | string | yes | - | + +### source_table_name [string] + +The source table name, the query SQL table name must match this field. + +### query [string] + +The query SQL, it's a simple SQL supported base function and criteria operation. But the complex SQL unsupported yet, include: multi source table/rows JOIN and AGGREGATE operation and the like. + +## Example + +The data read from source is a table like this: + +| id | name | age | +|----|----------|-----| +| 1 | Joy Ding | 20 | +| 2 | May Ding | 21 | +| 3 | Kin Dom | 24 | +| 4 | Joy Dom | 22 | + +We use SQL query to transform the source data like this: + +``` +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id, concat(name, '_') as name, age+1 as age from fake where id>0" + } +} +``` + +Then the data in result table `fake1` will update to + +| id | name | age | +|----|-----------|-----| +| 1 | Joy Ding_ | 21 | +| 2 | May Ding_ | 22 | +| 3 | Kin Dom_ | 25 | +| 4 | Joy Dom_ | 23 | + +## Job Config Example + +``` +env { + job.mode = "BATCH" +} + +source { + FakeSource { + result_table_name = "fake" + row.num = 100 + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id, concat(name, '_') as name, age+1 as age from fake where id>0" + } +} + +sink { + Console { + source_table_name = "fake1" + } +} +``` + +## Changelog + +### new version + +- Add SimpleSQL Transform Connector + diff --git a/pom.xml b/pom.xml index 683b2f38ea8..a0e0efb57c4 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,7 @@ 1.9.5 1.17.3 2.29.0 + 4.5 false true diff --git a/release-note.md b/release-note.md index d0a337566e1..64e3a755b89 100644 --- a/release-note.md +++ b/release-note.md @@ -11,6 +11,7 @@ ### Transformer - [Spark] Support transform-v2 for spark (#3409) - [ALL]Add FieldMapper Transform #3781 +- [ALL]Add SimpleSQL Transform #4148 ### Connectors - [Elasticsearch] Support https protocol & compatible with opensearch - [Hbase] Add hbase sink connector #4049 diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java index 452e903f9e3..292486f7db5 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java @@ -30,6 +30,9 @@ public interface SeaTunnelTransform SeaTunnelPluginLifeCycle, SeaTunnelJobAware { + /** call it when Transformer initialed */ + default void open() {} + /** * Set the data type info of input data. * @@ -51,4 +54,7 @@ public interface SeaTunnelTransform * @return transformed data. */ T map(T row); + + /** call it when Transformer completed */ + default void close() {} } diff --git a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/excecutor/AssertExecutor.java b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/excecutor/AssertExecutor.java index 01eb69661ec..c8666cd9a55 100644 --- a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/excecutor/AssertExecutor.java +++ b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/excecutor/AssertExecutor.java @@ -27,6 +27,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -101,6 +107,33 @@ private boolean pass(Object value, AssertFieldRule.AssertRule valueRule) { && AssertFieldRule.AssertRuleType.MIN.equals(valueRule.getRuleType())) { return ((Number) value).doubleValue() >= valueRule.getRuleValue(); } + if (valueRule.getEqualTo() != null) { + if (value instanceof String) { + return value.equals(valueRule.getEqualTo()); + } + if (value instanceof Number) { + return ((Number) value).doubleValue() == Double.parseDouble(valueRule.getEqualTo()); + } + if (value instanceof Boolean) { + return value.equals(Boolean.parseBoolean(valueRule.getEqualTo())); + } + if (value instanceof LocalDateTime) { + TemporalAccessor parsedTimestamp = + DateTimeFormatter.ISO_LOCAL_DATE_TIME.parse(valueRule.getEqualTo()); + LocalTime localTime = parsedTimestamp.query(TemporalQueries.localTime()); + LocalDate localDate = parsedTimestamp.query(TemporalQueries.localDate()); + return ((LocalDateTime) value).isEqual(LocalDateTime.of(localDate, localTime)); + } + if (value instanceof LocalDate) { + DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + return ((LocalDate) value).isEqual(LocalDate.parse(valueRule.getEqualTo(), fmt)); + } + if (value instanceof LocalTime) { + DateTimeFormatter fmt = DateTimeFormatter.ofPattern("HH:mm:ss"); + return value.equals(LocalTime.parse(valueRule.getEqualTo(), fmt)); + } + return false; + } String valueStr = Objects.isNull(value) ? StringUtils.EMPTY : String.valueOf(value); if (AssertFieldRule.AssertRuleType.MAX_LENGTH.equals(valueRule.getRuleType())) { diff --git a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertFieldRule.java b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertFieldRule.java index 187228e950d..12d2e28bad2 100644 --- a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertFieldRule.java +++ b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertFieldRule.java @@ -34,6 +34,7 @@ public class AssertFieldRule implements Serializable { public static class AssertRule implements Serializable { private AssertRuleType ruleType; private Double ruleValue; + private String equalTo; } /** diff --git a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertRuleParser.java b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertRuleParser.java index 13893e72665..f479dfa5c99 100644 --- a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertRuleParser.java +++ b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/rule/AssertRuleParser.java @@ -20,6 +20,7 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.LocalTimeType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import com.google.common.collect.Maps; @@ -28,6 +29,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.apache.seatunnel.connectors.seatunnel.assertion.sink.AssertConfig.EQUALS_TO; import static org.apache.seatunnel.connectors.seatunnel.assertion.sink.AssertConfig.FIELD_NAME; import static org.apache.seatunnel.connectors.seatunnel.assertion.sink.AssertConfig.FIELD_TYPE; import static org.apache.seatunnel.connectors.seatunnel.assertion.sink.AssertConfig.FIELD_VALUE; @@ -75,6 +77,9 @@ private List assembleFieldValueRules( if (config.hasPath(RULE_VALUE)) { valueRule.setRuleValue(config.getDouble(RULE_VALUE)); } + if (config.hasPath(EQUALS_TO)) { + valueRule.setEqualTo(config.getString(EQUALS_TO)); + } return valueRule; }) .collect(Collectors.toList()); @@ -96,5 +101,9 @@ private SeaTunnelDataType getFieldType(String fieldTypeStr) { TYPES.put("float", BasicType.FLOAT_TYPE); TYPES.put("double", BasicType.DOUBLE_TYPE); TYPES.put("void", BasicType.VOID_TYPE); + TYPES.put("timestamp", LocalTimeType.LOCAL_DATE_TIME_TYPE); + TYPES.put("datetime", LocalTimeType.LOCAL_DATE_TIME_TYPE); + TYPES.put("date", LocalTimeType.LOCAL_DATE_TYPE); + TYPES.put("time", LocalTimeType.LOCAL_TIME_TYPE); } } diff --git a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/sink/AssertConfig.java b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/sink/AssertConfig.java index d27b3b126eb..3e3da9f68b9 100644 --- a/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/sink/AssertConfig.java +++ b/seatunnel-connectors-v2/connector-assert/src/main/java/org/apache/seatunnel/connectors/seatunnel/assertion/sink/AssertConfig.java @@ -26,6 +26,8 @@ public class AssertConfig { public static final String RULE_VALUE = "rule_value"; + public static final String EQUALS_TO = "equals_to"; + public static final String ROW_RULES = "row_rules"; public static final String FIELD_NAME = "field_name"; diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java new file mode 100644 index 00000000000..40699811a41 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.transform; + +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.container.spark.AbstractTestSparkContainer; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; + +import java.io.IOException; + +public class TestSimpleSQLIT extends TestSuiteBase { + + @TestTemplate + public void testSimpleSQL(TestContainer container) throws IOException, InterruptedException { + if (container instanceof AbstractTestSparkContainer) { + return; // skip test for spark, because some problems of Timestamp convert unresolved. + } + + Container.ExecResult execResult = container.executeJob("/simple_sql_transform.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + // -------------binary expression------------- + execResult = container.executeJob("/simple_sql_functions/binary_expression.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + // -------------start test functions------------- + execResult = container.executeJob("/simple_sql_functions/func_ascii.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_bit_length.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_char_length.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_octet_length.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_chr.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_concat.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_hextoraw.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_rawtohex.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_insert.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_lower.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_upper.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_left.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_right.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_lpad.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_rpad.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_ltrim.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_rtrim.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_trim.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_regexp_replace.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_regexp_like.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_regexp_substr.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_repeat.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_replace.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_soundex.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_space.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_substr.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_to_char.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_translate.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_abs.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_acos.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_asin.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_atan.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_cos.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_cosh.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_sin.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_sinh.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_tan.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_tanh.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_mod.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_ceil.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_exp.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_floor.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_ln.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_log.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_log10.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_radians.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_sqrt.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_pi.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_power.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_rand.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_round.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_sign.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_trunc.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_current_date.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_current_time.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_current_timestamp.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_dateadd.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_datediff.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_date_trunc.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_dayname.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_day_of_week.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_day_of_year.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_extract.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_formatdatetime.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_hour.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_minute.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_month.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_monthname.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_parsedatetime.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_quarter.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_second.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_week.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_year.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_cast.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_coalesce.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_ifnull.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/func_nullif.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + // -------------start test filter------------- + execResult = container.executeJob("/simple_sql_functions/filter_equals_to.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_not_equals_to.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_great_than.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_great_than_equals.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_minor_than.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_minor_than_equals.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_is_null.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_is_not_null.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_in.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_not_in.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_function.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/simple_sql_functions/filter_and_or.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf new file mode 100644 index 00000000000..22bd6e399ef --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [1, "Joy Ding", 134.22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id+1 as id, id*4 as id2, price/3 as price, price-34.22 as price2, price%23.12 as price3, name||'_'||id as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "int" + field_value = [ + { equals_to = 2 } + ] + }, + { + field_name = "id2" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + }, + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 44.74 } + ] + }, + { + field_name = "price2" + field_type = "double" + field_value = [ + { equals_to = 100 } + ] + }, + { + field_name = "price3" + field_type = "double" + field_value = [ + { equals_to = 18.619999999999994 } + ] + }, + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding_1" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf new file mode 100644 index 00000000000..17b45701abc --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + { fields = [3, "Kin Dom", 21], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where name<>'Kin Dom' and (id=1 or age<22)" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf new file mode 100644 index 00000000000..912a4138797 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id=1" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf new file mode 100644 index 00000000000..18afb71a3aa --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where regexp_like(name, '[A-Z ]*')" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 2 + }, + { + rule_type = MAX_ROW + rule_value = 2 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf new file mode 100644 index 00000000000..143667b57ff --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id>1" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf new file mode 100644 index 00000000000..8ff815301e4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id>=2" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf new file mode 100644 index 00000000000..14498036291 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + { fields = [3, "Kin Dom", 21], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id in (1,2)" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 2 + }, + { + rule_type = MAX_ROW + rule_value = 2 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf new file mode 100644 index 00000000000..83d76359fa7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, null, 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where name is not null" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf new file mode 100644 index 00000000000..a21038f2d2e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, null, 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where name is null" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf new file mode 100644 index 00000000000..4c40926c629 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id<2" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf new file mode 100644 index 00000000000..0159c9aca66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id<=1" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf new file mode 100644 index 00000000000..ca5eb8e461f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id<>1" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf new file mode 100644 index 00000000000..33853e0834a --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 20], kind = INSERT } + { fields = [2, "May Ding", 22], kind = INSERT } + { fields = [3, "Kin Dom", 21], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select id,name,age from fake where id not in (1,2)" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 1 + }, + { + rule_type = MAX_ROW + rule_value = 1 + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf new file mode 100644 index 00000000000..b4ec5f6073c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [-1, "book", -120.72], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select abs(id) as id, abs(price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "int" + field_value = [ + { equals_to = 1 } + ] + } + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 120.72 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf new file mode 100644 index 00000000000..45dac4bba0c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select acos(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf new file mode 100644 index 00000000000..ff586629da2 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "string" + test2 = "string" + test3 = "string" + } + } + rows = [ + { fields = [1, "A", "b" , "&"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select ascii(test1) as test1, ascii(test2) as test2, ascii(test3) as test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 65 } + ] + }, + { + field_name = "test2" + field_type = "int" + field_value = [ + { equals_to = 98 } + ] + }, + { + field_name = "test3" + field_type = "int" + field_value = [ + { equals_to = 38 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf new file mode 100644 index 00000000000..998e57a0f4c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select asin(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf new file mode 100644 index 00000000000..77bcaec2254 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "double" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select atan(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0.0 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf new file mode 100644 index 00000000000..188f5fff83f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "string" + test2 = "string" + test3 = "string" + } + } + rows = [ + { fields = [1, "A", "My test" , "&^^$wef9"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select bit_length(test1) as test1, bit_length(test2) as test2, bit_length(test3) as test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "long" + field_value = [ + { equals_to = 8 } + ] + }, + { + field_name = "test2" + field_type = "long" + field_value = [ + { equals_to = 56 } + ] + }, + { + field_name = "test3" + field_type = "long" + field_value = [ + { equals_to = 64 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf new file mode 100644 index 00000000000..2a4b8b740de --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "bigint" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "12.4", "2012-12-21T12:34:56"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select cast(id as STRING) as id, cast(id as INT) as id2, cast(id as DOUBLE) as id3 , cast(name as double) as name, cast(name as DECIMAL(10,2)) as name2, cast(c_time as DATE) as c_time, cast(c_time as time) as c_time2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "string" + field_value = [ + { equals_to = "1" } + ] + }, + { + field_name = "id2" + field_type = "int" + field_value = [ + { equals_to = 1 } + ] + }, + { + field_name = "id3" + field_type = "double" + field_value = [ + { equals_to = 1 } + ] + }, + { + field_name = "name" + field_type = "double" + field_value = [ + { equals_to = 12.4 } + ] + }, + { + field_name = "c_time" + field_type = "date" + field_value = [ + { equals_to = "2012-12-21" } + ] + }, + { + field_name = "c_time2" + field_type = "time" + field_value = [ + { equals_to = "12:34:56" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf new file mode 100644 index 00000000000..f78e294d025 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [4, "Joy Ding", 13.2], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select ceil(id) as id, ceiling(price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + } + { + field_name = "price" + field_type = "int" + field_value = [ + { equals_to = 14 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf new file mode 100644 index 00000000000..192de77f546 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "string" + test2 = "string" + test3 = "string" + } + } + rows = [ + { fields = [1, "A", "My test" , "&^^$wef9"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select char_length(test1) as test1, char_length(test2) as test2, length(test3) as test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "long" + field_value = [ + { equals_to = 1 } + ] + }, + { + field_name = "test2" + field_type = "long" + field_value = [ + { equals_to = 7 } + ] + }, + { + field_name = "test3" + field_type = "long" + field_value = [ + { equals_to = 8 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf new file mode 100644 index 00000000000..8c0b2ce499f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "int" + test2 = "int" + test3 = "int" + } + } + rows = [ + { fields = [1, 65, 98 , 38], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select chr(test1) as test1, char(test2) as test2, char(test3) as test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "A" } + ] + }, + { + field_name = "test2" + field_type = "string" + field_value = [ + { equals_to = "b" } + ] + }, + { + field_name = "test3" + field_type = "string" + field_value = [ + { equals_to = "&" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf new file mode 100644 index 00000000000..a7396759919 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, null, "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select coalesce(name,'Unknown') test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "Unknown" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf new file mode 100644 index 00000000000..d2fcd23d18f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select concat(name,'_',age) as name, concat(name,id,'!') as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding_15" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding!" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf new file mode 100644 index 00000000000..086bdfb277a --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select concat_ws('_',name,age) as name, concat_ws('_',name,id,'!') as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding_15" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding_!" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf new file mode 100644 index 00000000000..408739ab728 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "double" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select cos(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 1 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf new file mode 100644 index 00000000000..a4485435b41 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "double" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select cosh(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 1 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf new file mode 100644 index 00000000000..53038601167 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "double" + name = "string" + age = "int" + } + } + rows = [ + { fields = [4, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select cot(pi()/id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 1.0000000000000002 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf new file mode 100644 index 00000000000..55d9aaf7d5d --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 14], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select current_date as cd, current_date() as cd2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "cd" + field_type = "date" + field_value = [ + { rule_type = NOT_NULL } + ] + } + { + field_name = "cd2" + field_type = "date" + field_value = [ + { rule_type = NOT_NULL } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf new file mode 100644 index 00000000000..79c94dbd803 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 14], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select current_time as ct, current_time() as ct2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "ct" + field_type = "time" + field_value = [ + { rule_type = NOT_NULL } + ] + } + { + field_name = "ct2" + field_type = "time" + field_value = [ + { rule_type = NOT_NULL } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf new file mode 100644 index 00000000000..a0679e79f60 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf @@ -0,0 +1,83 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 14], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select current_timestamp as cd, current_timestamp() as cd2, now() as now from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "cd" + field_type = "timestamp" + field_value = [ + { rule_type = NOT_NULL } + ] + } + { + field_name = "cd2" + field_type = "timestamp" + field_value = [ + { rule_type = NOT_NULL } + ] + } + { + field_name = "now" + field_type = "timestamp" + field_value = [ + { rule_type = NOT_NULL } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf new file mode 100644 index 00000000000..e0cc50324ed --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-15T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select date_trunc(c_time, 'YEAR') as test1, date_trunc(c_time, 'MONTH') as test2, date_trunc(c_time, 'DAY') as test3, date_trunc(c_time, 'HOUR') as test4, date_trunc(c_time, 'MINUTE') as test5, date_trunc(c_time, 'SECOND') as test6 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-01-01T00:00:00" } + ] + } + { + field_name = "test2" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-01T00:00:00" } + ] + } + { + field_name = "test3" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-15T00:00:00" } + ] + } + { + field_name = "test4" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-15T13:00:00" } + ] + } + { + field_name = "test5" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-15T13:34:00" } + ] + } + { + field_name = "test6" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-15T13:34:45" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf new file mode 100644 index 00000000000..6b6e4bf8181 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf @@ -0,0 +1,111 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-15T13:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select dateadd(c_time, 1) as test1, dateadd(c_time, 40, 'DAY') as test2, dateadd(c_time, 2, 'YEAR') as test3, dateadd(c_time, 10, 'MONTH') as test4, dateadd(c_time, 13, 'HOUR') as test5, dateadd(c_time, 40, 'MINUTE') as test6, dateadd(c_time, 30, 'SECOND') as test7 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-16T13:34:45" } + ] + } + { + field_name = "test2" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-05-25T13:34:45" } + ] + } + { + field_name = "test3" + field_type = "timestamp" + field_value = [ + { equals_to = "2023-04-15T13:34:45" } + ] + } + { + field_name = "test4" + field_type = "timestamp" + field_value = [ + { equals_to = "2022-02-15T13:34:45" } + ] + } + { + field_name = "test5" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-16T02:34:45" } + ] + } + { + field_name = "test6" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-15T14:14:45" } + ] + } + { + field_name = "test7" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-15T13:35:15" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf new file mode 100644 index 00000000000..5d658199cd6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf @@ -0,0 +1,112 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + m_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-15T13:34:45", "2022-01-23T12:34:56"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select datediff(c_time, m_time) as test, datediff(c_time, m_time, 'DAY') as test1, datediff(c_time, m_time, 'YEAR') as test2, datediff(c_time, m_time, 'MONTH') as test3, datediff(c_time, m_time, 'HOUR') as test4, datediff(c_time, m_time, 'MINUTE') as test5, datediff(c_time, m_time, 'SECOND') as test6 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test" + field_type = "long" + field_value = [ + { equals_to = 283 } + ] + } + { + field_name = "test1" + field_type = "long" + field_value = [ + { equals_to = 283 } + ] + } + { + field_name = "test2" + field_type = "long" + field_value = [ + { equals_to = 0 } + ] + } + { + field_name = "test3" + field_type = "long" + field_value = [ + { equals_to = 9 } + ] + } + { + field_name = "test4" + field_type = "long" + field_value = [ + { equals_to = 6791 } + ] + } + { + field_name = "test5" + field_type = "long" + field_value = [ + { equals_to = 407460 } + ] + } + { + field_name = "test6" + field_type = "long" + field_value = [ + { equals_to = 24447611 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf new file mode 100644 index 00000000000..dfb8ca5d6db --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + c_time2 = "timestamp" + c_time3 = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235", "2022-06-17T13:34:45", "2023-01-23T13:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select day_of_month(c_time) test1, day_of_month(c_time2) test2, day_of_month(c_time3) test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 8 } + ] + } + { + field_name = "test2" + field_type = "int" + field_value = [ + { equals_to = 17 } + ] + } + { + field_name = "test3" + field_type = "int" + field_value = [ + { equals_to = 23 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf new file mode 100644 index 00000000000..0457bd18ab1 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + c_time2 = "timestamp" + c_time3 = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235", "2021-04-05T13:34:45", "2021-04-11T13:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select day_of_week(c_time) test1, day_of_week(c_time2) test2, day_of_week(c_time3) test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + # Thursday + { equals_to = 4 } + ] + } + { + field_name = "test2" + field_type = "int" + field_value = [ + # Monday + { equals_to = 1 } + ] + } + { + field_name = "test3" + field_type = "int" + field_value = [ + # Sunday + { equals_to = 7 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf new file mode 100644 index 00000000000..e1067227ddf --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + c_time2 = "timestamp" + c_time3 = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235", "2022-06-17T13:34:45", "2023-01-23T13:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select day_of_year(c_time) test1, day_of_year(c_time2) test2, day_of_year(c_time3) test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 98 } + ] + } + { + field_name = "test2" + field_type = "int" + field_value = [ + { equals_to = 168 } + ] + } + { + field_name = "test3" + field_type = "int" + field_value = [ + { equals_to = 23 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf new file mode 100644 index 00000000000..7ba4cc55c57 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + c_time2 = "timestamp" + c_time3 = "timestamp" + c_time4 = "timestamp" + c_time5 = "timestamp" + c_time6 = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-15T13:34:45.235", "2021-04-17T13:34:45", "2021-04-18T13:34:45", "2021-04-19T13:34:45", "2021-04-20T13:34:45", "2021-04-21T13:34:45", "2021-04-22T13:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select dayname(c_time) as test1, dayname(to_date('2021-04-16','yyyy-MM-dd')) as test2, dayname(c_time2) as test3, dayname(c_time3) as test4, dayname(c_time4) as test5, dayname(c_time5) as test6, dayname(c_time6) as test7 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "Thursday" } + ] + } + { + field_name = "test2" + field_type = "string" + field_value = [ + { equals_to = "Friday" } + ] + } + { + field_name = "test3" + field_type = "string" + field_value = [ + { equals_to = "Saturday" } + ] + } + { + field_name = "test4" + field_type = "string" + field_value = [ + { equals_to = "Sunday" } + ] + } + { + field_name = "test5" + field_type = "string" + field_value = [ + { equals_to = "Monday" } + ] + } + { + field_name = "test6" + field_type = "string" + field_value = [ + { equals_to = "Tuesday" } + ] + } + { + field_name = "test7" + field_type = "string" + field_value = [ + { equals_to = "Wednesday" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf new file mode 100644 index 00000000000..0226b7023fc --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [2, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select exp(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 7.38905609893065 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf new file mode 100644 index 00000000000..c6b4816a961 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf @@ -0,0 +1,127 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + c_time2 = "timestamp" + c_time3 = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235", "2022-06-17T13:34:45", "2023-01-23T13:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select extract(YEAR FROM c_time) test1, extract(MONTH FROM c_time2) test2, extract(DAY FROM c_time3) test3, extract(HOUR FROM c_time) test4, extract(MINUTE from c_time) test5, extract(SECOND from c_time) test6, extract(MILLISECOND from c_time) test7, extract(DAYOFWEEK FROM c_time) test8, extract(DAYOFYEAR FROM c_time) test9 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 2021 } + ] + } + { + field_name = "test2" + field_type = "int" + field_value = [ + { equals_to = 6 } + ] + } + { + field_name = "test3" + field_type = "int" + field_value = [ + { equals_to = 23 } + ] + } + { + field_name = "test4" + field_type = "int" + field_value = [ + { equals_to = 13 } + ] + } + { + field_name = "test5" + field_type = "int" + field_value = [ + { equals_to = 34 } + ] + } + { + field_name = "test6" + field_type = "int" + field_value = [ + { equals_to = 45 } + ] + } + { + field_name = "test7" + field_type = "int" + field_value = [ + { equals_to = 235 } + ] + } + { + field_name = "test8" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + } + { + field_name = "test9" + field_type = "int" + field_value = [ + { equals_to = 98 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf new file mode 100644 index 00000000000..5efbc2eb83c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [4, "Joy Ding", 13.2], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select floor(id) as id, floor(price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + } + { + field_name = "price" + field_type = "int" + field_value = [ + { equals_to = 13 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf new file mode 100644 index 00000000000..3439acbe05f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf @@ -0,0 +1,83 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select formatdatetime(c_time,'yyyy-MM-dd HH:mm:ss.S') test1, formatdatetime(c_time,'yyyy-MM-dd') test2, formatdatetime(c_time,'HH:mm:ss.SSS') test3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "2021-04-08 13:34:45.2" } + ] + } + { + field_name = "test2" + field_type = "string" + field_value = [ + { equals_to = "2021-04-08" } + ] + } + { + field_name = "test3" + field_type = "string" + field_value = [ + { equals_to = "13:34:45.235" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf new file mode 100644 index 00000000000..8e924fc03c4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "string" + test2 = "string" + } + } + rows = [ + { fields = [1, "0031", "0037"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select hextoraw(test1) as test1, hextoraw(test2) as test2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "1" } + ] + }, + { + field_name = "test2" + field_type = "string" + field_value = [ + { equals_to = "7" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf new file mode 100644 index 00000000000..74fdac24115 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select hour(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 13 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf new file mode 100644 index 00000000000..aa5271512c7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, null, "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select ifnull(name,'Unknown') test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "Unknown" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf new file mode 100644 index 00000000000..b0a4e0be707 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select insert(name,1,1,'j') as name, insert(name,2,2,'**') as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "joy Ding" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "J** Ding" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf new file mode 100644 index 00000000000..e132c212ec2 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select left(name, 3) as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf new file mode 100644 index 00000000000..0fb3f57e8a0 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [4, "Joy Ding", 13.2], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select ln(id) as id, ln(price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 1.3862943611198906 } + ] + } + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 2.580216829592325 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf new file mode 100644 index 00000000000..6877c17838d --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf @@ -0,0 +1,83 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select locate(' ', name, 2) as name, instr(name, ' ', 3) as name2, position(' ', name) as name3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + }, + { + field_name = "name2" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + }, + { + field_name = "name3" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf new file mode 100644 index 00000000000..5d7253820dc --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [4, "Joy Ding", 13.2], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select log(2,id) as id, log(10,price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 2 } + ] + } + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 1.1205739312058498 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf new file mode 100644 index 00000000000..290196a9192 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [100, "Joy Ding", 1324.252], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select log10(id) as id, log10(price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 2 } + ] + } + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 3.1219706375172507 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf new file mode 100644 index 00000000000..5fc114202e7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select lower(name) as name, lcase(name) as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "joy ding" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "joy ding" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf new file mode 100644 index 00000000000..c2be1e34c6f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select lpad(name, 3) as name, lpad(name, 10, '*') as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "**Joy Ding" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf new file mode 100644 index 00000000000..258ebd4cf85 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + email = "string" + } + } + rows = [ + { fields = [null, " Joy Ding ", "*Joy_Ding@s.com" ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select ltrim(name) as name, ltrim(email, '*') as email from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding " } + ] + }, + { + field_name = "email" + field_type = "string" + field_value = [ + { equals_to = "Joy_Ding@s.com" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf new file mode 100644 index 00000000000..a3c48647e3e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select minute(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 34 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf new file mode 100644 index 00000000000..4ed2a68544f --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf @@ -0,0 +1,83 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "bigint" + } + } + rows = [ + { fields = [4, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select mod(id, 3) as id, mod(age, 5) as age, mod(age, 5.4) as age2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "int" + field_value = [ + { equals_to = 1 } + ] + } + { + field_name = "age" + field_type = "int" + field_value = [ + { equals_to = 3 } + ] + } + { + field_name = "age2" + field_type = "double" + field_value = [ + { equals_to = 2.2 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf new file mode 100644 index 00000000000..4b1310353c5 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select month(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 4 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf new file mode 100644 index 00000000000..69507c5c97e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select monthname(c_time,'yyyy-MM-dd HH:mm:ss.S') test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "April" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf new file mode 100644 index 00000000000..7a39164ae4d --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select ifnull(nullif(name,'Joy Ding'),'NULL') test1, nullif(name,'Joy Ding_') test2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "NULL" } + ] + } + { + field_name = "test2" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf new file mode 100644 index 00000000000..2eb736028a7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "string" + test2 = "string" + } + } + rows = [ + { fields = [1, "A", "My test"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select octet_length(test1) as test1, octet_length(test2) as test2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "long" + field_value = [ + { equals_to = 1 } + ] + }, + { + field_name = "test2" + field_type = "long" + field_value = [ + { equals_to = 7 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf new file mode 100644 index 00000000000..0043d83d321 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "string" + c_time2 = "string" + c_time3 = "string" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08 13:34:45.235", "2021-04-08"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select parsedatetime(c_time,'yyyy-MM-dd HH:mm:ss.SSS') test1,to_date(c_time2,'yyyy-MM-dd') test2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "timestamp" + field_value = [ + { equals_to = "2021-04-08T13:34:45.235" } + ] + } + { + field_name = "test2" + field_type = "date" + field_value = [ + { equals_to = "2021-04-08" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf new file mode 100644 index 00000000000..be5155b8819 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 14], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select pi() as pi from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "pi" + field_type = "double" + field_value = [ + { equals_to = 3.141592653589793 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf new file mode 100644 index 00000000000..a33014f8ffe --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [4, "book", 13.45], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select power(id,3) as id, power(price,2) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 64 } + ] + } + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 180.90249999999997 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf new file mode 100644 index 00000000000..96674c0720c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select quarter(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 2 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf new file mode 100644 index 00000000000..ead5be9ae80 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [180, "Joy Ding", 14], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select radians(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 3.141592653589793 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf new file mode 100644 index 00000000000..bbab563180a --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + row.num = 500 + schema = { + fields { + id = "int" + name = "string" + } + } + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select rand() as rand from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "rand" + field_type = "double" + field_value = [ + { + rule_type = MAX + rule_value = 1 + } + { + rule_type = MIN + rule_value = 0 + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf new file mode 100644 index 00000000000..352315df7b4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + test1 = "string" + test2 = "string" + } + } + rows = [ + { fields = [1, "1", "7"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select rawtohex(test1) as test1, rawtohex(test2) as test2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "string" + field_value = [ + { equals_to = "0031" } + ] + }, + { + field_name = "test2" + field_type = "string" + field_value = [ + { equals_to = "0037" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf new file mode 100644 index 00000000000..d6735ae2878 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + email = "string" + } + } + rows = [ + { fields = [null, "Joy Ding", "Joy_WWWDing@s.com" ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select regexp_like(name, '[A-Z ]*', 'i') as name, regexp_like(email, '[A-Z ]*', 'i') as email from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "boolean" + field_value = [ + { equals_to = true } + ] + }, + { + field_name = "email" + field_type = "boolean" + field_value = [ + { equals_to = true } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf new file mode 100644 index 00000000000..16257b8a65e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + email = "string" + } + } + rows = [ + { fields = [null, "Joy Ding", "Joy_WWWDing@s.com" ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select regexp_replace(name, '.+', 'b') as name, regexp_replace(email, 'w+', 'W', 'i') as email from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "b" } + ] + }, + { + field_name = "email" + field_type = "string" + field_value = [ + { equals_to = "Joy_WDing@s.com" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf new file mode 100644 index 00000000000..ab53ef20de4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_date = "string" + } + } + rows = [ + { fields = [null, "Joy Ding", "2020-10-01" ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select regexp_substr(c_date, '\\d{4}') as c_year, regexp_substr(c_date, '(\\d{4})-(\\d{2})-(\\d{2})', 1, 1, null, 2) as c_month from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "c_year" + field_type = "string" + field_value = [ + { equals_to = "2020" } + ] + }, + { + field_name = "c_month" + field_type = "string" + field_value = [ + { equals_to = "10" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf new file mode 100644 index 00000000000..c9d47febcb2 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select repeat(name||' ',3) as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding Joy Ding Joy Ding " } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf new file mode 100644 index 00000000000..4a46303a43e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select replace(name,' ') as name, replace(name,' ','_') as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "JoyDing" } + ] + } + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "Joy_Ding" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf new file mode 100644 index 00000000000..9d8ea0d4980 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select right(name, 4) as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Ding" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf new file mode 100644 index 00000000000..1f7231c62b6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [1, "book", 120.72124], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select round(price) as price, round(price,2) as price2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 121 } + ] + } + { + field_name = "price2" + field_type = "double" + field_value = [ + { equals_to = 120.72 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf new file mode 100644 index 00000000000..ad0df510ea4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select rpad(name, 3) as name, rpad(name, 10, '*') as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding**" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf new file mode 100644 index 00000000000..48b76e05800 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + email = "string" + } + } + rows = [ + { fields = [null, " Joy Ding ", "Joy_Ding@s.com*" ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select rtrim(name) as name, rtrim(email, '*') as email from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = " Joy Ding" } + ] + }, + { + field_name = "email" + field_type = "string" + field_value = [ + { equals_to = "Joy_Ding@s.com" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf new file mode 100644 index 00000000000..115cba705e7 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select second(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 45 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf new file mode 100644 index 00000000000..ad3522ca244 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + price2 = "double" + price3 = "double" + } + } + rows = [ + { fields = [1, "book", -0.23, 0, null], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select sign(id) as id, sign(price) as price, sign(price2) as price2, sign(price3) as price3 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "int" + field_value = [ + { equals_to = 1 } + ] + } + { + field_name = "price" + field_type = "int" + field_value = [ + { equals_to = -1 } + ] + } + { + field_name = "price2" + field_type = "int" + field_value = [ + { equals_to = 0 } + ] + } + { + field_name = "price3" + field_type = "int" + field_value = [ + { equals_to = null } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf new file mode 100644 index 00000000000..0806b3daa90 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select sin(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0.0 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf new file mode 100644 index 00000000000..983c7d333fb --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select sinh(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0.0 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf new file mode 100644 index 00000000000..623916a1200 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select soundex(name) as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "J352" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf new file mode 100644 index 00000000000..2e0a1a5f955 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select name || space(3) as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding " } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf new file mode 100644 index 00000000000..cb7caa877a0 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [100, "Joy Ding", 10.24], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select sqrt(id) as id, sqrt(price) as price from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 10 } + ] + } + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 3.2 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf new file mode 100644 index 00000000000..ef83811d75e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 12], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select substr(name, 5) as name, substring(name, 1, 3) as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Ding" } + ] + } + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "Joy" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf new file mode 100644 index 00000000000..73dd63cef3e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "double" + name = "string" + age = "int" + } + } + rows = [ + { fields = [4, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select tan(pi()/id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0.9999999999999999 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf new file mode 100644 index 00000000000..5fd4a408cef --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [0, "Joy Ding", 13], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select tanh(id) as id from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "id" + field_type = "double" + field_value = [ + { equals_to = 0.0 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf new file mode 100644 index 00000000000..3d4fb48d4ea --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2022-12-12T23:34:45"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select to_char(id) as id, to_char(c_time,'yyyy-MM-dd') as c_date from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "c_date" + field_type = "string" + field_value = [ + { equals_to = "2022-12-12" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf new file mode 100644 index 00000000000..cd4f102806e --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [1, "Joy Ding", 12], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select translate(name, 'ing', 'ING') as name from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy DING" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf new file mode 100644 index 00000000000..7a8fa166d5c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + email = "string" + } + } + rows = [ + { fields = [null, " Joy Ding ", "**Joy_Ding@s.com*" ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select trim(name) as name, trim(email, '*') as email from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "Joy Ding" } + ] + }, + { + field_name = "email" + field_type = "string" + field_value = [ + { equals_to = "Joy_Ding@s.com" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf new file mode 100644 index 00000000000..f829a9d0e3c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + price = "double" + } + } + rows = [ + { fields = [1, "book", 120.72124], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select trunc(price) as price, trunc(price,2) as price2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "price" + field_type = "double" + field_value = [ + { equals_to = 120 } + ] + } + { + field_name = "price2" + field_type = "double" + field_value = [ + { equals_to = 120.72 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf new file mode 100644 index 00000000000..6a3d0754e71 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + rows = [ + { fields = [null, "Joy Ding", 15 ], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select upper(name) as name, ucase(name) as name2 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "name" + field_type = "string" + field_value = [ + { equals_to = "JOY DING" } + ] + }, + { + field_name = "name2" + field_type = "string" + field_value = [ + { equals_to = "JOY DING" } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf new file mode 100644 index 00000000000..5001fcb3450 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select week(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 15 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf new file mode 100644 index 00000000000..e2b8fe6966c --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10000 +} + +source { + FakeSource { + result_table_name = "fake" + schema = { + fields { + id = "int" + name = "string" + c_time = "timestamp" + } + } + rows = [ + { fields = [1, "Joy Ding", "2021-04-08T13:34:45.235"], kind = INSERT } + ] + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + query = "select year(c_time) test1 from fake" + } +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = { + field_rules = [ + { + field_name = "test1" + field_type = "int" + field_value = [ + { equals_to = 2021 } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf new file mode 100644 index 00000000000..ec594956051 --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf @@ -0,0 +1,112 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + job.mode = "BATCH" +} + +source { + FakeSource { + result_table_name = "fake" + row.num = 100 + schema = { + fields { + id = "int" + name = "string" + age = "int" + } + } + } +} + +transform { + SimpleSQL { + source_table_name = "fake" + result_table_name = "fake1" + # the query table name must same as field 'source_table_name' + query = "select id, regexp_replace(name, '.+', 'b') as name, age+1 as age, pi() as pi from fake" + } + # The SimpleSQL transform support base function and criteria operation + # But the complex SQL unsupported yet, include: multi source table/rows JOIN and AGGREGATE operation and the like +} + +sink { + Console { + source_table_name = "fake1" + } + Assert { + source_table_name = "fake1" + rules = + { + row_rules = [ + { + rule_type = MIN_ROW + rule_value = 100 + } + ] + field_rules = [ + { + field_name = id + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + }, + { + rule_type = MIN_LENGTH + rule_value = 1 + }, + { + rule_type = MAX_LENGTH + rule_value = 1 + } + ] + }, + { + field_name = age + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = pi + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java index 36d0e2d0f46..d7a9cc4e3a0 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java @@ -28,6 +28,7 @@ import lombok.extern.slf4j.Slf4j; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -53,6 +54,22 @@ public TransformFlowLifeCycle( this.collector = collector; } + @Override + public void open() throws Exception { + super.open(); + for (SeaTunnelTransform t : transform) { + try { + t.open(); + } catch (Exception e) { + log.error( + "Open transform: {} failed, cause: {}", + t.getPluginName(), + e.getMessage(), + e); + } + } + } + @Override public void received(Record record) { if (record.getData() instanceof Barrier) { @@ -93,4 +110,20 @@ public void received(Record record) { public void restoreState(List actionStateList) throws Exception { // nothing } + + @Override + public void close() throws IOException { + for (SeaTunnelTransform t : transform) { + try { + t.close(); + } catch (Exception e) { + log.error( + "Close transform: {} failed, cause: {}", + t.getPluginName(), + e.getMessage(), + e); + } + } + super.close(); + } } diff --git a/seatunnel-transforms-v2/pom.xml b/seatunnel-transforms-v2/pom.xml index 53e385a95b6..5ff90ff2a7d 100644 --- a/seatunnel-transforms-v2/pom.xml +++ b/seatunnel-transforms-v2/pom.xml @@ -44,6 +44,11 @@ org.apache.seatunnel seatunnel-api + + com.github.jsqlparser + jsqlparser + ${jsqlparser.version} + diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java new file mode 100644 index 00000000000..b527fcc5909 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.transform.SeaTunnelTransform; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.transform.common.AbstractSeaTunnelTransform; +import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngine; +import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngineFactory; +import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngineFactory.EngineType; + +import com.google.auto.service.AutoService; + +import static org.apache.seatunnel.transform.sqlengine.SimpleSQLEngineFactory.EngineType.INTERNAL; + +@AutoService(SeaTunnelTransform.class) +public class SimpleSQLTransform extends AbstractSeaTunnelTransform { + + public static final Option KEY_QUERY = + Options.key("query") + .stringType() + .noDefaultValue() + .withDescription("The simple query SQL"); + + public static final Option KEY_ENGINE = + Options.key("engine") + .stringType() + .noDefaultValue() + .withDescription("The SQL engine for SimpleSQL transform"); + + private String query; + + private EngineType engineType; + + private transient SimpleSQLEngine simpleSQLEngine; + + @Override + public String getPluginName() { + return "SimpleSQL"; + } + + @Override + protected void setConfig(Config pluginConfig) { + CheckResult checkResult = CheckConfigUtil.checkAllExists(pluginConfig, KEY_QUERY.key()); + if (!checkResult.isSuccess()) { + throw new IllegalArgumentException("Failed to check config! " + checkResult.getMsg()); + } + query = pluginConfig.getString(KEY_QUERY.key()); + if (pluginConfig.hasPath(KEY_ENGINE.key())) { + engineType = EngineType.valueOf(pluginConfig.getString(KEY_ENGINE.key()).toUpperCase()); + } else { + engineType = INTERNAL; + } + } + + @Override + public void open() { + simpleSQLEngine = SimpleSQLEngineFactory.getSimpleSQLEngine(engineType); + simpleSQLEngine.init(inputTableName, inputRowType, query); + } + + private void tryOpen() { + if (simpleSQLEngine == null) { + open(); + } + } + + @Override + protected SeaTunnelRowType transformRowType(SeaTunnelRowType inputRowType) { + tryOpen(); + return simpleSQLEngine.typeMapping(); + } + + @Override + protected SeaTunnelRow transformRow(SeaTunnelRow inputRow) { + tryOpen(); + return simpleSQLEngine.transformBySQL(inputRow); + } + + @Override + public void close() { + simpleSQLEngine.close(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java new file mode 100644 index 00000000000..9805e8d1d88 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableTransformFactory; + +import com.google.auto.service.AutoService; + +import static org.apache.seatunnel.transform.SimpleSQLTransform.KEY_QUERY; + +@AutoService(Factory.class) +public class SimpleSQLTransformFactory implements TableTransformFactory { + @Override + public String factoryIdentifier() { + return "SimpleSQL"; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder().required(KEY_QUERY).build(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java index c3733134113..0610be68e27 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java @@ -34,11 +34,11 @@ public abstract class AbstractSeaTunnelTransform implements SeaTunnelTransform pair = + executeComparisonOperator((ComparisonOperator) whereExpr, inputFields); + if (whereExpr instanceof EqualsTo) { + return equalsToExpr(pair); + } + if (whereExpr instanceof NotEqualsTo) { + return notEqualsToExpr(pair); + } + if (whereExpr instanceof GreaterThan) { + return greaterThanExpr(pair); + } + if (whereExpr instanceof GreaterThanEquals) { + return greaterThanEqualsExpr(pair); + } + if (whereExpr instanceof MinorThan) { + return minorThanExpr(pair); + } + if (whereExpr instanceof MinorThanEquals) { + return minorThanEqualsExpr(pair); + } + } + if (whereExpr instanceof AndExpression) { + return andExpr((AndExpression) whereExpr, inputFields); + } + if (whereExpr instanceof OrExpression) { + return orExpr((OrExpression) whereExpr, inputFields); + } + if (whereExpr instanceof Parenthesis) { + return parenthesisExpr((Parenthesis) whereExpr, inputFields); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported SQL Expression: %s ", whereExpr)); + } + + private boolean functionExpr(Function function, Object[] inputFields) { + Boolean result = (Boolean) zetaSQLFunction.computeForValue(function, inputFields); + if (result == null) { + return false; + } + return result; + } + + private boolean isNullExpr(IsNullExpression isNullExpression, Object[] inputFields) { + Expression leftExpr = isNullExpression.getLeftExpression(); + Object leftVal = zetaSQLFunction.computeForValue(leftExpr, inputFields); + if (isNullExpression.isNot()) { + return leftVal != null; + } else { + return leftVal == null; + } + } + + private boolean inExpr(InExpression inExpression, Object[] inputFields) { + Expression leftExpr = inExpression.getLeftExpression(); + ExpressionList itemsList = (ExpressionList) inExpression.getRightItemsList(); + Object leftValue = zetaSQLFunction.computeForValue(leftExpr, inputFields); + for (Expression exprItem : itemsList.getExpressions()) { + Object rightValue = zetaSQLFunction.computeForValue(exprItem, inputFields); + if (leftValue == null && rightValue == null) { + return true; + } + if (leftValue != null) { + if (leftValue instanceof Number && rightValue instanceof Number) { + if (((Number) leftValue).doubleValue() == ((Number) rightValue).doubleValue()) { + return !inExpression.isNot(); + } + } else if (leftValue.equals(rightValue)) { + return !inExpression.isNot(); + } + + } else { + return false; + } + } + return inExpression.isNot(); // if all not in return true + } + + /** + * Like expression filter, unsupported yet + * + * @param likeExpression like expression + * @param inputFields input fields + * @return filter result + */ + private boolean likeExpr(LikeExpression likeExpression, Object[] inputFields) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + "Unsupported [like] filter for SimpleSQL yet"); + } + + private Pair executeComparisonOperator( + ComparisonOperator comparisonOperator, Object[] inputFields) { + Expression leftExpr = comparisonOperator.getLeftExpression(); + Expression rightExpr = comparisonOperator.getRightExpression(); + Object leftVal = zetaSQLFunction.computeForValue(leftExpr, inputFields); + Object rightVal = zetaSQLFunction.computeForValue(rightExpr, inputFields); + return Pair.of(leftVal, rightVal); + } + + private boolean equalsToExpr(Pair pair) { + Object leftVal = pair.getLeft(); + Object rightVal = pair.getRight(); + if (leftVal == null || rightVal == null) { + return false; + } + if (leftVal instanceof Number && rightVal instanceof Number) { + return ((Number) leftVal).doubleValue() == ((Number) rightVal).doubleValue(); + } + return leftVal.equals(rightVal); + } + + private boolean notEqualsToExpr(Pair pair) { + Object leftVal = pair.getLeft(); + Object rightVal = pair.getRight(); + if (leftVal == null) { + return rightVal != null; + } + if (leftVal instanceof Number && rightVal instanceof Number) { + return ((Number) leftVal).doubleValue() != ((Number) rightVal).doubleValue(); + } + return !leftVal.equals(rightVal); + } + + private boolean greaterThanExpr(Pair pair) { + Object leftVal = pair.getLeft(); + Object rightVal = pair.getRight(); + if (leftVal == null || rightVal == null) { + return false; + } + if (leftVal instanceof Number && rightVal instanceof Number) { + return ((Number) leftVal).doubleValue() > ((Number) rightVal).doubleValue(); + } + if (leftVal instanceof String && rightVal instanceof String) { + return ((String) leftVal).compareTo((String) rightVal) > 0; + } + if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { + return ((LocalDateTime) leftVal).isAfter((LocalDateTime) rightVal); + } + if (leftVal instanceof LocalDate && rightVal instanceof LocalDate) { + return ((LocalDate) leftVal).isAfter((LocalDate) rightVal); + } + if (leftVal instanceof LocalTime && rightVal instanceof LocalTime) { + return ((LocalTime) leftVal).isAfter((LocalTime) rightVal); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Filed types not matched, left: %s, right:%s ", + leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); + } + + private boolean greaterThanEqualsExpr(Pair pair) { + Object leftVal = pair.getLeft(); + Object rightVal = pair.getRight(); + if (leftVal == null || rightVal == null) { + return false; + } + if (leftVal instanceof Number && rightVal instanceof Number) { + return ((Number) leftVal).doubleValue() >= ((Number) rightVal).doubleValue(); + } + if (leftVal instanceof String && rightVal instanceof String) { + return ((String) leftVal).compareTo((String) rightVal) >= 0; + } + if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { + return ((LocalDateTime) leftVal).isAfter((LocalDateTime) rightVal) + || ((LocalDateTime) leftVal).isEqual((LocalDateTime) rightVal); + } + if (leftVal instanceof LocalDate && rightVal instanceof LocalDate) { + return ((LocalDate) leftVal).isAfter((LocalDate) rightVal) + || ((LocalDate) leftVal).isEqual((LocalDate) rightVal); + } + if (leftVal instanceof LocalTime && rightVal instanceof LocalTime) { + return ((LocalTime) leftVal).isAfter((LocalTime) rightVal) || leftVal.equals(rightVal); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Filed types not matched, left: %s, right:%s ", + leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); + } + + private boolean minorThanExpr(Pair pair) { + Object leftVal = pair.getLeft(); + Object rightVal = pair.getRight(); + if (leftVal == null || rightVal == null) { + return false; + } + if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { + return ((LocalDateTime) leftVal).isBefore((LocalDateTime) rightVal); + } + if (leftVal instanceof LocalDate && rightVal instanceof LocalDate) { + return ((LocalDate) leftVal).isBefore((LocalDate) rightVal); + } + if (leftVal instanceof LocalTime && rightVal instanceof LocalTime) { + return ((LocalTime) leftVal).isBefore((LocalTime) rightVal); + } + if (leftVal instanceof Number && rightVal instanceof Number) { + return ((Number) leftVal).doubleValue() < ((Number) rightVal).doubleValue(); + } + if (leftVal instanceof String && rightVal instanceof String) { + return ((String) leftVal).compareTo((String) rightVal) < 0; + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Filed types not matched, left: %s, right:%s ", + leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); + } + + private boolean minorThanEqualsExpr(Pair pair) { + Object leftVal = pair.getLeft(); + Object rightVal = pair.getRight(); + if (leftVal == null || rightVal == null) { + return false; + } + if (leftVal instanceof LocalDateTime && rightVal instanceof LocalDateTime) { + return ((LocalDateTime) leftVal).isBefore((LocalDateTime) rightVal) + || ((LocalDateTime) leftVal).isEqual((LocalDateTime) rightVal); + } + if (leftVal instanceof LocalDate && rightVal instanceof LocalDate) { + return ((LocalDate) leftVal).isBefore((LocalDate) rightVal) + || ((LocalDate) leftVal).isEqual((LocalDate) rightVal); + } + if (leftVal instanceof LocalTime && rightVal instanceof LocalTime) { + return ((LocalTime) leftVal).isBefore((LocalTime) rightVal) || leftVal.equals(rightVal); + } + if (leftVal instanceof Number && rightVal instanceof Number) { + return ((Number) leftVal).doubleValue() <= ((Number) rightVal).doubleValue(); + } + if (leftVal instanceof String && rightVal instanceof String) { + return ((String) leftVal).compareTo((String) rightVal) <= 0; + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Filed types not matched, left: %s, right:%s ", + leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); + } + + private boolean andExpr(AndExpression andExpression, Object[] inputFields) { + Expression leftExpr = andExpression.getLeftExpression(); + boolean leftRes = executeFilter(leftExpr, inputFields); + Expression rightExpr = andExpression.getRightExpression(); + boolean rightRes = executeFilter(rightExpr, inputFields); + return leftRes && rightRes; + } + + private boolean orExpr(OrExpression orExpression, Object[] inputFields) { + Expression leftExpr = orExpression.getLeftExpression(); + boolean leftRes = executeFilter(leftExpr, inputFields); + Expression rightExpr = orExpression.getRightExpression(); + boolean rightRes = executeFilter(rightExpr, inputFields); + return leftRes || rightRes; + } + + private boolean parenthesisExpr(Parenthesis parenthesis, Object[] inputFields) { + Expression expression = parenthesis.getExpression(); + return executeFilter(expression, inputFields); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java new file mode 100644 index 00000000000..8f2756842b9 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java @@ -0,0 +1,545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta; + +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; +import org.apache.seatunnel.transform.sqlengine.zeta.functions.DateTimeFunction; +import org.apache.seatunnel.transform.sqlengine.zeta.functions.NumericFunction; +import org.apache.seatunnel.transform.sqlengine.zeta.functions.StringFunction; +import org.apache.seatunnel.transform.sqlengine.zeta.functions.SystemFunction; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.CastExpression; +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExtractExpression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.NullValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.TimeKeyExpression; +import net.sf.jsqlparser.expression.operators.arithmetic.Addition; +import net.sf.jsqlparser.expression.operators.arithmetic.Concat; +import net.sf.jsqlparser.expression.operators.arithmetic.Division; +import net.sf.jsqlparser.expression.operators.arithmetic.Modulo; +import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication; +import net.sf.jsqlparser.expression.operators.arithmetic.Subtraction; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.schema.Column; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.List; + +public class ZetaSQLFunction { + // ============================internal functions===================== + + // -------------------------string functions---------------------------- + public static final String ASCII = "ASCII"; + public static final String BIT_LENGTH = "BIT_LENGTH"; + public static final String CHAR_LENGTH = "CHAR_LENGTH"; + public static final String LENGTH = "LENGTH"; + public static final String OCTET_LENGTH = "OCTET_LENGTH"; + public static final String CHAR = "CHAR"; + public static final String CHR = "CHR"; + public static final String CONCAT = "CONCAT"; + public static final String CONCAT_WS = "CONCAT_WS"; + public static final String HEXTORAW = "HEXTORAW"; + public static final String RAWTOHEX = "RAWTOHEX"; + public static final String INSERT = "INSERT"; + public static final String LOWER = "LOWER"; + public static final String LCASE = "LCASE"; + public static final String UPPER = "UPPER"; + public static final String UCASE = "UCASE"; + public static final String LEFT = "LEFT"; + public static final String RIGHT = "RIGHT"; + public static final String LOCATE = "LOCATE"; + public static final String INSTR = "INSTR"; + public static final String POSITION = "POSITION"; + public static final String LPAD = "LPAD"; + public static final String RPAD = "RPAD"; + public static final String LTRIM = "LTRIM"; + public static final String RTRIM = "RTRIM"; + public static final String TRIM = "TRIM"; + public static final String REGEXP_REPLACE = "REGEXP_REPLACE"; + public static final String REGEXP_LIKE = "REGEXP_LIKE"; + public static final String REGEXP_SUBSTR = "REGEXP_SUBSTR"; + public static final String REPEAT = "REPEAT"; + public static final String REPLACE = "REPLACE"; + public static final String SOUNDEX = "SOUNDEX"; + public static final String SPACE = "SPACE"; + public static final String SUBSTRING = "SUBSTRING"; + public static final String SUBSTR = "SUBSTR"; + public static final String TO_CHAR = "TO_CHAR"; + public static final String TRANSLATE = "TRANSLATE"; + + // -------------------------numeric functions---------------------------- + public static final String ABS = "ABS"; + public static final String ACOS = "ACOS"; + public static final String ASIN = "ASIN"; + public static final String ATAN = "ATAN"; + public static final String COS = "COS"; + public static final String COSH = "COSH"; + public static final String COT = "COT"; + public static final String SIN = "SIN"; + public static final String SINH = "SINH"; + public static final String TAN = "TAN"; + public static final String TANH = "TANH"; + public static final String ATAN2 = "ATAN2"; + public static final String MOD = "MOD"; + public static final String CEIL = "CEIL"; + public static final String CEILING = "CEILING"; + public static final String EXP = "EXP"; + public static final String FLOOR = "FLOOR"; + public static final String LN = "LN"; + public static final String LOG = "LOG"; + public static final String LOG10 = "LOG10"; + public static final String RADIANS = "RADIANS"; + public static final String SQRT = "SQRT"; + public static final String PI = "PI"; + public static final String POWER = "POWER"; + public static final String RAND = "RAND"; + public static final String RANDOM = "RANDOM"; + public static final String ROUND = "ROUND"; + public static final String SIGN = "SIGN"; + public static final String TRUNC = "TRUNC"; + public static final String TRUNCATE = "TRUNCATE"; + + // -------------------------time and date functions---------------------------- + public static final String CURRENT_DATE = "CURRENT_DATE"; + public static final String CURRENT_DATE_P = "CURRENT_DATE()"; + public static final String CURRENT_TIME = "CURRENT_TIME"; + public static final String CURRENT_TIME_P = "CURRENT_TIME()"; + public static final String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP"; + public static final String CURRENT_TIMESTAMP_P = "CURRENT_TIMESTAMP()"; + public static final String NOW = "NOW"; + public static final String DATEADD = "DATEADD"; + public static final String TIMESTAMPADD = "TIMESTAMPADD"; + public static final String DATEDIFF = "DATEDIFF"; + public static final String DATE_TRUNC = "DATE_TRUNC"; + public static final String DAYNAME = "DAYNAME"; + public static final String DAY_OF_MONTH = "DAY_OF_MONTH"; + public static final String DAY_OF_WEEK = "DAY_OF_WEEK"; + public static final String DAY_OF_YEAR = "DAY_OF_YEAR"; + public static final String EXTRACT = "EXTRACT"; + public static final String FORMATDATETIME = "FORMATDATETIME"; + public static final String HOUR = "HOUR"; + public static final String MINUTE = "MINUTE"; + public static final String MONTH = "MONTH"; + public static final String MONTHNAME = "MONTHNAME"; + public static final String PARSEDATETIME = "PARSEDATETIME"; + public static final String TO_DATE = "TO_DATE"; + public static final String QUARTER = "QUARTER"; + public static final String SECOND = "SECOND"; + public static final String WEEK = "WEEK"; + public static final String YEAR = "YEAR"; + + // -------------------------system functions---------------------------- + public static final String COALESCE = "COALESCE"; + public static final String IFNULL = "IFNULL"; + public static final String NULLIF = "NULLIF"; + + private final SeaTunnelRowType inputRowType; + private final ZetaSQLType zetaSQLType; + + public ZetaSQLFunction(SeaTunnelRowType inputRowType, ZetaSQLType zetaSQLType) { + this.inputRowType = inputRowType; + this.zetaSQLType = zetaSQLType; + } + + public Object computeForValue(Expression expression, Object[] inputFields) { + if (expression instanceof NullValue) { + return null; + } + if (expression instanceof DoubleValue) { + return ((DoubleValue) expression).getValue(); + } + if (expression instanceof LongValue) { + long longVal = ((LongValue) expression).getValue(); + if (longVal <= Integer.MAX_VALUE && longVal >= Integer.MIN_VALUE) { + return (int) longVal; + } else { + return longVal; + } + } + if (expression instanceof StringValue) { + return ((StringValue) expression).getValue(); + } + if (expression instanceof Column) { + int idx = inputRowType.indexOf(((Column) expression).getColumnName()); + return inputFields[idx]; + } + if (expression instanceof Function) { + Function function = (Function) expression; + ExpressionList expressionList = function.getParameters(); + List functionArgs = new ArrayList<>(); + if (expressionList != null) { + for (Expression funcArgExpression : expressionList.getExpressions()) { + functionArgs.add(computeForValue(funcArgExpression, inputFields)); + } + } + return executeFunctionExpr(function.getName(), functionArgs); + } + if (expression instanceof TimeKeyExpression) { + return executeTimeKeyExpr(((TimeKeyExpression) expression).getStringValue()); + } + if (expression instanceof ExtractExpression) { + ExtractExpression extract = (ExtractExpression) expression; + List functionArgs = new ArrayList<>(); + functionArgs.add(computeForValue(extract.getExpression(), inputFields)); + functionArgs.add(extract.getName()); + return executeFunctionExpr(ZetaSQLFunction.EXTRACT, functionArgs); + } + if (expression instanceof Parenthesis) { + Parenthesis parenthesis = (Parenthesis) expression; + return computeForValue(parenthesis.getExpression(), inputFields); + } + if (expression instanceof BinaryExpression) { + return executeBinaryExpr((BinaryExpression) expression, inputFields); + } + if (expression instanceof CastExpression) { + CastExpression castExpression = (CastExpression) expression; + Expression leftExpr = castExpression.getLeftExpression(); + Object leftValue = computeForValue(leftExpr, inputFields); + return executeCastExpr(castExpression, leftValue); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported SQL Expression: %s ", expression.toString())); + } + + public Object executeFunctionExpr(String functionName, List args) { + switch (functionName.toUpperCase()) { + case ASCII: + return StringFunction.ascii(args); + case BIT_LENGTH: + return StringFunction.bitLength(args); + case CHAR_LENGTH: + case LENGTH: + return StringFunction.charLength(args); + case OCTET_LENGTH: + return StringFunction.octetLength(args); + case CHAR: + case CHR: + return StringFunction.chr(args); + case CONCAT: + return StringFunction.concat(args); + case CONCAT_WS: + return StringFunction.concatWs(args); + case HEXTORAW: + return StringFunction.hextoraw(args); + case RAWTOHEX: + return StringFunction.rawtohex(args); + case INSERT: + return StringFunction.insert(args); + case LOWER: + case LCASE: + return StringFunction.lower(args); + case UPPER: + case UCASE: + return StringFunction.upper(args); + case LEFT: + return StringFunction.left(args); + case RIGHT: + return StringFunction.right(args); + case LOCATE: + case POSITION: + return StringFunction.location(functionName, args); + case INSTR: + return StringFunction.instr(args); + case LPAD: + case RPAD: + return StringFunction.pad(functionName, args); + case LTRIM: + return StringFunction.ltrim(args); + case RTRIM: + return StringFunction.rtrim(args); + case TRIM: + return StringFunction.trim(args); + case REGEXP_REPLACE: + return StringFunction.regexpReplace(args); + case REGEXP_LIKE: + return StringFunction.regexpLike(args); + case REGEXP_SUBSTR: + return StringFunction.regexpSubstr(args); + case REPEAT: + return StringFunction.repeat(args); + case REPLACE: + return StringFunction.replace(args); + case SOUNDEX: + return StringFunction.soundex(args); + case SPACE: + return StringFunction.space(args); + case SUBSTRING: + case SUBSTR: + return StringFunction.substring(args); + case TO_CHAR: + return StringFunction.toChar(args); + case TRANSLATE: + return StringFunction.translate(args); + case ABS: + return NumericFunction.abs(args); + case ACOS: + return NumericFunction.acos(args); + case ASIN: + return NumericFunction.asin(args); + case ATAN: + return NumericFunction.atan(args); + case COS: + return NumericFunction.cos(args); + case COSH: + return NumericFunction.cosh(args); + case COT: + return NumericFunction.cot(args); + case SIN: + return NumericFunction.sin(args); + case SINH: + return NumericFunction.sinh(args); + case TAN: + return NumericFunction.tan(args); + case TANH: + return NumericFunction.tanh(args); + case ATAN2: + return NumericFunction.atan2(args); + case MOD: + return NumericFunction.mod(args); + case CEIL: + case CEILING: + return NumericFunction.ceil(args); + case EXP: + return NumericFunction.exp(args); + case FLOOR: + return NumericFunction.floor(args); + case LN: + return NumericFunction.ln(args); + case LOG: + return NumericFunction.log(args); + case LOG10: + return NumericFunction.log10(args); + case RADIANS: + return NumericFunction.radians(args); + case SQRT: + return NumericFunction.sqrt(args); + case PI: + return NumericFunction.pi(args); + case POWER: + return NumericFunction.power(args); + case RAND: + case RANDOM: + return NumericFunction.random(args); + case ROUND: + return NumericFunction.round(args); + case SIGN: + return NumericFunction.sign(args); + case TRUNC: + case TRUNCATE: + return NumericFunction.trunc(args); + case NOW: + return DateTimeFunction.currentTimestamp(); + case DATEADD: + case TIMESTAMPADD: + return DateTimeFunction.dateadd(args); + case DATEDIFF: + return DateTimeFunction.datediff(args); + case DATE_TRUNC: + return DateTimeFunction.dateTrunc(args); + case DAYNAME: + return DateTimeFunction.dayname(args); + case DAY_OF_MONTH: + return DateTimeFunction.dayOfMonth(args); + case DAY_OF_WEEK: + return DateTimeFunction.dayOfWeek(args); + case DAY_OF_YEAR: + return DateTimeFunction.dayOfYear(args); + case EXTRACT: + return DateTimeFunction.extract(args); + case FORMATDATETIME: + return DateTimeFunction.formatdatetime(args); + case HOUR: + return DateTimeFunction.hour(args); + case MINUTE: + return DateTimeFunction.minute(args); + case MONTH: + return DateTimeFunction.month(args); + case MONTHNAME: + return DateTimeFunction.monthname(args); + case PARSEDATETIME: + case TO_DATE: + return DateTimeFunction.parsedatetime(args); + case QUARTER: + return DateTimeFunction.quarter(args); + case SECOND: + return DateTimeFunction.second(args); + case WEEK: + return DateTimeFunction.week(args); + case YEAR: + return DateTimeFunction.year(args); + case COALESCE: + return SystemFunction.coalesce(args); + case IFNULL: + return SystemFunction.ifnull(args); + case NULLIF: + return SystemFunction.nullif(args); + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Function: %s unsupported for SimpleSQL transform", functionName)); + } + } + + public Object executeTimeKeyExpr(String timeKeyExpr) { + switch (timeKeyExpr.toUpperCase()) { + case CURRENT_DATE: + case CURRENT_DATE_P: + return DateTimeFunction.currentDate(); + case CURRENT_TIME: + case CURRENT_TIME_P: + return DateTimeFunction.currentTime(); + case CURRENT_TIMESTAMP: + case CURRENT_TIMESTAMP_P: + return DateTimeFunction.currentTimestamp(); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("TimeKeyExpr: %s unsupported for SimpleSQL transform", timeKeyExpr)); + } + + public Object executeCastExpr(CastExpression castExpression, Object arg) { + String dataType = castExpression.getType().getDataType(); + List args = new ArrayList<>(2); + args.add(arg); + args.add(dataType.toUpperCase()); + if (dataType.equalsIgnoreCase("DECIMAL")) { + List ps = castExpression.getType().getArgumentsStringList(); + args.add(Integer.parseInt(ps.get(0))); + args.add(Integer.parseInt(ps.get(1))); + } + return SystemFunction.castAs(args); + } + + private Object executeBinaryExpr(BinaryExpression binaryExpression, Object[] inputFields) { + if (binaryExpression instanceof Concat) { + Concat concat = (Concat) binaryExpression; + Expression leftExpr = concat.getLeftExpression(); + Expression rightExpr = concat.getRightExpression(); + Function function = new Function(); + function.setName(ZetaSQLFunction.CONCAT); + ExpressionList expressionList = new ExpressionList(); + expressionList.setExpressions(new ArrayList<>()); + expressionList.getExpressions().add(leftExpr); + expressionList.getExpressions().add(rightExpr); + function.setParameters(expressionList); + return computeForValue(function, inputFields); + } + Number leftValue = + (Number) computeForValue(binaryExpression.getLeftExpression(), inputFields); + Number rightValue = + (Number) computeForValue(binaryExpression.getRightExpression(), inputFields); + if (leftValue == null || rightValue == null) { + return null; + } + SeaTunnelDataType resultType = zetaSQLType.getExpressionType(binaryExpression); + if (resultType.getSqlType() == SqlType.INT) { + if (binaryExpression instanceof Addition) { + return leftValue.intValue() + rightValue.intValue(); + } + if (binaryExpression instanceof Subtraction) { + return leftValue.intValue() - rightValue.intValue(); + } + if (binaryExpression instanceof Multiplication) { + return leftValue.intValue() * rightValue.intValue(); + } + if (binaryExpression instanceof Division) { + return leftValue.intValue() / rightValue.intValue(); + } + if (binaryExpression instanceof Modulo) { + return leftValue.intValue() % rightValue.intValue(); + } + } + if (resultType.getSqlType() == SqlType.DECIMAL) { + BigDecimal bigDecimal = BigDecimal.valueOf(leftValue.doubleValue()); + if (binaryExpression instanceof Addition) { + return bigDecimal.add(BigDecimal.valueOf(rightValue.doubleValue())); + } + if (binaryExpression instanceof Subtraction) { + return bigDecimal.subtract(BigDecimal.valueOf(rightValue.doubleValue())); + } + if (binaryExpression instanceof Multiplication) { + return bigDecimal.multiply(BigDecimal.valueOf(rightValue.doubleValue())); + } + if (binaryExpression instanceof Division) { + DecimalType decimalType = (DecimalType) resultType; + return bigDecimal.divide( + BigDecimal.valueOf(rightValue.doubleValue()), + decimalType.getScale(), + RoundingMode.UP); + } + if (binaryExpression instanceof Modulo) { + List args = new ArrayList<>(); + args.add(leftValue); + args.add(rightValue); + return NumericFunction.mod(args); + } + } + if (resultType.getSqlType() == SqlType.DOUBLE) { + if (binaryExpression instanceof Addition) { + return leftValue.doubleValue() + rightValue.doubleValue(); + } + if (binaryExpression instanceof Subtraction) { + return leftValue.doubleValue() - rightValue.doubleValue(); + } + if (binaryExpression instanceof Multiplication) { + return leftValue.doubleValue() * rightValue.doubleValue(); + } + if (binaryExpression instanceof Division) { + return leftValue.doubleValue() / rightValue.doubleValue(); + } + if (binaryExpression instanceof Modulo) { + return leftValue.doubleValue() % rightValue.doubleValue(); + } + } + if (resultType.getSqlType() == SqlType.BIGINT) { + if (binaryExpression instanceof Addition) { + return leftValue.longValue() + rightValue.longValue(); + } + if (binaryExpression instanceof Subtraction) { + return leftValue.longValue() - rightValue.longValue(); + } + if (binaryExpression instanceof Multiplication) { + return leftValue.longValue() * rightValue.longValue(); + } + if (binaryExpression instanceof Division) { + return leftValue.longValue() / rightValue.longValue(); + } + if (binaryExpression instanceof Modulo) { + return leftValue.longValue() % rightValue.longValue(); + } + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported SQL Expression: %s ", binaryExpression)); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java new file mode 100644 index 00000000000..cd80f5d2ca9 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta; + +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.CastExpression; +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExtractExpression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.NullValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.TimeKeyExpression; +import net.sf.jsqlparser.expression.operators.arithmetic.Concat; +import net.sf.jsqlparser.schema.Column; + +import java.util.List; + +public class ZetaSQLType { + public static final String DECIMAL = "DECIMAL"; + public static final String VARCHAR = "VARCHAR"; + public static final String STRING = "STRING"; + public static final String INT = "INT"; + public static final String INTEGER = "INTEGER"; + public static final String BIGINT = "BIGINT"; + public static final String LONG = "LONG"; + public static final String BYTE = "BYTE"; + public static final String DOUBLE = "DOUBLE"; + public static final String FLOAT = "FLOAT"; + public static final String TIMESTAMP = "TIMESTAMP"; + public static final String DATETIME = "DATETIME"; + public static final String DATE = "DATE"; + public static final String TIME = "TIME"; + + private final SeaTunnelRowType inputRowType; + + public ZetaSQLType(SeaTunnelRowType inputRowType) { + this.inputRowType = inputRowType; + } + + public SeaTunnelDataType getExpressionType(Expression expression) { + if (expression instanceof NullValue) { + return BasicType.VOID_TYPE; + } + if (expression instanceof DoubleValue) { + return BasicType.DOUBLE_TYPE; + } + if (expression instanceof LongValue) { + long longVal = ((LongValue) expression).getValue(); + if (longVal <= Integer.MAX_VALUE && longVal >= Integer.MIN_VALUE) { + return BasicType.INT_TYPE; + } + return BasicType.LONG_TYPE; + } + if (expression instanceof StringValue) { + return BasicType.STRING_TYPE; + } + if (expression instanceof Column) { + String columnName = ((Column) expression).getColumnName(); + return inputRowType.getFieldType(inputRowType.indexOf(columnName)); + } + if (expression instanceof Function) { + return getFunctionType((Function) expression); + } + if (expression instanceof TimeKeyExpression) { + return getTimeKeyExprType((TimeKeyExpression) expression); + } + if (expression instanceof ExtractExpression) { + return BasicType.INT_TYPE; + } + if (expression instanceof Parenthesis) { + Parenthesis parenthesis = (Parenthesis) expression; + return getExpressionType(parenthesis.getExpression()); + } + if (expression instanceof Concat) { + return BasicType.STRING_TYPE; + } + if (expression instanceof CastExpression) { + return getCastType((CastExpression) expression); + } + if (expression instanceof BinaryExpression) { + BinaryExpression binaryExpression = (BinaryExpression) expression; + SeaTunnelDataType leftType = getExpressionType(binaryExpression.getLeftExpression()); + SeaTunnelDataType rightType = + getExpressionType(binaryExpression.getRightExpression()); + if (leftType.getSqlType() == SqlType.INT && rightType.getSqlType() == SqlType.INT) { + return BasicType.INT_TYPE; + } + if ((leftType.getSqlType() == SqlType.INT || leftType.getSqlType() == SqlType.BIGINT) + && (rightType.getSqlType() == SqlType.INT + || rightType.getSqlType() == SqlType.BIGINT)) { + return BasicType.LONG_TYPE; + } + if (leftType.getSqlType() == SqlType.DECIMAL + || rightType.getSqlType() == SqlType.DECIMAL) { + int precision = 0; + int scale = 0; + if (leftType.getSqlType() == SqlType.DECIMAL) { + DecimalType decimalType = (DecimalType) leftType; + precision = decimalType.getPrecision(); + scale = decimalType.getScale(); + } + if (rightType.getSqlType() == SqlType.DECIMAL) { + DecimalType decimalType = (DecimalType) rightType; + precision = Math.max(decimalType.getPrecision(), precision); + scale = Math.max(decimalType.getScale(), scale); + } + return new DecimalType(precision, scale); + } + if ((leftType.getSqlType() == SqlType.FLOAT || leftType.getSqlType() == SqlType.DOUBLE) + || (rightType.getSqlType() == SqlType.FLOAT + || rightType.getSqlType() == SqlType.DOUBLE)) { + return BasicType.DOUBLE_TYPE; + } + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported SQL Expression: %s ", expression.toString())); + } + + private SeaTunnelDataType getCastType(CastExpression castExpression) { + String dataType = castExpression.getType().getDataType(); + switch (dataType.toUpperCase()) { + case DECIMAL: + List ps = castExpression.getType().getArgumentsStringList(); + return new DecimalType(Integer.parseInt(ps.get(0)), Integer.parseInt(ps.get(1))); + case VARCHAR: + case STRING: + return BasicType.STRING_TYPE; + case INT: + case INTEGER: + return BasicType.INT_TYPE; + case BIGINT: + case LONG: + return BasicType.LONG_TYPE; + case BYTE: + return BasicType.BYTE_TYPE; + case DOUBLE: + return BasicType.DOUBLE_TYPE; + case FLOAT: + return BasicType.FLOAT_TYPE; + case TIMESTAMP: + case DATETIME: + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case DATE: + return LocalTimeType.LOCAL_DATE_TYPE; + case TIME: + return LocalTimeType.LOCAL_TIME_TYPE; + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported cast as type: %s", dataType)); + } + } + + private SeaTunnelDataType getFunctionType(Function function) { + switch (function.getName().toUpperCase()) { + case ZetaSQLFunction.CHAR: + case ZetaSQLFunction.CHR: + case ZetaSQLFunction.CONCAT: + case ZetaSQLFunction.CONCAT_WS: + case ZetaSQLFunction.HEXTORAW: + case ZetaSQLFunction.RAWTOHEX: + case ZetaSQLFunction.INSERT: + case ZetaSQLFunction.LOWER: + case ZetaSQLFunction.LCASE: + case ZetaSQLFunction.UPPER: + case ZetaSQLFunction.UCASE: + case ZetaSQLFunction.LEFT: + case ZetaSQLFunction.RIGHT: + case ZetaSQLFunction.LPAD: + case ZetaSQLFunction.RPAD: + case ZetaSQLFunction.LTRIM: + case ZetaSQLFunction.RTRIM: + case ZetaSQLFunction.TRIM: + case ZetaSQLFunction.REGEXP_REPLACE: + case ZetaSQLFunction.REGEXP_SUBSTR: + case ZetaSQLFunction.REPEAT: + case ZetaSQLFunction.REPLACE: + case ZetaSQLFunction.SOUNDEX: + case ZetaSQLFunction.SPACE: + case ZetaSQLFunction.SUBSTRING: + case ZetaSQLFunction.SUBSTR: + case ZetaSQLFunction.TO_CHAR: + case ZetaSQLFunction.TRANSLATE: + case ZetaSQLFunction.DAYNAME: + case ZetaSQLFunction.MONTHNAME: + case ZetaSQLFunction.FORMATDATETIME: + return BasicType.STRING_TYPE; + case ZetaSQLFunction.ASCII: + case ZetaSQLFunction.LOCATE: + case ZetaSQLFunction.INSTR: + case ZetaSQLFunction.POSITION: + case ZetaSQLFunction.CEIL: + case ZetaSQLFunction.CEILING: + case ZetaSQLFunction.FLOOR: + case ZetaSQLFunction.DAY_OF_MONTH: + case ZetaSQLFunction.DAY_OF_WEEK: + case ZetaSQLFunction.DAY_OF_YEAR: + case ZetaSQLFunction.EXTRACT: + case ZetaSQLFunction.HOUR: + case ZetaSQLFunction.MINUTE: + case ZetaSQLFunction.MONTH: + case ZetaSQLFunction.QUARTER: + case ZetaSQLFunction.SECOND: + case ZetaSQLFunction.WEEK: + case ZetaSQLFunction.YEAR: + case ZetaSQLFunction.SIGN: + return BasicType.INT_TYPE; + case ZetaSQLFunction.BIT_LENGTH: + case ZetaSQLFunction.CHAR_LENGTH: + case ZetaSQLFunction.LENGTH: + case ZetaSQLFunction.OCTET_LENGTH: + case ZetaSQLFunction.DATEDIFF: + return BasicType.LONG_TYPE; + case ZetaSQLFunction.REGEXP_LIKE: + return BasicType.BOOLEAN_TYPE; + case ZetaSQLFunction.ACOS: + case ZetaSQLFunction.ASIN: + case ZetaSQLFunction.ATAN: + case ZetaSQLFunction.COS: + case ZetaSQLFunction.COSH: + case ZetaSQLFunction.COT: + case ZetaSQLFunction.SIN: + case ZetaSQLFunction.SINH: + case ZetaSQLFunction.TAN: + case ZetaSQLFunction.TANH: + case ZetaSQLFunction.ATAN2: + case ZetaSQLFunction.EXP: + case ZetaSQLFunction.LN: + case ZetaSQLFunction.LOG: + case ZetaSQLFunction.LOG10: + case ZetaSQLFunction.RADIANS: + case ZetaSQLFunction.SQRT: + case ZetaSQLFunction.PI: + case ZetaSQLFunction.POWER: + case ZetaSQLFunction.RAND: + case ZetaSQLFunction.RANDOM: + case ZetaSQLFunction.TRUNC: + case ZetaSQLFunction.TRUNCATE: + return BasicType.DOUBLE_TYPE; + case ZetaSQLFunction.NOW: + case ZetaSQLFunction.DATE_TRUNC: + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case ZetaSQLFunction.PARSEDATETIME: + case ZetaSQLFunction.TO_DATE: + { + String format = function.getParameters().getExpressions().get(1).toString(); + if (format.contains("yy") && format.contains("mm")) { + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + } + if (format.contains("yy")) { + return LocalTimeType.LOCAL_DATE_TYPE; + } + if (format.contains("mm")) { + return LocalTimeType.LOCAL_TIME_TYPE; + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unknown pattern letter %s for function: %s", + format, function.getName())); + } + case ZetaSQLFunction.ABS: + case ZetaSQLFunction.DATEADD: + case ZetaSQLFunction.TIMESTAMPADD: + case ZetaSQLFunction.ROUND: + case ZetaSQLFunction.NULLIF: + case ZetaSQLFunction.COALESCE: + case ZetaSQLFunction.IFNULL: + // Result has the same type as first argument + return getExpressionType(function.getParameters().getExpressions().get(0)); + case ZetaSQLFunction.MOD: + // Result has the same type as second argument + return getExpressionType(function.getParameters().getExpressions().get(1)); + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Function: %s unsupported", function.getName())); + } + } + + private SeaTunnelDataType getTimeKeyExprType(TimeKeyExpression timeKeyExpression) { + switch (timeKeyExpression.getStringValue().toUpperCase()) { + case ZetaSQLFunction.CURRENT_DATE: + case ZetaSQLFunction.CURRENT_DATE_P: + return LocalTimeType.LOCAL_DATE_TYPE; + case ZetaSQLFunction.CURRENT_TIME: + case ZetaSQLFunction.CURRENT_TIME_P: + return LocalTimeType.LOCAL_TIME_TYPE; + case ZetaSQLFunction.CURRENT_TIMESTAMP: + case ZetaSQLFunction.CURRENT_TIMESTAMP_P: + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "TimeKeyExpression: %s unsupported", + timeKeyExpression.getStringValue())); + } + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java new file mode 100644 index 00000000000..04d12683c96 --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; +import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngine; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.AllColumns; +import net.sf.jsqlparser.statement.select.FromItem; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectExpressionItem; +import net.sf.jsqlparser.statement.select.SelectItem; + +import java.util.List; + +public class ZetaSimpleSQLEngine implements SimpleSQLEngine { + private String inputTableName; + + private String sql; + private PlainSelect selectBody; + + private ZetaSQLFunction zetaSQLFunction; + private ZetaSQLFilter zetaSQLFilter; + private ZetaSQLType zetaSQLType; + + public ZetaSimpleSQLEngine() {} + + @Override + public void init(String inputTableName, SeaTunnelRowType inputRowType, String sql) { + this.inputTableName = inputTableName; + this.sql = sql; + + this.zetaSQLType = new ZetaSQLType(inputRowType); + this.zetaSQLFunction = new ZetaSQLFunction(inputRowType, zetaSQLType); + this.zetaSQLFilter = new ZetaSQLFilter(zetaSQLFunction); + + parseSQL(); + } + + private void parseSQL() { + try { + Statement statement = CCJSqlParserUtil.parse(sql); + // validate SQL statement + validateSQL(statement); + this.selectBody = (PlainSelect) ((Select) statement).getSelectBody(); + } catch (JSQLParserException e) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("SQL parse failed: %s, cause: %s", sql, e.getMessage())); + } + } + + private void validateSQL(Statement statement) { + try { + if (!(statement instanceof Select)) { + throw new IllegalArgumentException("Only supported DQL(select) SQL"); + } + Select select = (Select) statement; + if (!(select.getSelectBody() instanceof PlainSelect)) { + throw new IllegalArgumentException("Unsupported simple SQL: %s"); + } + if (!(select.getSelectBody() instanceof PlainSelect)) { + throw new IllegalArgumentException("Unsupported simple SQL: %s"); + } + PlainSelect selectBody = (PlainSelect) select.getSelectBody(); + + FromItem fromItem = selectBody.getFromItem(); + if (fromItem instanceof Table) { + Table table = (Table) fromItem; + if (table.getSchemaName() != null) { + throw new IllegalArgumentException("Unsupported schema syntax"); + } + if (table.getAlias() != null) { + throw new IllegalArgumentException("Unsupported table alias name syntax"); + } + String tableName = table.getName(); + if (!inputTableName.equalsIgnoreCase(tableName)) { + throw new IllegalArgumentException( + String.format("Table name: %s not found", tableName)); + } + } else { + throw new IllegalArgumentException("Unsupported sub table syntax"); + } + + if (selectBody.getJoins() != null) { + throw new IllegalArgumentException("Unsupported table join syntax"); + } + + if (selectBody.getOrderByElements() != null) { + throw new IllegalArgumentException("Unsupported ORDER BY syntax"); + } + + if (selectBody.getGroupBy() != null) { + throw new IllegalArgumentException("Unsupported GROUP BY syntax"); + } + + if (selectBody.getLimit() != null || selectBody.getOffset() != null) { + throw new IllegalArgumentException("Unsupported LIMIT,OFFSET syntax"); + } + + for (SelectItem selectItem : selectBody.getSelectItems()) { + if (selectItem instanceof AllColumns) { + throw new IllegalArgumentException("Unsupported all columns select syntax"); + } + } + } catch (Exception e) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("SQL validate failed: %s, cause: %s", sql, e.getMessage())); + } + } + + @Override + public SeaTunnelRowType typeMapping() { + List selectItems = selectBody.getSelectItems(); + + String[] fieldNames = new String[selectItems.size()]; + SeaTunnelDataType[] seaTunnelDataTypes = new SeaTunnelDataType[selectItems.size()]; + + for (int i = 0; i < selectItems.size(); i++) { + SelectItem selectItem = selectItems.get(i); + if (selectItem instanceof SelectExpressionItem) { + SelectExpressionItem expressionItem = (SelectExpressionItem) selectItem; + Expression expression = expressionItem.getExpression(); + + if (expressionItem.getAlias() != null) { + fieldNames[i] = expressionItem.getAlias().getName(); + } else { + if (expression instanceof Column) { + fieldNames[i] = ((Column) expression).getColumnName(); + } else { + fieldNames[i] = expression.toString(); + } + } + + seaTunnelDataTypes[i] = zetaSQLType.getExpressionType(expression); + } + } + return new SeaTunnelRowType(fieldNames, seaTunnelDataTypes); + } + + @Override + public SeaTunnelRow transformBySQL(SeaTunnelRow inputRow) { + // ------Physical Query Plan Execution------ + // Scan Table + Object[] inputFields = scanTable(inputRow); + + // Filter + boolean retain = zetaSQLFilter.executeFilter(selectBody.getWhere(), inputFields); + if (!retain) { + return null; + } + + // Project + Object[] outputFields = project(inputFields); + + return new SeaTunnelRow(outputFields); + } + + private Object[] scanTable(SeaTunnelRow inputRow) { + // do nothing, only return the input fields + return inputRow.getFields(); + } + + private Object[] project(Object[] inputFields) { + List selectItems = selectBody.getSelectItems(); + Object[] fields = new Object[selectItems.size()]; + for (int i = 0; i < selectItems.size(); i++) { + SelectItem selectItem = selectItems.get(i); + SelectExpressionItem expressionItem = (SelectExpressionItem) selectItem; + Expression expression = expressionItem.getExpression(); + fields[i] = zetaSQLFunction.computeForValue(expression, inputFields); + } + return fields; + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/DateTimeFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/DateTimeFunction.java new file mode 100644 index 00000000000..89ee452be0c --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/DateTimeFunction.java @@ -0,0 +1,544 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta.functions; + +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; +import org.apache.seatunnel.transform.sqlengine.zeta.ZetaSQLFunction; + +import java.text.DateFormatSymbols; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.Period; +import java.time.format.DateTimeFormatter; +import java.time.temporal.Temporal; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.WeekFields; +import java.util.List; +import java.util.Locale; + +public class DateTimeFunction { + /** English names of months and week days. */ + private static volatile String[][] MONTHS_AND_WEEKS; + + public static LocalDate currentDate() { + return LocalDate.now(); + } + + public static LocalTime currentTime() { + return LocalTime.now(); + } + + public static LocalDateTime currentTimestamp() { + return LocalDateTime.now(); + } + + public static Object dateadd(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + long count = ((Number) args.get(1)).longValue(); + String datetimeField = "DAY"; + if (args.size() >= 3) { + String df = (String) args.get(2); + if (df != null) { + datetimeField = df.toUpperCase(); + } + } + switch (datetimeField) { + case "YEAR": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).plusYears(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusYears(count); + } + break; + case "MONTH": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).plusMonths(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusMonths(count); + } + break; + case "WEEK": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).plusWeeks(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusWeeks(count); + } + break; + case "DAY": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).plusDays(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusDays(count); + } + break; + case "HOUR": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).plusHours(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusHours(count); + } + break; + case "MINUTE": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).plusMinutes(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusMinutes(count); + } + break; + case "SECOND": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).plusSeconds(count); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusSeconds(count); + } + break; + case "MILLISECOND": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).plusNanos(count * 1000_000L); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).plusNanos(count * 1000_000L); + } + break; + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported dateTimeField: %s for function: %s", + datetimeField, ZetaSQLFunction.DATEDIFF)); + } + return datetime; + } + + public static Long datediff(List args) { + Temporal datetime1 = (Temporal) args.get(0); + if (datetime1 == null) { + return null; + } + Temporal datetime2 = (Temporal) args.get(1); + if (datetime2 == null) { + return null; + } + String datetimeField = "DAY"; + if (args.size() >= 3) { + String df = (String) args.get(2); + if (df != null) { + datetimeField = df.toUpperCase(); + } + } + + LocalDate date1 = null; + LocalDate date2 = null; + if ("YEAR".equals(datetimeField) + || "MONTH".equals(datetimeField) + || "DAY".equals(datetimeField)) { + if (datetime1 instanceof LocalDateTime) { + date1 = ((LocalDateTime) datetime1).toLocalDate(); + } + if (datetime1 instanceof LocalDate) { + date1 = (LocalDate) datetime1; + } + if (datetime2 instanceof LocalDateTime) { + date2 = ((LocalDateTime) datetime2).toLocalDate(); + } + if (datetime2 instanceof LocalDate) { + date2 = (LocalDate) datetime2; + } + } + + switch (datetimeField) { + case "YEAR": + if (date1 != null && date2 != null) { + return (long) Period.between(date1, date2).getYears(); + } + break; + case "MONTH": + if (date1 != null && date2 != null) { + return (long) Period.between(date1, date2).getMonths(); + } + break; + case "WEEK": + return Duration.between(datetime1, datetime2).toDays() / 7L; + case "DAY": + if (date1 != null && date2 != null) { + LocalTime lt = LocalTime.of(0, 0, 0); + LocalDateTime d1 = LocalDateTime.of(date1, lt); + LocalDateTime d2 = LocalDateTime.of(date2, lt); + return Duration.between(d1, d2).toDays(); + } + break; + case "DAYTIME": + return Duration.between(datetime1, datetime2).toDays(); + case "HOUR": + return Duration.between(datetime1, datetime2).toHours(); + case "MINUTE": + return Duration.between(datetime1, datetime2).toMinutes(); + case "SECOND": + return Duration.between(datetime1, datetime2).toMillis() / 1000L; + case "MILLISECOND": + return Duration.between(datetime1, datetime2).toMillis(); + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported dateTimeField: %s for function: %s", + datetimeField, ZetaSQLFunction.DATEDIFF)); + } + return null; + } + + public static LocalDateTime dateTrunc(List args) { + LocalDateTime datetime = (LocalDateTime) args.get(0); + if (datetime == null) { + return null; + } + String datetimeField = "DAY"; + if (args.size() >= 2) { + String df = (String) args.get(1); + if (df != null) { + datetimeField = df.toUpperCase(); + } + } + int year = datetime.getYear(); + int month = datetime.getMonthValue(); + int day = datetime.getDayOfMonth(); + int hour = datetime.getHour(); + int minute = datetime.getMinute(); + int second = datetime.getSecond(); + + switch (datetimeField) { + case "YEAR": + month = 1; + day = 1; + hour = 0; + minute = 0; + second = 0; + break; + case "MONTH": + day = 1; + hour = 0; + minute = 0; + second = 0; + break; + case "DAY": + hour = 0; + minute = 0; + second = 0; + break; + case "HOUR": + minute = 0; + second = 0; + break; + case "MINUTE": + second = 0; + break; + case "SECOND": + break; + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported dateTimeField: %s for function: %s", + datetimeField, ZetaSQLFunction.DATEDIFF)); + } + + return LocalDateTime.of(year, month, day, hour, minute, second); + } + + public static String dayname(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + int dow = localDate.getDayOfWeek().getValue(); + dow++; + if (dow == 8) { + dow = 1; + } + return getMonthsAndWeeks(1)[dow]; + } + + private static String[] getMonthsAndWeeks(int field) { + String[][] result = MONTHS_AND_WEEKS; + if (result == null) { + result = new String[2][]; + DateFormatSymbols dfs = DateFormatSymbols.getInstance(Locale.ENGLISH); + result[0] = dfs.getMonths(); + result[1] = dfs.getWeekdays(); + MONTHS_AND_WEEKS = result; + } + return result[field]; + } + + private static LocalDate convertToLocalDate(Temporal datetime) { + LocalDate localDate = null; + if (datetime instanceof LocalDateTime) { + localDate = ((LocalDateTime) datetime).toLocalDate(); + } else if (datetime instanceof LocalDate) { + localDate = (LocalDate) datetime; + } + return localDate; + } + + public static Integer dayOfMonth(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + return localDate.getDayOfMonth(); + } + + public static Integer dayOfWeek(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + return localDate.getDayOfWeek().getValue(); + } + + public static Integer dayOfYear(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + return localDate.getDayOfYear(); + } + + public static Integer extract(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + String datetimeField = (String) args.get(1); + switch (datetimeField.toUpperCase()) { + case "YEAR": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).getYear(); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getYear(); + } + break; + case "MONTH": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).getMonthValue(); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getMonthValue(); + } + break; + case "DAY": + if (datetime instanceof LocalDate) { + return ((LocalDate) datetime).getDayOfMonth(); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getDayOfMonth(); + } + break; + case "HOUR": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).getHour(); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getHour(); + } + break; + case "MINUTE": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).getMinute(); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getMinute(); + } + break; + case "SECOND": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).getSecond(); + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getSecond(); + } + break; + case "MILLISECOND": + if (datetime instanceof LocalTime) { + return ((LocalTime) datetime).getNano() / 1000_000; + } + if (datetime instanceof LocalDateTime) { + return ((LocalDateTime) datetime).getNano() / 1000_000; + } + break; + case "DAYOFWEEK": + return dayOfWeek(args); + case "DAYOFYEAR": + return dayOfYear(args); + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported dateTimeField: %s for function: %s", + datetimeField, ZetaSQLFunction.EXTRACT)); + } + return null; + } + + public static String formatdatetime(List args) { + TemporalAccessor datetime = (TemporalAccessor) args.get(0); + if (datetime == null) { + return null; + } + String format = (String) args.get(1); + DateTimeFormatter df = DateTimeFormatter.ofPattern(format); + return df.format(datetime); + } + + public static Integer hour(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalTime localTime = convertToLocalTime(datetime); + return localTime.getHour(); + } + + private static LocalTime convertToLocalTime(Temporal datetime) { + LocalTime localTime = null; + if (datetime instanceof LocalDateTime) { + localTime = ((LocalDateTime) datetime).toLocalTime(); + } else if (datetime instanceof LocalTime) { + localTime = (LocalTime) datetime; + } + return localTime; + } + + public static Integer minute(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalTime localTime = convertToLocalTime(datetime); + return localTime.getMinute(); + } + + public static Integer month(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + return localDate.getMonthValue(); + } + + public static String monthname(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + int dow = localDate.getMonthValue(); + return getMonthsAndWeeks(0)[dow - 1]; + } + + public static Temporal parsedatetime(List args) { + String str = (String) args.get(0); + if (str == null) { + return null; + } + String format = (String) args.get(1); + if (format.contains("yy") && format.contains("mm")) { + DateTimeFormatter df = DateTimeFormatter.ofPattern(format); + return LocalDateTime.parse(str, df); + } + if (format.contains("yy")) { + DateTimeFormatter df = DateTimeFormatter.ofPattern(format); + return LocalDate.parse(str, df); + } + if (format.contains("mm")) { + DateTimeFormatter df = DateTimeFormatter.ofPattern(format); + return LocalTime.parse(str, df); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unknown pattern letter %s for function: %s", + format, ZetaSQLFunction.PARSEDATETIME)); + } + + public static Integer quarter(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + int month = localDate.getMonthValue(); + if (month <= 3) { + return 1; + } + if (month <= 6) { + return 2; + } + if (month <= 9) { + return 3; + } + return 4; + } + + public static Integer second(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalTime localTime = convertToLocalTime(datetime); + return localTime.getSecond(); + } + + public static Integer week(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + WeekFields weekFields = WeekFields.ISO; + return localDate.get(weekFields.weekOfYear()) + 1; + } + + public static Integer year(List args) { + Temporal datetime = (Temporal) args.get(0); + if (datetime == null) { + return null; + } + LocalDate localDate = convertToLocalDate(datetime); + return localDate.getYear(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java new file mode 100644 index 00000000000..6d4e5ae8dca --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java @@ -0,0 +1,448 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta.functions; + +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; +import org.apache.seatunnel.transform.sqlengine.zeta.ZetaSQLFunction; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.List; +import java.util.Random; + +public class NumericFunction { + public static Number abs(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + if (arg instanceof Integer) { + return Math.abs(arg.intValue()); + } + if (arg instanceof Long) { + return Math.abs(arg.longValue()); + } + if (arg instanceof Float) { + return Math.abs(arg.floatValue()); + } + if (arg instanceof Double) { + return Math.abs(arg.doubleValue()); + } + if (arg instanceof BigDecimal) { + return ((BigDecimal) arg).abs(); + } + + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported arg type %s of function %s", + arg.getClass().getName(), ZetaSQLFunction.ABS)); + } + + public static Double acos(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.acos(arg.doubleValue()); + } + + public static Double asin(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.asin(arg.doubleValue()); + } + + public static Double atan(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.atan(arg.doubleValue()); + } + + public static Double cos(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.cos(arg.doubleValue()); + } + + public static Double cosh(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.cosh(arg.doubleValue()); + } + + public static Double cot(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + double d = Math.tan(arg.doubleValue()); + if (d == 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Division by zero for function: %s", ZetaSQLFunction.COT)); + } + return 1d / d; + } + + public static Double sin(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.sin(arg.doubleValue()); + } + + public static Double sinh(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.sinh(arg.doubleValue()); + } + + public static Double tan(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.tan(arg.doubleValue()); + } + + public static Double tanh(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + return Math.tanh(arg.doubleValue()); + } + + public static Double atan2(List args) { + Number arg = (Number) args.get(0); + if (arg == null) { + return null; + } + Number arg2 = (Number) args.get(1); + if (arg2 == null) { + return null; + } + return Math.atan2(arg.doubleValue(), arg2.doubleValue()); + } + + public static Number mod(List args) { + Number leftValue = (Number) args.get(0); + if (leftValue == null) { + return null; + } + Number rightValue = (Number) args.get(1); + if (rightValue == null) { + return null; + } + if (rightValue.doubleValue() == 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Mod by zero for function: %s", ZetaSQLFunction.MOD)); + } + BigDecimal leftBD = BigDecimal.valueOf(leftValue.doubleValue()); + BigDecimal rightBD = BigDecimal.valueOf(rightValue.doubleValue()); + BigDecimal[] res = leftBD.divideAndRemainder(rightBD); + if (rightValue instanceof Integer) { + return res[1].intValue(); + } + if (rightValue instanceof Long) { + return res[1].longValue(); + } + if (rightValue instanceof Float) { + return res[1].floatValue(); + } + if (rightValue instanceof Double) { + return res[1].doubleValue(); + } + if (rightValue instanceof BigDecimal) { + return res[1]; + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported arg type %s of function %s", + rightValue.getClass().getName(), ZetaSQLFunction.MOD)); + } + + public static Integer ceil(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + Number v2 = null; + if (args.size() >= 2) { + v2 = (Number) args.get(1); + } + return round(v1, v2, RoundingMode.CEILING).intValue(); + } + + private static Number round(Number v1, Number v2, RoundingMode roundingMode) { + int scale = v2 != null ? v2.intValue() : 0; + String t = v1.getClass().getSimpleName(); + c: + switch (t.toUpperCase()) { + case "INTEGER": + case "SHOT": + case "LONG": + { + if (scale < 0) { + long original = v1.longValue(); + long scaled = + BigDecimal.valueOf(original) + .setScale(scale, roundingMode) + .longValue(); + if (original != scaled) { + v1 = convertTo(t, scaled); + } + } + break; + } + case "BIGDECIMAL": + { + BigDecimal bd = BigDecimal.valueOf(v1.doubleValue()); + v1 = bd.setScale(scale, roundingMode); + break; + } + case "DOUBLE": + case "FLOAT": + { + l: + if (scale == 0) { + double d; + switch (roundingMode) { + case DOWN: + d = v1.doubleValue(); + d = d < 0 ? Math.ceil(d) : Math.floor(d); + break; + case CEILING: + d = Math.ceil(v1.doubleValue()); + break; + case FLOOR: + d = Math.floor(v1.doubleValue()); + break; + default: + break l; + } + v1 = t.equals("FLOAT") ? (float) d : d; + break c; + } + BigDecimal bd = + BigDecimal.valueOf(v1.doubleValue()).setScale(scale, roundingMode); + v1 = t.equals("FLOAT") ? bd.floatValue() : bd.doubleValue(); + break; + } + } + return v1; + } + + private static Number convertTo(String valueType, Number column) { + switch (valueType) { + case "INTEGER": + return column.intValue(); + case "SHOT": + return column.shortValue(); + case "LONG": + return column.longValue(); + default: + throw new IllegalArgumentException(); + } + } + + public static Double exp(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + return Math.exp(v1.doubleValue()); + } + + public static Integer floor(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + Number v2 = null; + if (args.size() >= 2) { + v2 = (Number) args.get(1); + } + return round(v1, v2, RoundingMode.FLOOR).intValue(); + } + + public static Double ln(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + if (v1.doubleValue() <= 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported function LN() argument: %s", v1)); + } + return Math.log(v1.doubleValue()); + } + + public static Double log(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + if (v1.doubleValue() <= 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported function LOG() base: %s", v1)); + } + Number v2 = (Number) args.get(1); + if (v2 == null) { + return null; + } + if (v2.doubleValue() <= 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported function LOG() argument: %s", v1)); + } + if (v1.doubleValue() == Math.E) { + return Math.log(v2.doubleValue()); + } else if (v1.doubleValue() == 10d) { + return Math.log10(v2.doubleValue()); + } else { + return Math.log(v2.doubleValue()) / Math.log(v1.doubleValue()); + } + } + + public static Double log10(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + if (v1.doubleValue() <= 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported function LOG10() argument: %s", v1)); + } + return Math.log10(v1.doubleValue()); + } + + public static Double radians(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + return Math.toRadians(v1.doubleValue()); + } + + public static Double sqrt(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + return Math.sqrt(v1.doubleValue()); + } + + public static Double pi(List args) { + return Math.PI; + } + + public static Double power(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + Number v2 = (Number) args.get(1); + if (v2 == null) { + return null; + } + return Math.pow(v1.doubleValue(), v2.doubleValue()); + } + + public static Double random(List args) { + Random random = new Random(); + if (args.size() >= 1) { + Number v1 = (Number) args.get(0); + if (v1 != null) { + random.setSeed(v1.intValue()); + } + } + return random.nextDouble(); + } + + public static Number round(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + Number v2 = null; + if (args.size() >= 2) { + v2 = (Number) args.get(1); + } + return round(v1, v2, RoundingMode.HALF_UP); + } + + public static int sign(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return 0; + } + if (v1 instanceof Integer) { + return Integer.signum((Integer) v1); + } + if (v1 instanceof Long) { + return Long.signum((Long) v1); + } + if (v1 instanceof Double) { + double value = (Double) v1; + return value == 0 || Double.isNaN(value) ? 0 : value < 0 ? -1 : 1; + } + if (v1 instanceof Float) { + float value = (Float) v1; + return value == 0 || Float.isNaN(value) ? 0 : value < 0 ? -1 : 1; + } + if (v1 instanceof BigDecimal) { + double value = v1.doubleValue(); + return value == 0 || Double.isNaN(value) ? 0 : value < 0 ? -1 : 1; + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported function SIGN() argument type: %s", v1.getClass().getName())); + } + + public static Number trunc(List args) { + Number v1 = (Number) args.get(0); + if (v1 == null) { + return null; + } + Number v2 = null; + if (args.size() >= 2) { + v2 = (Number) args.get(1); + } + return round(v1, v2, RoundingMode.DOWN); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/StringFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/StringFunction.java new file mode 100644 index 00000000000..57e83dd275f --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/StringFunction.java @@ -0,0 +1,667 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta.functions; + +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; +import org.apache.seatunnel.transform.sqlengine.zeta.ZetaSQLFunction; + +import java.nio.charset.StandardCharsets; +import java.time.temporal.Temporal; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class StringFunction { + private static final byte[] SOUNDEX_INDEX = + "71237128722455712623718272\000\000\000\000\000\00071237128722455712623718272" + .getBytes(StandardCharsets.ISO_8859_1); + + public static Integer ascii(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } else { + return (int) arg.charAt(0); + } + } + + public static Long bitLength(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + return arg.getBytes(StandardCharsets.UTF_8).length * 8L; + } + + public static Long charLength(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + return (long) arg.length(); + } + + public static Long octetLength(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + return (long) arg.getBytes(StandardCharsets.UTF_8).length; + } + + public static String chr(List args) { + Object arg = args.get(0); + if (arg == null) { + return null; + } + return String.valueOf((char) ((Number) arg).intValue()); + } + + public static String concat(List args) { + int i = 0; + StringBuilder builder = new StringBuilder(); + for (int l = args.size(); i < l; i++) { + Object v = args.get(i); + if (v == null) { + continue; + } + builder.append(v); + } + return builder.toString(); + } + + public static String concatWs(List args) { + int i = 1; + String separator = (String) args.get(0); + StringBuilder builder = new StringBuilder(); + boolean f = false; + for (int l = args.size(); i < l; i++) { + Object arg = args.get(i); + if (arg == null) { + continue; + } + if (separator != null) { + if (f) { + builder.append(separator); + } + f = true; + } + builder.append(arg); + } + return builder.toString(); + } + + public static String hextoraw(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + int len = arg.length(); + if (len % 4 != 0) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported arg for function: %s", ZetaSQLFunction.HEXTORAW)); + } + StringBuilder builder = new StringBuilder(len / 4); + for (int i = 0; i < len; i += 4) { + builder.append((char) Integer.parseInt(arg.substring(i, i + 4), 16)); + } + return builder.toString(); + } + + public static String rawtohex(List args) { + Object arg = args.get(0); + if (arg == null) { + return null; + } + if (arg instanceof byte[]) { + int len = ((byte[]) arg).length; + byte[] bytes = new byte[len * 2]; + char[] hex = "0123456789abcdef".toCharArray(); + for (int i = 0, j = 0; i < len; i++) { + int c = ((byte[]) arg)[i] & 0xff; + bytes[j++] = (byte) hex[c >> 4]; + bytes[j++] = (byte) hex[c & 0xf]; + } + return new String(bytes, StandardCharsets.ISO_8859_1); + } + String s = arg.toString(); + + int length = s.length(); + StringBuilder buff = new StringBuilder(4 * length); + for (int i = 0; i < length; i++) { + String hex = Integer.toHexString(s.charAt(i) & 0xffff); + for (int j = hex.length(); j < 4; j++) { + buff.append('0'); + } + buff.append(hex); + } + return buff.toString(); + } + + public static String insert(List args) { + String s1 = (String) args.get(0); + int start = ((Number) args.get(1)).intValue(); + int length = ((Number) args.get(2)).intValue(); + String s2 = (String) args.get(3); + if (s1 == null) { + return s2; + } + if (s2 == null) { + return s1; + } + int len1 = s1.length(); + int len2 = s2.length(); + start--; + if (start < 0 || length <= 0 || len2 == 0 || start > len1) { + return s1; + } + if (start + length > len1) { + length = len1 - start; + } + return s1.substring(0, start) + s2 + s1.substring(start + length); + } + + public static String lower(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + return arg.toLowerCase(); + } + + public static String upper(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + return arg.toUpperCase(); + } + + public static String left(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + int count = ((Number) args.get(1)).intValue(); + if (count > arg.length()) { + count = arg.length(); + } + return arg.substring(0, count); + } + + public static String right(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + int count = ((Number) args.get(1)).intValue(); + int length = arg.length(); + if (count > length) { + count = length; + } + return arg.substring(length - count); + } + + public static Integer location(String functionName, List args) { + String search = (String) args.get(0); + String s = (String) args.get(1); + if (s == null) { + return 0; + } + int start = 1; + if (args.size() == 3 && functionName.equalsIgnoreCase(ZetaSQLFunction.LOCATE)) { + start = ((Number) args.get(2)).intValue(); + } + if (start < 0) { + return s.lastIndexOf(search, s.length() + start) + 1; + } + return s.indexOf(search, start == 0 ? 0 : start - 1) + 1; + } + + public static Integer instr(List args) { + String s = (String) args.get(0); + if (s == null) { + return 0; + } + String search = (String) args.get(1); + int start = 1; + if (args.size() == 3) { + start = ((Number) args.get(2)).intValue(); + } + if (start < 0) { + return s.lastIndexOf(search, s.length() + start) + 1; + } + return s.indexOf(search, start == 0 ? 0 : start - 1) + 1; + } + + public static String pad(String functionName, List args) { + String padding; + if (args.size() >= 3) { + padding = (String) args.get(2); + } else { + padding = null; + } + String v1 = (String) args.get(0); + if (v1 == null) { + return null; + } + int v2 = ((Number) args.get(1)).intValue(); + return pad(v1, v2, padding, functionName.equalsIgnoreCase(ZetaSQLFunction.RPAD)); + } + + public static String pad(String string, int n, String padding, boolean right) { + if (n < 0) { + n = 0; + } + if (n < string.length()) { + return string.substring(0, n); + } else if (n == string.length()) { + return string; + } + char paddingChar; + if (padding == null || padding.isEmpty()) { + paddingChar = ' '; + } else { + paddingChar = padding.charAt(0); + } + StringBuilder buff = new StringBuilder(n); + n -= string.length(); + if (right) { + buff.append(string); + } + for (int i = 0; i < n; i++) { + buff.append(paddingChar); + } + if (!right) { + buff.append(string); + } + return buff.toString(); + } + + public static String ltrim(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + String sp = null; + if (args.size() >= 2) { + sp = (String) args.get(1); + } + return trim(arg, true, false, sp); + } + + public static String rtrim(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + String sp = null; + if (args.size() >= 2) { + sp = (String) args.get(1); + } + return trim(arg, false, true, sp); + } + + public static String trim(List args) { + String arg = (String) args.get(0); + if (arg == null) { + return null; + } + String sp = null; + if (args.size() >= 2) { + sp = (String) args.get(1); + } + return trim(arg, true, true, sp); + } + + public static String trim(String s, boolean leading, boolean trailing, String sp) { + char space = sp == null || sp.isEmpty() ? ' ' : sp.charAt(0); + int begin = 0, end = s.length(); + if (leading) { + while (begin < end && s.charAt(begin) == space) { + begin++; + } + } + if (trailing) { + while (end > begin && s.charAt(end - 1) == space) { + end--; + } + } + // substring() returns self if start == 0 && end == length() + return s.substring(begin, end); + } + + public static String regexpReplace(List args) { + String input = (String) args.get(0); + if (input == null) { + return null; + } + String regexp = (String) args.get(1); + String replacement = (String) args.get(2); + String regexpMode = null; + if (args.size() >= 4) { + regexpMode = (String) args.get(3); + } + return regexpReplace(input, regexp, replacement, 1, 0, regexpMode); + } + + private static String regexpReplace( + String input, + String regexp, + String replacement, + int position, + int occurrence, + String regexpMode) { + int flags = makeRegexpFlags(regexpMode, false); + Matcher matcher = + Pattern.compile(regexp, flags).matcher(input).region(position - 1, input.length()); + if (occurrence == 0) { + return matcher.replaceAll(replacement); + } else { + StringBuffer sb = new StringBuffer(); + int index = 1; + while (matcher.find()) { + if (index == occurrence) { + matcher.appendReplacement(sb, replacement); + break; + } + index++; + } + matcher.appendTail(sb); + return sb.toString(); + } + } + + public static Boolean regexpLike(List args) { + String input = (String) args.get(0); + if (input == null) { + return null; + } + String regexp = (String) args.get(1); + String regexpMode = null; + if (args.size() >= 3) { + regexpMode = (String) args.get(2); + } + int flags = makeRegexpFlags(regexpMode, false); + return Pattern.compile(regexp, flags).matcher(input).find(); + } + + private static int makeRegexpFlags(String stringFlags, boolean ignoreGlobalFlag) { + int flags = Pattern.UNICODE_CASE; + if (stringFlags != null) { + for (int i = 0; i < stringFlags.length(); ++i) { + switch (stringFlags.charAt(i)) { + case 'i': + flags |= Pattern.CASE_INSENSITIVE; + break; + case 'c': + flags &= ~Pattern.CASE_INSENSITIVE; + break; + case 'n': + flags |= Pattern.DOTALL; + break; + case 'm': + flags |= Pattern.MULTILINE; + break; + case 'g': + if (ignoreGlobalFlag) { + break; + } + // $FALL-THROUGH$ + default: + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format( + "Unsupported regexpMode arg: %s for function: %s", + flags, ZetaSQLFunction.HEXTORAW)); + } + } + } + return flags; + } + + public static String regexpSubstr(List args) { + String input = (String) args.get(0); + if (input == null) { + return null; + } + String regexp = (String) args.get(1); + if (args.size() == 2) { + return regexpSubstr(input, regexp, null, null, null, null); + } + if (args.size() >= 6) { + Integer positionArg = null; + if (args.get(2) != null) { + positionArg = ((Number) args.get(2)).intValue(); + } + Integer occurrenceArg = null; + if (args.get(3) != null) { + occurrenceArg = ((Number) args.get(3)).intValue(); + } + String regexpMode = (String) args.get(4); + Integer subexpressionArg = null; + if (args.get(5) != null) { + subexpressionArg = ((Number) args.get(5)).intValue(); + } + return regexpSubstr( + input, regexp, positionArg, occurrenceArg, regexpMode, subexpressionArg); + } + + return null; + } + + public static String regexpSubstr( + String input, + String regexp, + Integer positionArg, + Integer occurrenceArg, + String regexpMode, + Integer subexpressionArg) { + int position = positionArg != null ? positionArg - 1 : 0; + int requestedOccurrence = occurrenceArg != null ? occurrenceArg : 1; + int subexpression = subexpressionArg != null ? subexpressionArg : 0; + int flags = makeRegexpFlags(regexpMode, false); + Matcher m = Pattern.compile(regexp, flags).matcher(input); + + boolean found = m.find(position); + for (int occurrence = 1; occurrence < requestedOccurrence && found; occurrence++) { + found = m.find(); + } + + if (!found) { + return null; + } else { + return m.group(subexpression); + } + } + + public static String repeat(List args) { + String s = (String) args.get(0); + if (s == null) { + return null; + } + int count = ((Number) args.get(1)).intValue(); + if (count <= 0) { + return ""; + } + int length = s.length(); + StringBuilder builder = new StringBuilder(length * count); + while (count-- > 0) { + builder.append(s); + } + return builder.toString(); + } + + public static String replace(List args) { + String v1 = (String) args.get(0); + if (v1 == null) { + return null; + } + String v2 = (String) args.get(1); + String after; + if (args.size() >= 3) { + after = (String) args.get(2); + if (after == null) { + after = ""; + } + } else { + after = ""; + } + return replaceAll(v1, v2, after); + } + + public static String replaceAll(String s, String before, String after) { + int next = s.indexOf(before); + if (next < 0 || before.isEmpty()) { + return s; + } + StringBuilder buff = new StringBuilder(s.length() - before.length() + after.length()); + int index = 0; + while (true) { + buff.append(s, index, next).append(after); + index = next + before.length(); + next = s.indexOf(before, index); + if (next < 0) { + buff.append(s, index, s.length()); + break; + } + } + return buff.toString(); + } + + public static String soundex(List args) { + String v1 = (String) args.get(0); + if (v1 == null) { + return null; + } + return new String(getSoundex(v1), StandardCharsets.ISO_8859_1); + } + + private static byte[] getSoundex(String s) { + byte[] chars = {'0', '0', '0', '0'}; + byte lastDigit = '0'; + for (int i = 0, j = 0, l = s.length(); i < l && j < 4; i++) { + char c = s.charAt(i); + if (c >= 'A' && c <= 'z') { + byte newDigit = SOUNDEX_INDEX[c - 'A']; + if (newDigit != 0) { + if (j == 0) { + chars[j++] = (byte) (c & 0xdf); // Converts a-z to A-Z + lastDigit = newDigit; + } else if (newDigit <= '6') { + if (newDigit != lastDigit) { + chars[j++] = lastDigit = newDigit; + } + } else if (newDigit == '7') { + lastDigit = newDigit; + } + } + } + } + return chars; + } + + public static String space(List args) { + Object arg = args.get(0); + if (arg == null) { + return null; + } + + byte[] chars = new byte[Math.max(0, ((Number) arg).intValue())]; + Arrays.fill(chars, (byte) ' '); + return new String(chars, StandardCharsets.ISO_8859_1); + } + + public static String substring(List args) { + String s = (String) args.get(0); + if (s == null) { + return null; + } + int sl = s.length(); + int start = ((Number) args.get(1)).intValue(); + Object v3 = null; + if (args.size() >= 3) { + v3 = args.get(2); + } + // These compatibility conditions violate the Standard + if (start == 0) { + start = 1; + } else if (start < 0) { + start = sl + start + 1; + } + int end = v3 == null ? Math.max(sl + 1, start) : start + ((Number) v3).intValue(); + // SQL Standard requires "data exception - substring error" when + // end < start but H2 does not throw it for compatibility + start = Math.max(start, 1); + end = Math.min(end, sl + 1); + if (start > sl || end <= start) { + return null; + } + return s.substring(start - 1, end - 1); + } + + public static String toChar(List args) { + Object arg = args.get(0); + if (arg == null) { + return null; + } + if (arg instanceof Number) { + return arg.toString(); + } + if (arg instanceof Temporal) { + return DateTimeFunction.formatdatetime(args); + } + return arg.toString(); + } + + public static String translate(List args) { + String original = (String) args.get(0); + if (original == null) { + return null; + } + String findChars = (String) args.get(1); + String replaceChars = (String) args.get(2); + // if it stays null, then no replacements have been made + StringBuilder builder = null; + // if shorter than findChars, then characters are removed + // (if null, we don't access replaceChars at all) + int replaceSize = replaceChars == null ? 0 : replaceChars.length(); + for (int i = 0, size = original.length(); i < size; i++) { + char ch = original.charAt(i); + int index = findChars.indexOf(ch); + if (index >= 0) { + if (builder == null) { + builder = new StringBuilder(size); + if (i > 0) { + builder.append(original, 0, i); + } + } + if (index < replaceSize) { + ch = replaceChars.charAt(index); + } + } + if (builder != null) { + builder.append(ch); + } + } + return builder == null ? original : builder.toString(); + } +} diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java new file mode 100644 index 00000000000..22b283fb3db --- /dev/null +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.transform.sqlengine.zeta.functions; + +import org.apache.seatunnel.common.exception.CommonErrorCode; +import org.apache.seatunnel.transform.exception.TransformException; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.List; + +public class SystemFunction { + public static Object coalesce(List args) { + Object v = null; + for (Object v2 : args) { + if (v2 != null) { + v = v2; + break; + } + } + return v; + } + + public static Object ifnull(List args) { + if (args.size() != 2) { + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported function IFNULL() arguments: %s", args)); + } + return coalesce(args); + } + + public static Object nullif(List args) { + Object v1 = args.get(0); + Object v2 = args.get(1); + if (v1 == null) { + return null; + } + if (v1.equals(v2)) { + return null; + } + return v1; + } + + public static Object castAs(List args) { + Object v1 = args.get(0); + String v2 = (String) args.get(1); + if (v1 == null) { + return null; + } + if (v1.equals(v2)) { + return null; + } + switch (v2) { + case "VARCHAR": + case "STRING": + return v1.toString(); + case "INT": + case "INTEGER": + return Integer.parseInt(v1.toString()); + case "BIGINT": + case "LONG": + return Long.parseLong(v1.toString()); + case "BYTE": + return Byte.parseByte(v1.toString()); + case "DOUBLE": + return Double.parseDouble(v1.toString()); + case "FLOAT": + return Float.parseFloat(v1.toString()); + case "TIMESTAMP": + case "DATETIME": + if (v1 instanceof LocalDateTime) { + return v1; + } + if (v1 instanceof LocalDate) { + return LocalDateTime.of((LocalDate) v1, LocalTime.of(0, 0, 0)); + } + if (v1 instanceof LocalTime) { + return LocalDateTime.of(LocalDate.now(), (LocalTime) v1); + } + case "DATE": + if (v1 instanceof LocalDateTime) { + return ((LocalDateTime) v1).toLocalDate(); + } + if (v1 instanceof LocalDate) { + return v1; + } + case "TIME": + if (v1 instanceof LocalDateTime) { + return ((LocalDateTime) v1).toLocalTime(); + } + if (v1 instanceof LocalDate) { + return LocalDateTime.of((LocalDate) v1, LocalTime.of(0, 0, 0)); + } + if (v1 instanceof LocalTime) { + return LocalDateTime.of(LocalDate.now(), (LocalTime) v1); + } + case "DECIMAL": + BigDecimal bigDecimal = new BigDecimal(v1.toString()); + Integer scale = (Integer) args.get(3); + return bigDecimal.setScale(scale, RoundingMode.CEILING); + } + throw new TransformException( + CommonErrorCode.UNSUPPORTED_OPERATION, + String.format("Unsupported cast as type: %s", v2)); + } +} diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index a46c3033b04..a44b7c979a1 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -25,4 +25,5 @@ scala-library-2.11.12.jar seatunnel-config-base-2.1.1.jar seatunnel-config-shade-2.1.1.jar seatunnel-jackson-2.3.1-SNAPSHOT-optional.jar -slf4j-api-1.7.25.jar \ No newline at end of file +slf4j-api-1.7.25.jar +jsqlparser-4.5.jar \ No newline at end of file From 1b4a47b1104ec7fe7a81c2da4a94244591d24af5 Mon Sep 17 00:00:00 2001 From: mcy Date: Sat, 4 Mar 2023 23:56:05 +0800 Subject: [PATCH 4/4] change plugin name SimpleSQL to Sql --- LICENSE | 2 +- ...mple-sql-functions.md => sql-functions.md} | 4 +- .../en/transform-v2/{simple-sql.md => sql.md} | 16 +- release-note.md | 2 +- .../seatunnel/e2e/transform/TestSQLIT.java | 313 ++++++++++++++++++ .../e2e/transform/TestSimpleSQLIT.java | 313 ------------------ .../binary_expression.conf | 2 +- .../filter_and_or.conf | 2 +- .../filter_equals_to.conf | 2 +- .../filter_function.conf | 2 +- .../filter_great_than.conf | 2 +- .../filter_great_than_equals.conf | 2 +- .../filter_in.conf | 2 +- .../filter_is_not_null.conf | 2 +- .../filter_is_null.conf | 2 +- .../filter_minor_than.conf | 2 +- .../filter_minor_than_equals.conf | 2 +- .../filter_not_equals_to.conf | 2 +- .../filter_not_in.conf | 2 +- .../func_abs.conf | 2 +- .../func_acos.conf | 2 +- .../func_ascii.conf | 2 +- .../func_asin.conf | 2 +- .../func_atan.conf | 2 +- .../func_bit_length.conf | 2 +- .../func_cast.conf | 2 +- .../func_ceil.conf | 2 +- .../func_char_length.conf | 2 +- .../func_chr.conf | 2 +- .../func_coalesce.conf | 2 +- .../func_concat.conf | 2 +- .../func_concat_ws.conf | 2 +- .../func_cos.conf | 2 +- .../func_cosh.conf | 2 +- .../func_cot.conf | 2 +- .../func_current_date.conf | 2 +- .../func_current_time.conf | 2 +- .../func_current_timestamp.conf | 2 +- .../func_date_trunc.conf | 2 +- .../func_dateadd.conf | 2 +- .../func_datediff.conf | 2 +- .../func_day_of_month.conf | 2 +- .../func_day_of_week.conf | 2 +- .../func_day_of_year.conf | 2 +- .../func_dayname.conf | 2 +- .../func_exp.conf | 2 +- .../func_extract.conf | 2 +- .../func_floor.conf | 2 +- .../func_formatdatetime.conf | 2 +- .../func_hextoraw.conf | 2 +- .../func_hour.conf | 2 +- .../func_ifnull.conf | 2 +- .../func_insert.conf | 2 +- .../func_left.conf | 2 +- .../func_ln.conf | 2 +- .../func_locate.conf | 2 +- .../func_log.conf | 2 +- .../func_log10.conf | 2 +- .../func_lower.conf | 2 +- .../func_lpad.conf | 2 +- .../func_ltrim.conf | 2 +- .../func_minute.conf | 2 +- .../func_mod.conf | 2 +- .../func_month.conf | 2 +- .../func_monthname.conf | 2 +- .../func_nullif.conf | 2 +- .../func_octet_length.conf | 2 +- .../func_parsedatetime.conf | 2 +- .../func_pi.conf | 2 +- .../func_power.conf | 2 +- .../func_quarter.conf | 2 +- .../func_radians.conf | 2 +- .../func_rand.conf | 2 +- .../func_rawtohex.conf | 2 +- .../func_regexp_like.conf | 2 +- .../func_regexp_replace.conf | 2 +- .../func_regexp_substr.conf | 2 +- .../func_repeat.conf | 2 +- .../func_replace.conf | 2 +- .../func_right.conf | 2 +- .../func_round.conf | 2 +- .../func_rpad.conf | 2 +- .../func_rtrim.conf | 2 +- .../func_second.conf | 2 +- .../func_sign.conf | 2 +- .../func_sin.conf | 2 +- .../func_sinh.conf | 2 +- .../func_soundex.conf | 2 +- .../func_space.conf | 2 +- .../func_sqrt.conf | 2 +- .../func_substr.conf | 2 +- .../func_tan.conf | 2 +- .../func_tanh.conf | 2 +- .../func_to_char.conf | 2 +- .../func_translate.conf | 2 +- .../func_trim.conf | 2 +- .../func_trunc.conf | 2 +- .../func_upper.conf | 2 +- .../func_week.conf | 2 +- .../func_year.conf | 2 +- ..._sql_transform.conf => sql_transform.conf} | 4 +- ...pleSQLTransform.java => SQLTransform.java} | 37 +-- ...mFactory.java => SQLTransformFactory.java} | 6 +- .../{SimpleSQLEngine.java => SQLEngine.java} | 2 +- ...gineFactory.java => SQLEngineFactory.java} | 8 +- ...impleSQLEngine.java => ZetaSQLEngine.java} | 10 +- .../sqlengine/zeta/ZetaSQLFilter.java | 11 +- .../sqlengine/zeta/ZetaSQLFunction.java | 5 +- .../transform/sqlengine/zeta/ZetaSQLType.java | 6 +- .../zeta/functions/NumericFunction.java | 8 +- .../zeta/functions/SystemFunction.java | 2 +- 111 files changed, 464 insertions(+), 473 deletions(-) rename docs/en/transform-v2/{simple-sql-functions.md => sql-functions.md} (99%) rename docs/en/transform-v2/{simple-sql.md => sql.md} (80%) create mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSQLIT.java delete mode 100644 seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/binary_expression.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_and_or.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_equals_to.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_function.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_great_than.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_great_than_equals.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_in.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_is_not_null.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_is_null.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_minor_than.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_minor_than_equals.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_not_equals_to.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/filter_not_in.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_abs.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_acos.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_ascii.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_asin.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_atan.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_bit_length.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_cast.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_ceil.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_char_length.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_chr.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_coalesce.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_concat.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_concat_ws.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_cos.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_cosh.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_cot.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_current_date.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_current_time.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_current_timestamp.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_date_trunc.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_dateadd.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_datediff.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_day_of_month.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_day_of_week.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_day_of_year.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_dayname.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_exp.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_extract.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_floor.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_formatdatetime.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_hextoraw.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_hour.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_ifnull.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_insert.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_left.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_ln.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_locate.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_log.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_log10.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_lower.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_lpad.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_ltrim.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_minute.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_mod.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_month.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_monthname.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_nullif.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_octet_length.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_parsedatetime.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_pi.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_power.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_quarter.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_radians.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_rand.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_rawtohex.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_regexp_like.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_regexp_replace.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_regexp_substr.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_repeat.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_replace.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_right.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_round.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_rpad.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_rtrim.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_second.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_sign.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_sin.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_sinh.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_soundex.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_space.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_sqrt.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_substr.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_tan.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_tanh.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_to_char.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_translate.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_trim.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_trunc.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_upper.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_week.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_functions => sql_functions}/func_year.conf (99%) rename seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/{simple_sql_transform.conf => sql_transform.conf} (96%) rename seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/{SimpleSQLTransform.java => SQLTransform.java} (70%) rename seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/{SimpleSQLTransformFactory.java => SQLTransformFactory.java} (87%) rename seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/{SimpleSQLEngine.java => SQLEngine.java} (97%) rename seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/{SimpleSQLEngineFactory.java => SQLEngineFactory.java} (84%) rename seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/{ZetaSimpleSQLEngine.java => ZetaSQLEngine.java} (97%) diff --git a/LICENSE b/LICENSE index af47dc7d901..bd06a03806b 100644 --- a/LICENSE +++ b/LICENSE @@ -236,7 +236,7 @@ seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/StandaloneCheckpointIDCounter.java from https://github.com/apache/flink seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/metrics from https://github.com/hazelcast/hazelcast seatunnel-api/src/main/java/org/apache/seatunnel/api/common/metrics from https://github.com/hazelcast/hazelcast -seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java from https://github.com/JSQLParser/JSqlParser +seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLEngine.java from https://github.com/JSQLParser/JSqlParser seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java from https://github.com/JSQLParser/JSqlParser seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java from https://github.com/JSQLParser/JSqlParser seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java from https://github.com/JSQLParser/JSqlParser \ No newline at end of file diff --git a/docs/en/transform-v2/simple-sql-functions.md b/docs/en/transform-v2/sql-functions.md similarity index 99% rename from docs/en/transform-v2/simple-sql-functions.md rename to docs/en/transform-v2/sql-functions.md index bdb783fe67d..dd8b8dbfdd1 100644 --- a/docs/en/transform-v2/simple-sql-functions.md +++ b/docs/en/transform-v2/sql-functions.md @@ -1,6 +1,6 @@ -# SimpleSQL Functions +# SQL Functions -> The Functions of SimpleSQL transform plugin +> The Functions of SQL transform plugin ## String Functions diff --git a/docs/en/transform-v2/simple-sql.md b/docs/en/transform-v2/sql.md similarity index 80% rename from docs/en/transform-v2/simple-sql.md rename to docs/en/transform-v2/sql.md index d7a527e039e..b6c2306c656 100644 --- a/docs/en/transform-v2/simple-sql.md +++ b/docs/en/transform-v2/sql.md @@ -1,12 +1,12 @@ -# SimpleSQL +# SQL -> SimpleSQL transform plugin +> SQL transform plugin ## Description -Use simple SQL to transform given input row. +Use SQL to transform given input row. -SimpleSQL transform use memory SQL engine, we can via SQL functions and ability of SQL engine to implement the transform task. +SQL transform use memory SQL engine, we can via SQL functions and ability of SQL engine to implement the transform task. ## Options @@ -22,7 +22,7 @@ The source table name, the query SQL table name must match this field. ### query [string] -The query SQL, it's a simple SQL supported base function and criteria operation. But the complex SQL unsupported yet, include: multi source table/rows JOIN and AGGREGATE operation and the like. +The query SQL, it's a simple SQL supported base function and criteria filter operation. But the complex SQL unsupported yet, include: multi source table/rows JOIN and AGGREGATE operation and the like. ## Example @@ -39,7 +39,7 @@ We use SQL query to transform the source data like this: ``` transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id, concat(name, '_') as name, age+1 as age from fake where id>0" @@ -78,7 +78,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id, concat(name, '_') as name, age+1 as age from fake where id>0" @@ -96,5 +96,5 @@ sink { ### new version -- Add SimpleSQL Transform Connector +- Add SQL Transform Connector diff --git a/release-note.md b/release-note.md index 64e3a755b89..a655713ef82 100644 --- a/release-note.md +++ b/release-note.md @@ -11,7 +11,7 @@ ### Transformer - [Spark] Support transform-v2 for spark (#3409) - [ALL]Add FieldMapper Transform #3781 -- [ALL]Add SimpleSQL Transform #4148 +- [ALL]Add SQL Transform #4148 ### Connectors - [Elasticsearch] Support https protocol & compatible with opensearch - [Hbase] Add hbase sink connector #4049 diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSQLIT.java b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSQLIT.java new file mode 100644 index 00000000000..ba923e825fe --- /dev/null +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSQLIT.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.transform; + +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.container.spark.AbstractTestSparkContainer; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; + +import java.io.IOException; + +public class TestSQLIT extends TestSuiteBase { + + @TestTemplate + public void testSQL(TestContainer container) throws IOException, InterruptedException { + if (container instanceof AbstractTestSparkContainer) { + return; // skip test for spark, because some problems of Timestamp convert unresolved. + } + + Container.ExecResult execResult = container.executeJob("/sql_transform.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + // -------------binary expression------------- + execResult = container.executeJob("/sql_functions/binary_expression.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + // -------------start test functions------------- + execResult = container.executeJob("/sql_functions/func_ascii.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_bit_length.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_char_length.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_octet_length.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_chr.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_concat.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_hextoraw.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_rawtohex.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_insert.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_lower.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_upper.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_left.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_right.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_lpad.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_rpad.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_ltrim.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_rtrim.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_trim.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_regexp_replace.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_regexp_like.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_regexp_substr.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_repeat.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_replace.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_soundex.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_space.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_substr.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_to_char.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_translate.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_abs.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_acos.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_asin.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_atan.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_cos.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_cosh.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_sin.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_sinh.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_tan.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_tanh.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_mod.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_ceil.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_exp.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_floor.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_ln.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_log.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_log10.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_radians.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_sqrt.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_pi.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_power.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_rand.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_round.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_sign.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_trunc.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_current_date.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_current_time.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_current_timestamp.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_dateadd.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_datediff.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_date_trunc.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_dayname.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_day_of_week.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_day_of_year.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_extract.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_formatdatetime.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_hour.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_minute.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_month.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_monthname.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_parsedatetime.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_quarter.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_second.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_week.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_year.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_cast.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_coalesce.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_ifnull.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/func_nullif.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + // -------------start test filter------------- + execResult = container.executeJob("/sql_functions/filter_equals_to.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_not_equals_to.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_great_than.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_great_than_equals.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_minor_than.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_minor_than_equals.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_is_null.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_is_not_null.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_in.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_not_in.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_function.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + execResult = container.executeJob("/sql_functions/filter_and_or.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java deleted file mode 100644 index 40699811a41..00000000000 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/java/org/apache/seatunnel/e2e/transform/TestSimpleSQLIT.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.e2e.transform; - -import org.apache.seatunnel.e2e.common.container.TestContainer; -import org.apache.seatunnel.e2e.common.container.spark.AbstractTestSparkContainer; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.TestTemplate; -import org.testcontainers.containers.Container; - -import java.io.IOException; - -public class TestSimpleSQLIT extends TestSuiteBase { - - @TestTemplate - public void testSimpleSQL(TestContainer container) throws IOException, InterruptedException { - if (container instanceof AbstractTestSparkContainer) { - return; // skip test for spark, because some problems of Timestamp convert unresolved. - } - - Container.ExecResult execResult = container.executeJob("/simple_sql_transform.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - // -------------binary expression------------- - execResult = container.executeJob("/simple_sql_functions/binary_expression.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - // -------------start test functions------------- - execResult = container.executeJob("/simple_sql_functions/func_ascii.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_bit_length.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_char_length.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_octet_length.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_chr.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_concat.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_hextoraw.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_rawtohex.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_insert.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_lower.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_upper.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_left.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_right.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_lpad.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_rpad.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_ltrim.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_rtrim.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_trim.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_regexp_replace.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_regexp_like.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_regexp_substr.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_repeat.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_replace.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_soundex.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_space.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_substr.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_to_char.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_translate.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_abs.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_acos.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_asin.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_atan.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_cos.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_cosh.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_sin.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_sinh.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_tan.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_tanh.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_mod.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_ceil.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_exp.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_floor.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_ln.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_log.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_log10.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_radians.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_sqrt.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_pi.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_power.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_rand.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_round.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_sign.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_trunc.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_current_date.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_current_time.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_current_timestamp.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_dateadd.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_datediff.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_date_trunc.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_dayname.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_day_of_week.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_day_of_year.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_extract.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_formatdatetime.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_hour.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_minute.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_month.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_monthname.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_parsedatetime.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_quarter.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_second.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_week.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_year.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_cast.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_coalesce.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_ifnull.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/func_nullif.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - // -------------start test filter------------- - execResult = container.executeJob("/simple_sql_functions/filter_equals_to.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_not_equals_to.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_great_than.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_great_than_equals.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_minor_than.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_minor_than_equals.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_is_null.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_is_not_null.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_in.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_not_in.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_function.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - - execResult = container.executeJob("/simple_sql_functions/filter_and_or.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - } -} diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/binary_expression.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/binary_expression.conf index 22bd6e399ef..595e9ea5f83 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/binary_expression.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/binary_expression.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id+1 as id, id*4 as id2, price/3 as price, price-34.22 as price2, price%23.12 as price3, name||'_'||id as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_and_or.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_and_or.conf index 17b45701abc..3a125426dcc 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_and_or.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_and_or.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where name<>'Kin Dom' and (id=1 or age<22)" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_equals_to.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_equals_to.conf index 912a4138797..647de59df1e 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_equals_to.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_equals_to.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id=1" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_function.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_function.conf index 18afb71a3aa..eebedf771b0 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_function.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_function.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where regexp_like(name, '[A-Z ]*')" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_great_than.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_great_than.conf index 143667b57ff..1ed18af6ebc 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_great_than.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id>1" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_great_than_equals.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_great_than_equals.conf index 8ff815301e4..a8f49794ead 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_great_than_equals.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_great_than_equals.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id>=2" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_in.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_in.conf index 14498036291..18d15974465 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_in.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_in.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id in (1,2)" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_is_not_null.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_is_not_null.conf index 83d76359fa7..dedad35c802 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_not_null.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_is_not_null.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where name is not null" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_is_null.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_is_null.conf index a21038f2d2e..8a9d5f6d73d 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_is_null.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_is_null.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where name is null" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_minor_than.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_minor_than.conf index 4c40926c629..37d98e91079 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_minor_than.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id<2" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_minor_than_equals.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_minor_than_equals.conf index 0159c9aca66..3f4762eb67a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_minor_than_equals.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_minor_than_equals.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id<=1" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_not_equals_to.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_not_equals_to.conf index ca5eb8e461f..90a21dce5a0 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_equals_to.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_not_equals_to.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id<>1" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_not_in.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_not_in.conf index 33853e0834a..141be6b5a23 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/filter_not_in.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/filter_not_in.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select id,name,age from fake where id not in (1,2)" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_abs.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_abs.conf index b4ec5f6073c..d7a0d9e04f2 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_abs.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_abs.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select abs(id) as id, abs(price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_acos.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_acos.conf index 45dac4bba0c..403a58d25e5 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_acos.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_acos.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select acos(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ascii.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ascii.conf index ff586629da2..9595753d1b7 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ascii.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ascii.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + sql { source_table_name = "fake" result_table_name = "fake1" query = "select ascii(test1) as test1, ascii(test2) as test2, ascii(test3) as test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_asin.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_asin.conf index 998e57a0f4c..fbe33e0f03a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_asin.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_asin.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select asin(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_atan.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_atan.conf index 77bcaec2254..72c73a3e11f 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_atan.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_atan.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select atan(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_bit_length.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_bit_length.conf index 188f5fff83f..0bcceacd649 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_bit_length.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_bit_length.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select bit_length(test1) as test1, bit_length(test2) as test2, bit_length(test3) as test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cast.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cast.conf index 2a4b8b740de..7961c2ffd9f 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cast.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cast.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select cast(id as STRING) as id, cast(id as INT) as id2, cast(id as DOUBLE) as id3 , cast(name as double) as name, cast(name as DECIMAL(10,2)) as name2, cast(c_time as DATE) as c_time, cast(c_time as time) as c_time2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ceil.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ceil.conf index f78e294d025..68404a2e257 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ceil.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ceil.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select ceil(id) as id, ceiling(price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_char_length.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_char_length.conf index 192de77f546..19bffde1940 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_char_length.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_char_length.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select char_length(test1) as test1, char_length(test2) as test2, length(test3) as test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_chr.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_chr.conf index 8c0b2ce499f..db30093cf6d 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_chr.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_chr.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select chr(test1) as test1, char(test2) as test2, char(test3) as test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_coalesce.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_coalesce.conf index a7396759919..f1dd93d2e2d 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_coalesce.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_coalesce.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select coalesce(name,'Unknown') test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_concat.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_concat.conf index d2fcd23d18f..b130e569a57 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_concat.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select concat(name,'_',age) as name, concat(name,id,'!') as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_concat_ws.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_concat_ws.conf index 086bdfb277a..5dc5efe1c29 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_concat_ws.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_concat_ws.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select concat_ws('_',name,age) as name, concat_ws('_',name,id,'!') as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cos.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cos.conf index 408739ab728..74afb29e4f2 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cos.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cos.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select cos(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cosh.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cosh.conf index a4485435b41..1e99bb986e7 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cosh.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cosh.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select cosh(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cot.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cot.conf index 53038601167..bfde0933cb1 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_cot.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_cot.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select cot(pi()/id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_date.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_date.conf index 55d9aaf7d5d..0302a3580b7 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_date.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_date.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select current_date as cd, current_date() as cd2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_time.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_time.conf index 79c94dbd803..3dde98408ab 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_time.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_time.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select current_time as ct, current_time() as ct2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_timestamp.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_timestamp.conf index a0679e79f60..c5360f22fd5 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_current_timestamp.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_current_timestamp.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select current_timestamp as cd, current_timestamp() as cd2, now() as now from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_date_trunc.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_date_trunc.conf index e0cc50324ed..f84cd2dfbb0 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_date_trunc.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_date_trunc.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select date_trunc(c_time, 'YEAR') as test1, date_trunc(c_time, 'MONTH') as test2, date_trunc(c_time, 'DAY') as test3, date_trunc(c_time, 'HOUR') as test4, date_trunc(c_time, 'MINUTE') as test5, date_trunc(c_time, 'SECOND') as test6 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_dateadd.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_dateadd.conf index 6b6e4bf8181..580549acd23 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dateadd.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_dateadd.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select dateadd(c_time, 1) as test1, dateadd(c_time, 40, 'DAY') as test2, dateadd(c_time, 2, 'YEAR') as test3, dateadd(c_time, 10, 'MONTH') as test4, dateadd(c_time, 13, 'HOUR') as test5, dateadd(c_time, 40, 'MINUTE') as test6, dateadd(c_time, 30, 'SECOND') as test7 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_datediff.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_datediff.conf index 5d658199cd6..790fd1b8a81 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_datediff.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_datediff.conf @@ -42,7 +42,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select datediff(c_time, m_time) as test, datediff(c_time, m_time, 'DAY') as test1, datediff(c_time, m_time, 'YEAR') as test2, datediff(c_time, m_time, 'MONTH') as test3, datediff(c_time, m_time, 'HOUR') as test4, datediff(c_time, m_time, 'MINUTE') as test5, datediff(c_time, m_time, 'SECOND') as test6 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_month.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_month.conf index dfb8ca5d6db..825ee3f0ad2 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_month.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_month.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select day_of_month(c_time) test1, day_of_month(c_time2) test2, day_of_month(c_time3) test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_week.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_week.conf index 0457bd18ab1..d2c941611c9 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_week.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_week.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select day_of_week(c_time) test1, day_of_week(c_time2) test2, day_of_week(c_time3) test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_year.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_year.conf index e1067227ddf..68b22cbe6ab 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_day_of_year.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_day_of_year.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select day_of_year(c_time) test1, day_of_year(c_time2) test2, day_of_year(c_time3) test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_dayname.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_dayname.conf index 7ba4cc55c57..6e44ba49b49 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_dayname.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_dayname.conf @@ -46,7 +46,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select dayname(c_time) as test1, dayname(to_date('2021-04-16','yyyy-MM-dd')) as test2, dayname(c_time2) as test3, dayname(c_time3) as test4, dayname(c_time4) as test5, dayname(c_time5) as test6, dayname(c_time6) as test7 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_exp.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_exp.conf index 0226b7023fc..a0d961e5f9e 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_exp.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_exp.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select exp(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_extract.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_extract.conf index c6b4816a961..45871cc5f29 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_extract.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_extract.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select extract(YEAR FROM c_time) test1, extract(MONTH FROM c_time2) test2, extract(DAY FROM c_time3) test3, extract(HOUR FROM c_time) test4, extract(MINUTE from c_time) test5, extract(SECOND from c_time) test6, extract(MILLISECOND from c_time) test7, extract(DAYOFWEEK FROM c_time) test8, extract(DAYOFYEAR FROM c_time) test9 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_floor.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_floor.conf index 5efbc2eb83c..ec7d29f2580 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_floor.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_floor.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select floor(id) as id, floor(price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_formatdatetime.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_formatdatetime.conf index 3439acbe05f..fa4758faa89 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_formatdatetime.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_formatdatetime.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select formatdatetime(c_time,'yyyy-MM-dd HH:mm:ss.S') test1, formatdatetime(c_time,'yyyy-MM-dd') test2, formatdatetime(c_time,'HH:mm:ss.SSS') test3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_hextoraw.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_hextoraw.conf index 8e924fc03c4..f20510e793a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hextoraw.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_hextoraw.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select hextoraw(test1) as test1, hextoraw(test2) as test2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_hour.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_hour.conf index 74fdac24115..48b69cb2543 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_hour.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_hour.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select hour(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ifnull.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ifnull.conf index aa5271512c7..ed121304b76 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ifnull.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ifnull.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select ifnull(name,'Unknown') test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_insert.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_insert.conf index b0a4e0be707..f702ed13efb 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_insert.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_insert.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select insert(name,1,1,'j') as name, insert(name,2,2,'**') as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_left.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_left.conf index e132c212ec2..cefa0824af1 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_left.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_left.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select left(name, 3) as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ln.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ln.conf index 0fb3f57e8a0..4d9dddd5018 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ln.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ln.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select ln(id) as id, ln(price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_locate.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_locate.conf index 6877c17838d..ddafca7c71a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_locate.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_locate.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select locate(' ', name, 2) as name, instr(name, ' ', 3) as name2, position(' ', name) as name3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_log.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_log.conf index 5d7253820dc..5a6faf09f2d 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_log.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select log(2,id) as id, log(10,price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_log10.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_log10.conf index 290196a9192..52554790a78 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_log10.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_log10.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select log10(id) as id, log10(price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_lower.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_lower.conf index 5fc114202e7..2f8d2f0426a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lower.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_lower.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select lower(name) as name, lcase(name) as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_lpad.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_lpad.conf index c2be1e34c6f..6e4c48dcff6 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_lpad.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_lpad.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select lpad(name, 3) as name, lpad(name, 10, '*') as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ltrim.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ltrim.conf index 258ebd4cf85..af093b9b0bc 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_ltrim.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_ltrim.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select ltrim(name) as name, ltrim(email, '*') as email from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_minute.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_minute.conf index a3c48647e3e..94ae9d69d3a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_minute.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_minute.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select minute(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_mod.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_mod.conf index 4ed2a68544f..13d9ba40822 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_mod.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_mod.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select mod(id, 3) as id, mod(age, 5) as age, mod(age, 5.4) as age2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_month.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_month.conf index 4b1310353c5..9e25a679cfa 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_month.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_month.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select month(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_monthname.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_monthname.conf index 69507c5c97e..af201da5b10 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_monthname.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_monthname.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select monthname(c_time,'yyyy-MM-dd HH:mm:ss.S') test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_nullif.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_nullif.conf index 7a39164ae4d..3f76f31ea65 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_nullif.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_nullif.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select ifnull(nullif(name,'Joy Ding'),'NULL') test1, nullif(name,'Joy Ding_') test2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_octet_length.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_octet_length.conf index 2eb736028a7..a166e0e3734 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_octet_length.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_octet_length.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select octet_length(test1) as test1, octet_length(test2) as test2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_parsedatetime.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_parsedatetime.conf index 0043d83d321..51dcb666f7a 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_parsedatetime.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_parsedatetime.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select parsedatetime(c_time,'yyyy-MM-dd HH:mm:ss.SSS') test1,to_date(c_time2,'yyyy-MM-dd') test2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_pi.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_pi.conf index be5155b8819..d99de437c51 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_pi.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_pi.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select pi() as pi from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_power.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_power.conf index a33014f8ffe..df385fbcadb 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_power.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_power.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select power(id,3) as id, power(price,2) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_quarter.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_quarter.conf index 96674c0720c..76350d7e427 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_quarter.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_quarter.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select quarter(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_radians.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_radians.conf index ead5be9ae80..6f72bd6b76e 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_radians.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_radians.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select radians(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rand.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rand.conf index bbab563180a..72f2228a8e9 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rand.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rand.conf @@ -38,7 +38,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select rand() as rand from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rawtohex.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rawtohex.conf index 352315df7b4..add30b16179 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rawtohex.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rawtohex.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select rawtohex(test1) as test1, rawtohex(test2) as test2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_like.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_like.conf index d6735ae2878..1a217fb2040 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_like.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_like.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select regexp_like(name, '[A-Z ]*', 'i') as name, regexp_like(email, '[A-Z ]*', 'i') as email from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_replace.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_replace.conf index 16257b8a65e..4113938f0ed 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_replace.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_replace.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select regexp_replace(name, '.+', 'b') as name, regexp_replace(email, 'w+', 'W', 'i') as email from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_substr.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_substr.conf index ab53ef20de4..ec016427eee 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_regexp_substr.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_regexp_substr.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select regexp_substr(c_date, '\\d{4}') as c_year, regexp_substr(c_date, '(\\d{4})-(\\d{2})-(\\d{2})', 1, 1, null, 2) as c_month from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_repeat.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_repeat.conf index c9d47febcb2..bdb26687b0e 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_repeat.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_repeat.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select repeat(name||' ',3) as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_replace.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_replace.conf index 4a46303a43e..7f874eb864d 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_replace.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_replace.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select replace(name,' ') as name, replace(name,' ','_') as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_right.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_right.conf index 9d8ea0d4980..a6d0146c42c 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_right.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_right.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select right(name, 4) as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_round.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_round.conf index 1f7231c62b6..77c35b8a500 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_round.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_round.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select round(price) as price, round(price,2) as price2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rpad.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rpad.conf index ad0df510ea4..cada9b4ac68 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rpad.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rpad.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select rpad(name, 3) as name, rpad(name, 10, '*') as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rtrim.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rtrim.conf index 48b76e05800..4289887cea1 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_rtrim.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_rtrim.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select rtrim(name) as name, rtrim(email, '*') as email from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_second.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_second.conf index 115cba705e7..9564b7884c1 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_second.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_second.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select second(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sign.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sign.conf index ad3522ca244..4cb87ead328 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sign.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sign.conf @@ -43,7 +43,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select sign(id) as id, sign(price) as price, sign(price2) as price2, sign(price3) as price3 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sin.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sin.conf index 0806b3daa90..f03f4de337e 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sin.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sin.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select sin(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sinh.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sinh.conf index 983c7d333fb..82062499417 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sinh.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sinh.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select sinh(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_soundex.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_soundex.conf index 623916a1200..c3273770f01 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_soundex.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_soundex.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select soundex(name) as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_space.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_space.conf index 2e0a1a5f955..17309ff0c52 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_space.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_space.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select name || space(3) as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sqrt.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sqrt.conf index cb7caa877a0..7f4bccd60a1 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_sqrt.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_sqrt.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select sqrt(id) as id, sqrt(price) as price from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_substr.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_substr.conf index ef83811d75e..88efac20c71 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_substr.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_substr.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select substr(name, 5) as name, substring(name, 1, 3) as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_tan.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_tan.conf index 73dd63cef3e..fc4c84863d6 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tan.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_tan.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select tan(pi()/id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_tanh.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_tanh.conf index 5fd4a408cef..2756a70c6df 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_tanh.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_tanh.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select tanh(id) as id from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_to_char.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_to_char.conf index 3d4fb48d4ea..6ba924f28aa 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_to_char.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_to_char.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select to_char(id) as id, to_char(c_time,'yyyy-MM-dd') as c_date from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_translate.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_translate.conf index cd4f102806e..cb3cde92006 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_translate.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_translate.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select translate(name, 'ing', 'ING') as name from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_trim.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_trim.conf index 7a8fa166d5c..b4d2caae572 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trim.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_trim.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select trim(name) as name, trim(email, '*') as email from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_trunc.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_trunc.conf index f829a9d0e3c..2a2e90a2d28 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_trunc.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_trunc.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select trunc(price) as price, trunc(price,2) as price2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_upper.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_upper.conf index 6a3d0754e71..24e96b988d1 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_upper.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_upper.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select upper(name) as name, ucase(name) as name2 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_week.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_week.conf index 5001fcb3450..d4610ba3ebb 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_week.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_week.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select week(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_year.conf similarity index 99% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_year.conf index e2b8fe6966c..fa87c237941 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_functions/func_year.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_functions/func_year.conf @@ -41,7 +41,7 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" query = "select year(c_time) test1 from fake" diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_transform.conf similarity index 96% rename from seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf rename to seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_transform.conf index ec594956051..8273213b377 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/simple_sql_transform.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/src/test/resources/sql_transform.conf @@ -37,13 +37,13 @@ source { } transform { - SimpleSQL { + Sql { source_table_name = "fake" result_table_name = "fake1" # the query table name must same as field 'source_table_name' query = "select id, regexp_replace(name, '.+', 'b') as name, age+1 as age, pi() as pi from fake" } - # The SimpleSQL transform support base function and criteria operation + # The SQL transform support base function and criteria operation # But the complex SQL unsupported yet, include: multi source table/rows JOIN and AGGREGATE operation and the like } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SQLTransform.java similarity index 70% rename from seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java rename to seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SQLTransform.java index b527fcc5909..164c6ce26ef 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SQLTransform.java @@ -27,38 +27,35 @@ import org.apache.seatunnel.common.config.CheckConfigUtil; import org.apache.seatunnel.common.config.CheckResult; import org.apache.seatunnel.transform.common.AbstractSeaTunnelTransform; -import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngine; -import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngineFactory; -import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngineFactory.EngineType; +import org.apache.seatunnel.transform.sqlengine.SQLEngine; +import org.apache.seatunnel.transform.sqlengine.SQLEngineFactory; +import org.apache.seatunnel.transform.sqlengine.SQLEngineFactory.EngineType; import com.google.auto.service.AutoService; -import static org.apache.seatunnel.transform.sqlengine.SimpleSQLEngineFactory.EngineType.INTERNAL; +import static org.apache.seatunnel.transform.sqlengine.SQLEngineFactory.EngineType.ZETA; @AutoService(SeaTunnelTransform.class) -public class SimpleSQLTransform extends AbstractSeaTunnelTransform { +public class SQLTransform extends AbstractSeaTunnelTransform { public static final Option KEY_QUERY = - Options.key("query") - .stringType() - .noDefaultValue() - .withDescription("The simple query SQL"); + Options.key("query").stringType().noDefaultValue().withDescription("The query SQL"); public static final Option KEY_ENGINE = Options.key("engine") .stringType() - .noDefaultValue() - .withDescription("The SQL engine for SimpleSQL transform"); + .defaultValue(ZETA.name()) + .withDescription("The SQL engine type"); private String query; private EngineType engineType; - private transient SimpleSQLEngine simpleSQLEngine; + private transient SQLEngine sqlEngine; @Override public String getPluginName() { - return "SimpleSQL"; + return "Sql"; } @Override @@ -71,18 +68,18 @@ protected void setConfig(Config pluginConfig) { if (pluginConfig.hasPath(KEY_ENGINE.key())) { engineType = EngineType.valueOf(pluginConfig.getString(KEY_ENGINE.key()).toUpperCase()); } else { - engineType = INTERNAL; + engineType = ZETA; } } @Override public void open() { - simpleSQLEngine = SimpleSQLEngineFactory.getSimpleSQLEngine(engineType); - simpleSQLEngine.init(inputTableName, inputRowType, query); + sqlEngine = SQLEngineFactory.getSQLEngine(engineType); + sqlEngine.init(inputTableName, inputRowType, query); } private void tryOpen() { - if (simpleSQLEngine == null) { + if (sqlEngine == null) { open(); } } @@ -90,17 +87,17 @@ private void tryOpen() { @Override protected SeaTunnelRowType transformRowType(SeaTunnelRowType inputRowType) { tryOpen(); - return simpleSQLEngine.typeMapping(); + return sqlEngine.typeMapping(); } @Override protected SeaTunnelRow transformRow(SeaTunnelRow inputRow) { tryOpen(); - return simpleSQLEngine.transformBySQL(inputRow); + return sqlEngine.transformBySQL(inputRow); } @Override public void close() { - simpleSQLEngine.close(); + sqlEngine.close(); } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SQLTransformFactory.java similarity index 87% rename from seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java rename to seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SQLTransformFactory.java index 9805e8d1d88..13b54508509 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SimpleSQLTransformFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/SQLTransformFactory.java @@ -23,13 +23,13 @@ import com.google.auto.service.AutoService; -import static org.apache.seatunnel.transform.SimpleSQLTransform.KEY_QUERY; +import static org.apache.seatunnel.transform.SQLTransform.KEY_QUERY; @AutoService(Factory.class) -public class SimpleSQLTransformFactory implements TableTransformFactory { +public class SQLTransformFactory implements TableTransformFactory { @Override public String factoryIdentifier() { - return "SimpleSQL"; + return "Sql"; } @Override diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngine.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SQLEngine.java similarity index 97% rename from seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngine.java rename to seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SQLEngine.java index 945bbdef3b7..116e5319f3b 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngine.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SQLEngine.java @@ -20,7 +20,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -public interface SimpleSQLEngine { +public interface SQLEngine { void init(String inputTableName, SeaTunnelRowType inputRowType, String sql); SeaTunnelRowType typeMapping(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngineFactory.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SQLEngineFactory.java similarity index 84% rename from seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngineFactory.java rename to seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SQLEngineFactory.java index d9fe32a52e8..c7e1aebe07d 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SimpleSQLEngineFactory.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/SQLEngineFactory.java @@ -19,14 +19,14 @@ import org.apache.seatunnel.common.exception.CommonErrorCode; import org.apache.seatunnel.transform.exception.TransformException; -import org.apache.seatunnel.transform.sqlengine.zeta.ZetaSimpleSQLEngine; +import org.apache.seatunnel.transform.sqlengine.zeta.ZetaSQLEngine; -public class SimpleSQLEngineFactory { - public static SimpleSQLEngine getSimpleSQLEngine(EngineType engineType) { +public class SQLEngineFactory { + public static SQLEngine getSQLEngine(EngineType engineType) { switch (engineType) { case ZETA: case INTERNAL: - return new ZetaSimpleSQLEngine(); + return new ZetaSQLEngine(); } throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLEngine.java similarity index 97% rename from seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java rename to seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLEngine.java index 04d12683c96..4e6d6eec4d3 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSimpleSQLEngine.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLEngine.java @@ -22,7 +22,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.common.exception.CommonErrorCode; import org.apache.seatunnel.transform.exception.TransformException; -import org.apache.seatunnel.transform.sqlengine.SimpleSQLEngine; +import org.apache.seatunnel.transform.sqlengine.SQLEngine; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -39,7 +39,7 @@ import java.util.List; -public class ZetaSimpleSQLEngine implements SimpleSQLEngine { +public class ZetaSQLEngine implements SQLEngine { private String inputTableName; private String sql; @@ -49,7 +49,7 @@ public class ZetaSimpleSQLEngine implements SimpleSQLEngine { private ZetaSQLFilter zetaSQLFilter; private ZetaSQLType zetaSQLType; - public ZetaSimpleSQLEngine() {} + public ZetaSQLEngine() {} @Override public void init(String inputTableName, SeaTunnelRowType inputRowType, String sql) { @@ -83,10 +83,10 @@ private void validateSQL(Statement statement) { } Select select = (Select) statement; if (!(select.getSelectBody() instanceof PlainSelect)) { - throw new IllegalArgumentException("Unsupported simple SQL: %s"); + throw new IllegalArgumentException("Unsupported SQL syntax: %s"); } if (!(select.getSelectBody() instanceof PlainSelect)) { - throw new IllegalArgumentException("Unsupported simple SQL: %s"); + throw new IllegalArgumentException("Unsupported SQL syntax: %s"); } PlainSelect selectBody = (PlainSelect) select.getSelectBody(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java index 9d032a9e8c5..0c1f0e97520 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFilter.java @@ -154,8 +154,7 @@ private boolean inExpr(InExpression inExpression, Object[] inputFields) { */ private boolean likeExpr(LikeExpression likeExpression, Object[] inputFields) { throw new TransformException( - CommonErrorCode.UNSUPPORTED_OPERATION, - "Unsupported [like] filter for SimpleSQL yet"); + CommonErrorCode.UNSUPPORTED_OPERATION, "Unsupported [LIKE] filter expression yet"); } private Pair executeComparisonOperator( @@ -215,7 +214,7 @@ private boolean greaterThanExpr(Pair pair) { throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, String.format( - "Filed types not matched, left: %s, right:%s ", + "Filed types not matched, left is: %s, right is: %s ", leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); } @@ -245,7 +244,7 @@ private boolean greaterThanEqualsExpr(Pair pair) { throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, String.format( - "Filed types not matched, left: %s, right:%s ", + "Filed types not matched, left is: %s, right is: %s ", leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); } @@ -273,7 +272,7 @@ private boolean minorThanExpr(Pair pair) { throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, String.format( - "Filed types not matched, left: %s, right:%s ", + "Filed types not matched, left is: %s, right is: %s ", leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); } @@ -303,7 +302,7 @@ private boolean minorThanEqualsExpr(Pair pair) { throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, String.format( - "Filed types not matched, left: %s, right:%s ", + "Filed types not matched, left is: %s, right is: %s ", leftVal.getClass().getSimpleName(), rightVal.getClass().getSimpleName())); } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java index 8f2756842b9..f73b60a8919 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLFunction.java @@ -405,8 +405,7 @@ public Object executeFunctionExpr(String functionName, List args) { default: throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, - String.format( - "Function: %s unsupported for SimpleSQL transform", functionName)); + String.format("Unsupported function: %s", functionName)); } } @@ -424,7 +423,7 @@ public Object executeTimeKeyExpr(String timeKeyExpr) { } throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, - String.format("TimeKeyExpr: %s unsupported for SimpleSQL transform", timeKeyExpr)); + String.format("Unsupported TimeKey expression: %s", timeKeyExpr)); } public Object executeCastExpr(CastExpression castExpression, Object arg) { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java index cd80f5d2ca9..4d58f0670fe 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/ZetaSQLType.java @@ -175,7 +175,7 @@ private SeaTunnelDataType getCastType(CastExpression castExpression) { default: throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, - String.format("Unsupported cast as type: %s", dataType)); + String.format("Unsupported CAST AS type: %s", dataType)); } } @@ -302,7 +302,7 @@ private SeaTunnelDataType getFunctionType(Function function) { default: throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, - String.format("Function: %s unsupported", function.getName())); + String.format("Unsupported function: %s ", function.getName())); } } @@ -321,7 +321,7 @@ private SeaTunnelDataType getTimeKeyExprType(TimeKeyExpression timeKeyExpress throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, String.format( - "TimeKeyExpression: %s unsupported", + "Unsupported TimeKey expression: %s ", timeKeyExpression.getStringValue())); } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java index 6d4e5ae8dca..963822884cc 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/NumericFunction.java @@ -102,9 +102,7 @@ public static Double cot(List args) { } double d = Math.tan(arg.doubleValue()); if (d == 0) { - throw new TransformException( - CommonErrorCode.UNSUPPORTED_OPERATION, - String.format("Division by zero for function: %s", ZetaSQLFunction.COT)); + throw new TransformException(CommonErrorCode.UNSUPPORTED_OPERATION, "Division by zero"); } return 1d / d; } @@ -163,9 +161,7 @@ public static Number mod(List args) { return null; } if (rightValue.doubleValue() == 0) { - throw new TransformException( - CommonErrorCode.UNSUPPORTED_OPERATION, - String.format("Mod by zero for function: %s", ZetaSQLFunction.MOD)); + throw new TransformException(CommonErrorCode.UNSUPPORTED_OPERATION, "Mod by zero"); } BigDecimal leftBD = BigDecimal.valueOf(leftValue.doubleValue()); BigDecimal rightBD = BigDecimal.valueOf(rightValue.doubleValue()); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java index 22b283fb3db..6da784db035 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sqlengine/zeta/functions/SystemFunction.java @@ -120,6 +120,6 @@ public static Object castAs(List args) { } throw new TransformException( CommonErrorCode.UNSUPPORTED_OPERATION, - String.format("Unsupported cast as type: %s", v2)); + String.format("Unsupported CAST AS type: %s", v2)); } }