From a94cdef0af0342dc92637e3d62c4004881fe5f99 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Mon, 20 Feb 2023 22:02:43 -0800 Subject: [PATCH 01/18] Added Basic Tests Signed-off-by: GabeFernandez310 --- .../datetime/DateTimeFunctionTest.java | 38 +++++++++++++++++++ .../sql/sql/antlr/SQLSyntaxParserTest.java | 6 +++ 2 files changed, 44 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index fdb029ca90..c8e719af12 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1799,6 +1799,44 @@ public void to_days() { assertEquals(longValue(719527L), eval(expression)); } + private static Stream getTestDataForToSeconds() { + return Stream.of( + Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), + Arguments.of(new ExprStringValue("2009-11-29"), new ExprLongValue(634266720000L)), + Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), + //Arguments.of( NOW(), 63426721458), + Arguments.of( new ExprStringValue("0000-00-00"), ExprNullValue.of()), + Arguments.of(new ExprStringValue("0000-01-01"), new ExprLongValue(86400L)) + ); + } + + @ParameterizedTest + @MethodSource("getTestDataForToSeconds") + public void testToSeconds(ExprValue arg, ExprValue expected) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + FunctionExpression expr = DSL.to_seconds(arg); + assertEquals(LONG, expr.type()); + assertEquals(expected, eval(expr)); + } + + private static Stream getInvalidTestDataForToSeconds() { + return Stream.of( + Arguments.of(new ExprLongValue(-123L)) + ); + } + + @ParameterizedTest + @MethodSource("getInvalidTestDataForToSeconds") + public void testToSecondsInvalidArg(ExprValue arg) { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + FunctionExpression expr = DSL.to_seconds(arg); + assertThrows(SemanticCheckException.class, () -> eval(expr)); + } + @Test public void year() { when(nullRef.type()).thenReturn(DATE); diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 0b8e64d0bb..d50356e41d 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -527,6 +527,12 @@ public void can_parse_last_day_function() { assertNotNull(parser.parse("SELECT last_day('2004-01-01 01:01:01')")); } + @Test + public void can_parse_to_days_function() { + assertNotNull(parser.parse("SELECT to_days(\"2023-02-20\")")); + assertNotNull(parser.parse("SELECT to_days(950501)")); + } + @Test public void can_parse_wildcard_query_relevance_function() { assertNotNull( From ab66d156862062aeaf15c5eb2f0bb836e4b6c402 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 21 Feb 2023 07:23:38 -0800 Subject: [PATCH 02/18] Added IT Test Signed-off-by: GabeFernandez310 --- .../sql/sql/DateTimeFunctionIT.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 4254641524..ba2266d288 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -900,6 +900,25 @@ public void testToDays() throws IOException { verifyDataRows(result, rows(738049)); } + @Test + public void testToSeconds() throws IOException { + JSONObject result = executeQuery( + String.format("select to_days(date(date0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + verifyDataRows(result, rows(63249206400L), rows(62246275200L)); + + result = executeQuery( + String.format("select to_days(time(time0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + verifyDataRows(result, rows(63844232852L), rows(63844206528L)); + + result = executeQuery( + String.format("select to_days(datetime(datetime0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + verifyDataRows(result, rows(63256587455L), rows(63258064234L)); + + result = executeQuery(String.format( + "select to_days(timestamp(CAST(datetime0 AS STRING))) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + verifyDataRows(result, rows(63256587455L), rows(63258064234L)); + } + @Test public void testYear() throws IOException { JSONObject result = executeQuery("select year(date('2020-09-16'))"); From 3af206ebf0710d33eb85901905c132229f25a710 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 21 Feb 2023 10:34:15 -0800 Subject: [PATCH 03/18] Added Implementation Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 4 + .../expression/datetime/DateTimeFunction.java | 73 +++++++++++++++++++ .../function/BuiltinFunctionName.java | 1 + .../datetime/DateTimeFunctionTest.java | 11 +-- .../sql/sql/DateTimeFunctionIT.java | 4 - sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + .../sql/sql/antlr/SQLSyntaxParserTest.java | 6 +- 8 files changed, 89 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 616f431283..7ac36984ef 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -434,6 +434,10 @@ public static FunctionExpression to_days(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.TO_DAYS, expressions); } + public static FunctionExpression to_seconds(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.TO_SECONDS, expressions); + } + public static FunctionExpression week( FunctionProperties functionProperties, Expression... expressions) { return compile(functionProperties, BuiltinFunctionName.WEEK, expressions); diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index fc8cdc93ef..7d08012692 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -103,6 +103,9 @@ public class DateTimeFunction { // Mode used for week/week_of_year function by default when no argument is provided private static final ExprIntegerValue DEFAULT_WEEK_OF_YEAR_MODE = new ExprIntegerValue(0); + //The number of seconds per day + private static final long SECONDS_PER_DAY = 86400; + // Map used to determine format output for the get_format function private static final Table formats = ImmutableTable.builder() @@ -191,6 +194,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(utc_timestamp()); repository.register(date_format()); repository.register(to_days()); + repository.register(to_seconds()); repository.register(unix_timestamp()); repository.register(week(BuiltinFunctionName.WEEK)); repository.register(week(BuiltinFunctionName.WEEKOFYEAR)); @@ -826,6 +830,18 @@ private DefaultFunctionResolver to_days() { impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME)); } + /** + * TO_SECONDS(STRING/DATE/DATETIME/TIMESTAMP). return the day number of the given date. + */ + private DefaultFunctionResolver to_seconds() { + return define(BuiltinFunctionName.TO_SECONDS.getName(), + impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, STRING), + impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, DATE), + impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, DATETIME), + impl(nullMissingHandling(DateTimeFunction::exprToSecondsForIntType), LONG, LONG)); + } + private FunctionResolver unix_timestamp() { return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(), implWithProperties(functionProperties @@ -1626,6 +1642,63 @@ private ExprValue exprToDays(ExprValue date) { return new ExprLongValue(date.dateValue().toEpochDay() + DAYS_0000_TO_1970); } + //Helper function to handle a '0000' year for the 'to_seconds' function according to MySQL. + private ExprValue handleZeroYear(ExprValue date) { + String dateStringVal = date.stringValue(); + + //Handle corner case for date of 0000-00-00 according to MySQL docs. + if (dateStringVal.contains("0000-00-00")){ + return ExprNullValue.of(); + } + + //1970 used as a placeholder to extract correct seconds value. + if(dateStringVal.length() > 10) { + return(new ExprLongValue( + LocalDateTime.parse(date.stringValue().replace("0000-", "1970-")) + .toEpochSecond(ZoneOffset.UTC)) + ); + } + + return(new ExprLongValue( + LocalDate.parse(date.stringValue().replace("0000-", "1970-")) + .toEpochSecond(LocalTime.MAX , ZoneOffset.UTC) + 1) + ); + } + + /** + * To_seconds implementation for ExprValue. + * + * @param date ExprValue of Date/Datetime/Timestamp/String type. + * @return ExprValue. + */ + private ExprValue exprToSeconds(ExprValue date) { + + //Handle special case of year 0000 according to MySQL docs. + if (date.stringValue().contains("0000-")) { + return handleZeroYear(date); + } + + return new ExprLongValue( + date.datetimeValue().toEpochSecond(ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } + + /** + * To_seconds implementation with an integer argument for ExprValue. + * + * @param date ExprValue of an Integer/Long formatted for a date (e.g., 950501 = 1995-05-01) + * @return ExprValue. + */ + private ExprValue exprToSecondsForIntType(ExprValue date) { + int day = date.integerValue() % 10; + int month = date.integerValue() % 10000 - day; + int year = date.integerValue() - month - day; + + LocalDateTime datetime = LocalDateTime.of(1900 + year/10000, month/100, day, 0, 0, 0); + + return new ExprLongValue( + datetime.toEpochSecond(ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } + /** * Week for date implementation for ExprValue. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ec4a7bc140..19ef978148 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -104,6 +104,7 @@ public enum BuiltinFunctionName { TIMESTAMP(FunctionName.of("timestamp")), TIME_FORMAT(FunctionName.of("time_format")), TO_DAYS(FunctionName.of("to_days")), + TO_SECONDS(FunctionName.of("to_seconds")), UTC_DATE(FunctionName.of("utc_date")), UTC_TIME(FunctionName.of("utc_time")), UTC_TIMESTAMP(FunctionName.of("utc_timestamp")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index c8e719af12..d74c53b75f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -26,6 +26,8 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; + +import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -1802,9 +1804,8 @@ public void to_days() { private static Stream getTestDataForToSeconds() { return Stream.of( Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), - Arguments.of(new ExprStringValue("2009-11-29"), new ExprLongValue(634266720000L)), + Arguments.of(new ExprStringValue("2009-11-29"), new ExprLongValue(63426672000L)), Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), - //Arguments.of( NOW(), 63426721458), Arguments.of( new ExprStringValue("0000-00-00"), ExprNullValue.of()), Arguments.of(new ExprStringValue("0000-01-01"), new ExprLongValue(86400L)) ); @@ -1816,7 +1817,7 @@ public void testToSeconds(ExprValue arg, ExprValue expected) { lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - FunctionExpression expr = DSL.to_seconds(arg); + FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); assertEquals(LONG, expr.type()); assertEquals(expected, eval(expr)); } @@ -1833,8 +1834,8 @@ public void testToSecondsInvalidArg(ExprValue arg) { lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - FunctionExpression expr = DSL.to_seconds(arg); - assertThrows(SemanticCheckException.class, () -> eval(expr)); + FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); + assertThrows(DateTimeException.class, () -> eval(expr)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index ba2266d288..70ccbee293 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -906,10 +906,6 @@ public void testToSeconds() throws IOException { String.format("select to_days(date(date0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); verifyDataRows(result, rows(63249206400L), rows(62246275200L)); - result = executeQuery( - String.format("select to_days(time(time0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); - verifyDataRows(result, rows(63844232852L), rows(63844206528L)); - result = executeQuery( String.format("select to_days(datetime(datetime0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); verifyDataRows(result, rows(63256587455L), rows(63258064234L)); diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 25f23a7bd6..0ffb4c1d0c 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -265,6 +265,7 @@ TIME_TO_SEC: 'TIME_TO_SEC'; TIMESTAMP: 'TIMESTAMP'; TRUNCATE: 'TRUNCATE'; TO_DAYS: 'TO_DAYS'; +TO_SECONDS: 'TO_SECONDS'; UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; UPPER: 'UPPER'; UTC_DATE: 'UTC_DATE'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index e5efeabba0..9b93faf2cf 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -477,6 +477,7 @@ dateTimeFunctionName | TIMEDIFF | TIMESTAMP | TO_DAYS + | TO_SECONDS | UNIX_TIMESTAMP | WEEK | WEEK_OF_YEAR diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index d50356e41d..a70a19c1da 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -528,9 +528,9 @@ public void can_parse_last_day_function() { } @Test - public void can_parse_to_days_function() { - assertNotNull(parser.parse("SELECT to_days(\"2023-02-20\")")); - assertNotNull(parser.parse("SELECT to_days(950501)")); + public void can_parse_to_seconds_function() { + assertNotNull(parser.parse("SELECT to_seconds(\"2023-02-20\")")); + assertNotNull(parser.parse("SELECT to_seconds(950501)")); } @Test From cad2e4741f0c0f98837c31d4979589f04613964f Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 21 Feb 2023 12:33:27 -0800 Subject: [PATCH 04/18] Changed Integration Tests Signed-off-by: GabeFernandez310 --- .../sql/expression/datetime/DateTimeFunction.java | 2 +- .../java/org/opensearch/sql/sql/DateTimeFunctionIT.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 7d08012692..2c2e903f01 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -1674,7 +1674,7 @@ private ExprValue handleZeroYear(ExprValue date) { private ExprValue exprToSeconds(ExprValue date) { //Handle special case of year 0000 according to MySQL docs. - if (date.stringValue().contains("0000-")) { + if (date.toString().contains("0000-")) { return handleZeroYear(date); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 70ccbee293..39fbc16054 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -903,15 +903,15 @@ public void testToDays() throws IOException { @Test public void testToSeconds() throws IOException { JSONObject result = executeQuery( - String.format("select to_days(date(date0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + String.format("select to_seconds(date(date0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); verifyDataRows(result, rows(63249206400L), rows(62246275200L)); result = executeQuery( - String.format("select to_days(datetime(datetime0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + String.format("SELECT to_seconds(datetime(cast(datetime0 AS string))) FROM %s LIMIT 2", TEST_INDEX_CALCS)); verifyDataRows(result, rows(63256587455L), rows(63258064234L)); result = executeQuery(String.format( - "select to_days(timestamp(CAST(datetime0 AS STRING))) FROM %s LIMIT 2", TEST_INDEX_CALCS)); + "select to_seconds(timestamp(datetime0)) FROM %s LIMIT 2", TEST_INDEX_CALCS)); verifyDataRows(result, rows(63256587455L), rows(63258064234L)); } From 51f2410978bc77a9f3cc8f32d3d8324238bd9ea0 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 21 Feb 2023 16:01:24 -0800 Subject: [PATCH 05/18] Added Test For Time Type Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 7 +++++- .../datetime/DateTimeFunctionTest.java | 24 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 7ac36984ef..654aaf4bed 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -433,11 +433,16 @@ public static FunctionExpression date_format( public static FunctionExpression to_days(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.TO_DAYS, expressions); } + public static FunctionExpression to_seconds(FunctionProperties functionProperties, + Expression... expressions) { + return compile(functionProperties, BuiltinFunctionName.TO_SECONDS, expressions); + } public static FunctionExpression to_seconds(Expression... expressions) { - return compile(FunctionProperties.None, BuiltinFunctionName.TO_SECONDS, expressions); + return to_seconds(FunctionProperties.None, expressions); } + public static FunctionExpression week( FunctionProperties functionProperties, Expression... expressions) { return compile(functionProperties, BuiltinFunctionName.WEEK, expressions); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index d74c53b75f..675d7f3218 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -30,6 +30,9 @@ import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.stream.Stream; @@ -151,6 +154,8 @@ public void setup() { ImmutableList.of("1998-01-31b 13:14:15 b")) ); + private static final long SECONDS_FROM_0001_01_01_TO_EPOCH_START = 62167219200L; + @AllArgsConstructor private class DateFormatTester { private final String date; @@ -1806,7 +1811,10 @@ private static Stream getTestDataForToSeconds() { Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), Arguments.of(new ExprStringValue("2009-11-29"), new ExprLongValue(63426672000L)), Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), - Arguments.of( new ExprStringValue("0000-00-00"), ExprNullValue.of()), + Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)), + Arguments.of(new ExprDatetimeValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), + Arguments.of(new ExprTimestampValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), + Arguments.of(new ExprStringValue("0000-00-00"), ExprNullValue.of()), Arguments.of(new ExprStringValue("0000-01-01"), new ExprLongValue(86400L)) ); } @@ -1822,6 +1830,20 @@ public void testToSeconds(ExprValue arg, ExprValue expected) { assertEquals(expected, eval(expr)); } + @Test + public void testToSecondsWithTimeType() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + FunctionExpression expr = DSL.to_seconds(functionProperties, DSL.literal(new ExprTimeValue("10:11:12"))); + + long expected = SECONDS_FROM_0001_01_01_TO_EPOCH_START + + LocalDate.now(functionProperties.getQueryStartClock()) + .toEpochSecond(LocalTime.parse("10:11:12"), ZoneOffset.UTC); + + assertEquals(expected, eval(expr).longValue()); + } + private static Stream getInvalidTestDataForToSeconds() { return Stream.of( Arguments.of(new ExprLongValue(-123L)) From 07e0b6052a1576e3651b934c23479548e509f112 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 21 Feb 2023 16:17:07 -0800 Subject: [PATCH 06/18] Added Documentation Signed-off-by: GabeFernandez310 --- docs/user/dql/functions.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 5d5a3e1f96..c8b0d49dd0 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2459,6 +2459,27 @@ Example:: | 733687 | +------------------------------+ +TO_SECONDS +---------- + +Description +>>>>>>>>>>> + +Usage: to_seconds(date) returns the seconds number (the number of days since year 0) of the given date. Returns NULL if date is invalid. + +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP + +Return type: LONG + +Example:: + + os> SELECT TO_SECONDS(DATE '2008-10-07') + fetched rows / total rows = 1/1 + +---------------------------------+ + | TO_SECONDS(DATE '2008-10-07') | + |---------------------------------| + | 63390556800 | + +---------------------------------+ UNIX_TIMESTAMP -------------- From 9a2377a5b8ee4bcf5df64a5cea5bf20bd069d32b Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 28 Feb 2023 08:41:54 -0800 Subject: [PATCH 07/18] Addressed PR Comments Signed-off-by: GabeFernandez310 --- .../expression/datetime/DateTimeFunction.java | 54 ++------ .../datetime/DateTimeFunctionTest.java | 56 --------- .../expression/datetime/ToSecondsTest.java | 117 ++++++++++++++++++ docs/user/dql/functions.rst | 4 +- 4 files changed, 132 insertions(+), 99 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 2c2e903f01..38b0400d0d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -831,14 +831,11 @@ private DefaultFunctionResolver to_days() { } /** - * TO_SECONDS(STRING/DATE/DATETIME/TIMESTAMP). return the day number of the given date. + * TO_SECONDS(STRING/DATE/DATETIME/TIMESTAMP/LONG). return the seconds number of the given date. */ private DefaultFunctionResolver to_seconds() { return define(BuiltinFunctionName.TO_SECONDS.getName(), - impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, STRING), impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, TIMESTAMP), - impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, DATE), - impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprToSecondsForIntType), LONG, LONG)); } @@ -1642,29 +1639,6 @@ private ExprValue exprToDays(ExprValue date) { return new ExprLongValue(date.dateValue().toEpochDay() + DAYS_0000_TO_1970); } - //Helper function to handle a '0000' year for the 'to_seconds' function according to MySQL. - private ExprValue handleZeroYear(ExprValue date) { - String dateStringVal = date.stringValue(); - - //Handle corner case for date of 0000-00-00 according to MySQL docs. - if (dateStringVal.contains("0000-00-00")){ - return ExprNullValue.of(); - } - - //1970 used as a placeholder to extract correct seconds value. - if(dateStringVal.length() > 10) { - return(new ExprLongValue( - LocalDateTime.parse(date.stringValue().replace("0000-", "1970-")) - .toEpochSecond(ZoneOffset.UTC)) - ); - } - - return(new ExprLongValue( - LocalDate.parse(date.stringValue().replace("0000-", "1970-")) - .toEpochSecond(LocalTime.MAX , ZoneOffset.UTC) + 1) - ); - } - /** * To_seconds implementation for ExprValue. * @@ -1672,12 +1646,6 @@ private ExprValue handleZeroYear(ExprValue date) { * @return ExprValue. */ private ExprValue exprToSeconds(ExprValue date) { - - //Handle special case of year 0000 according to MySQL docs. - if (date.toString().contains("0000-")) { - return handleZeroYear(date); - } - return new ExprLongValue( date.datetimeValue().toEpochSecond(ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); } @@ -1685,18 +1653,22 @@ private ExprValue exprToSeconds(ExprValue date) { /** * To_seconds implementation with an integer argument for ExprValue. * - * @param date ExprValue of an Integer/Long formatted for a date (e.g., 950501 = 1995-05-01) + * @param dateExpr ExprValue of an Integer/Long formatted for a date (e.g., 950501 = 1995-05-01) * @return ExprValue. */ - private ExprValue exprToSecondsForIntType(ExprValue date) { - int day = date.integerValue() % 10; - int month = date.integerValue() % 10000 - day; - int year = date.integerValue() - month - day; + private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { + try { + if(dateExpr.longValue() < 0 || dateExpr.longValue() > 999999) { + throw new DateTimeException("Integer argument was out of range"); + } - LocalDateTime datetime = LocalDateTime.of(1900 + year/10000, month/100, day, 0, 0, 0); + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), DATE_FORMATTER_SHORT_YEAR); + return new ExprLongValue( + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - return new ExprLongValue( - datetime.toEpochSecond(ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } catch (DateTimeParseException e) { + return null; + } } /** diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 675d7f3218..8b46e4210a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -154,8 +154,6 @@ public void setup() { ImmutableList.of("1998-01-31b 13:14:15 b")) ); - private static final long SECONDS_FROM_0001_01_01_TO_EPOCH_START = 62167219200L; - @AllArgsConstructor private class DateFormatTester { private final String date; @@ -1806,60 +1804,6 @@ public void to_days() { assertEquals(longValue(719527L), eval(expression)); } - private static Stream getTestDataForToSeconds() { - return Stream.of( - Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), - Arguments.of(new ExprStringValue("2009-11-29"), new ExprLongValue(63426672000L)), - Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), - Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)), - Arguments.of(new ExprDatetimeValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), - Arguments.of(new ExprTimestampValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), - Arguments.of(new ExprStringValue("0000-00-00"), ExprNullValue.of()), - Arguments.of(new ExprStringValue("0000-01-01"), new ExprLongValue(86400L)) - ); - } - - @ParameterizedTest - @MethodSource("getTestDataForToSeconds") - public void testToSeconds(ExprValue arg, ExprValue expected) { - lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); - lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - - FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); - assertEquals(LONG, expr.type()); - assertEquals(expected, eval(expr)); - } - - @Test - public void testToSecondsWithTimeType() { - lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); - lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - - FunctionExpression expr = DSL.to_seconds(functionProperties, DSL.literal(new ExprTimeValue("10:11:12"))); - - long expected = SECONDS_FROM_0001_01_01_TO_EPOCH_START + - LocalDate.now(functionProperties.getQueryStartClock()) - .toEpochSecond(LocalTime.parse("10:11:12"), ZoneOffset.UTC); - - assertEquals(expected, eval(expr).longValue()); - } - - private static Stream getInvalidTestDataForToSeconds() { - return Stream.of( - Arguments.of(new ExprLongValue(-123L)) - ); - } - - @ParameterizedTest - @MethodSource("getInvalidTestDataForToSeconds") - public void testToSecondsInvalidArg(ExprValue arg) { - lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); - lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); - - FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); - assertThrows(DateTimeException.class, () -> eval(expr)); - } - @Test public void year() { when(nullRef.type()).thenReturn(DATE); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java new file mode 100644 index 0000000000..68399c26b3 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.datetime; + +import com.google.common.collect.ImmutableList; +import lombok.AllArgsConstructor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTimeValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.env.Environment; + +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Stream; + +import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.missingValue; +import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; + +@ExtendWith(MockitoExtension.class) +class ToSecondsTest extends ExpressionTestBase { + + @Mock + Environment env; + + private static final long SECONDS_FROM_0001_01_01_TO_EPOCH_START = 62167219200L; + + private static Stream getTestDataForToSeconds() { + return Stream.of( + Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), + Arguments.of(new ExprStringValue("2009-11-29 00:00:00"), new ExprLongValue(63426672000L)), + Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), + Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)), + Arguments.of(new ExprDatetimeValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), + Arguments.of(new ExprTimestampValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)) + ); + } + + @ParameterizedTest + @MethodSource("getTestDataForToSeconds") + public void testToSeconds(ExprValue arg, ExprValue expected) { + FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); + assertEquals(LONG, expr.type()); + assertEquals(expected, eval(expr)); + } + + @Test + public void testToSecondsWithTimeType() { + FunctionExpression expr = DSL.to_seconds(functionProperties, DSL.literal(new ExprTimeValue("10:11:12"))); + + long expected = SECONDS_FROM_0001_01_01_TO_EPOCH_START + + LocalDate.now(functionProperties.getQueryStartClock()) + .toEpochSecond(LocalTime.parse("10:11:12"), ZoneOffset.UTC); + + assertEquals(expected, eval(expr).longValue()); + } + + private static Stream getInvalidTestDataForToSeconds() { + return Stream.of( + Arguments.of(new ExprLongValue(-123L)) + ); + } + + @ParameterizedTest + @MethodSource("getInvalidTestDataForToSeconds") + public void testToSecondsInvalidArg(ExprValue arg) { + FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); + assertThrows(DateTimeException.class, () -> eval(expr)); + } + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index c8b0d49dd0..3b72abbe2b 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2465,9 +2465,9 @@ TO_SECONDS Description >>>>>>>>>>> -Usage: to_seconds(date) returns the seconds number (the number of days since year 0) of the given date. Returns NULL if date is invalid. +Usage: to_seconds(date) returns the number of seconds since the year 0 of the given value. Returns NULL if value is invalid. -Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP +Argument type: STRING/LONG/DATE/DATETIME/TIME/TIMESTAMP Return type: LONG From a2045b9582514653ba86924078df70b0c9919d90 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 28 Feb 2023 16:03:01 -0800 Subject: [PATCH 08/18] Fixed Docs and Implementation Signed-off-by: GabeFernandez310 --- .../sql/expression/datetime/DateTimeFunction.java | 15 +++++++++++---- .../sql/expression/datetime/ToSecondsTest.java | 1 + docs/user/dql/functions.rst | 13 +++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 38b0400d0d..a09e16c47d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -1662,12 +1662,19 @@ private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { throw new DateTimeException("Integer argument was out of range"); } - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), DATE_FORMATTER_SHORT_YEAR); + try { + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), DATE_FORMATTER_SHORT_YEAR); + return new ExprLongValue( + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } catch (DateTimeParseException ignored) { + //ignore parse exception and try next format + } + + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), DATE_TIME_FORMATTER_LONG_YEAR); return new ExprLongValue( date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - - } catch (DateTimeParseException e) { - return null; + } catch (DateTimeException e) { + return ExprNullValue.of(); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index 68399c26b3..c18291f401 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -72,6 +72,7 @@ class ToSecondsTest extends ExpressionTestBase { private static Stream getTestDataForToSeconds() { return Stream.of( Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), + Arguments.of(new ExprLongValue(9950501), ExprNullValue.of()), Arguments.of(new ExprStringValue("2009-11-29 00:00:00"), new ExprLongValue(63426672000L)), Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)), diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 3b72abbe2b..f90d90247c 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2466,6 +2466,7 @@ Description >>>>>>>>>>> Usage: to_seconds(date) returns the number of seconds since the year 0 of the given value. Returns NULL if value is invalid. +An argument of a LONG type can be used. It must be formatted as YMMDD, YYMMDD, or YYYYMMDD. Argument type: STRING/LONG/DATE/DATETIME/TIME/TIMESTAMP @@ -2473,13 +2474,13 @@ Return type: LONG Example:: - os> SELECT TO_SECONDS(DATE '2008-10-07') + os> SELECT TO_SECONDS(DATE '2008-10-07'), TO_SECONDS(950228) fetched rows / total rows = 1/1 - +---------------------------------+ - | TO_SECONDS(DATE '2008-10-07') | - |---------------------------------| - | 63390556800 | - +---------------------------------+ + +---------------------------------+----------------------+ + | TO_SECONDS(DATE '2008-10-07') | TO_SECONDS(950228) | + |---------------------------------+----------------------| + | 63390556800 | | + +---------------------------------+----------------------+ UNIX_TIMESTAMP -------------- From 26f4cda4e5d5f6f58717805183f1f01f09586e6d Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 28 Feb 2023 16:19:41 -0800 Subject: [PATCH 09/18] Fixed Checkstyle Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 1 + .../expression/datetime/DateTimeFunction.java | 16 ++++-- .../datetime/DateTimeFunctionTest.java | 5 -- .../expression/datetime/ToSecondsTest.java | 56 ++++++------------- docs/user/dql/functions.rst | 2 +- 5 files changed, 31 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 654aaf4bed..cc7ade42ec 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -433,6 +433,7 @@ public static FunctionExpression date_format( public static FunctionExpression to_days(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.TO_DAYS, expressions); } + public static FunctionExpression to_seconds(FunctionProperties functionProperties, Expression... expressions) { return compile(functionProperties, BuiltinFunctionName.TO_SECONDS, expressions); diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index a09e16c47d..0408b9c38f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -1658,21 +1658,27 @@ private ExprValue exprToSeconds(ExprValue date) { */ private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { try { - if(dateExpr.longValue() < 0 || dateExpr.longValue() > 999999) { + if (dateExpr.longValue() < 0 || dateExpr.longValue() > 999999) { throw new DateTimeException("Integer argument was out of range"); } try { - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), DATE_FORMATTER_SHORT_YEAR); + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), + DATE_FORMATTER_SHORT_YEAR); + return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + + DAYS_0000_TO_1970 * SECONDS_PER_DAY); } catch (DateTimeParseException ignored) { //ignore parse exception and try next format } - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), DATE_TIME_FORMATTER_LONG_YEAR); + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), + DATE_TIME_FORMATTER_LONG_YEAR); + return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + + DAYS_0000_TO_1970 * SECONDS_PER_DAY); } catch (DateTimeException e) { return ExprNullValue.of(); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 8b46e4210a..fdb029ca90 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -26,13 +26,8 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; - -import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.stream.Stream; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index c18291f401..fe9444e92b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -6,9 +6,15 @@ package org.opensearch.sql.expression.datetime; -import com.google.common.collect.ImmutableList; -import lombok.AllArgsConstructor; -import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; + +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -24,42 +30,12 @@ import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.env.Environment; -import java.time.DateTimeException; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.stream.Stream; - -import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.model.ExprValueUtils.longValue; -import static org.opensearch.sql.data.model.ExprValueUtils.missingValue; -import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; -import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; -import static org.opensearch.sql.data.type.ExprCoreType.DATE; -import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.LONG; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.data.type.ExprCoreType.TIME; -import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; @ExtendWith(MockitoExtension.class) class ToSecondsTest extends ExpressionTestBase { @@ -76,8 +52,10 @@ private static Stream getTestDataForToSeconds() { Arguments.of(new ExprStringValue("2009-11-29 00:00:00"), new ExprLongValue(63426672000L)), Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)), - Arguments.of(new ExprDatetimeValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), - Arguments.of(new ExprTimestampValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)) + Arguments.of(new ExprDatetimeValue("2009-11-29 13:43:32"), + new ExprLongValue(63426721412L)), + Arguments.of(new ExprTimestampValue("2009-11-29 13:43:32"), + new ExprLongValue(63426721412L)) ); } @@ -91,10 +69,11 @@ public void testToSeconds(ExprValue arg, ExprValue expected) { @Test public void testToSecondsWithTimeType() { - FunctionExpression expr = DSL.to_seconds(functionProperties, DSL.literal(new ExprTimeValue("10:11:12"))); + FunctionExpression expr = DSL.to_seconds(functionProperties, + DSL.literal(new ExprTimeValue("10:11:12"))); - long expected = SECONDS_FROM_0001_01_01_TO_EPOCH_START + - LocalDate.now(functionProperties.getQueryStartClock()) + long expected = SECONDS_FROM_0001_01_01_TO_EPOCH_START + + LocalDate.now(functionProperties.getQueryStartClock()) .toEpochSecond(LocalTime.parse("10:11:12"), ZoneOffset.UTC); assertEquals(expected, eval(expr).longValue()); @@ -112,6 +91,7 @@ public void testToSecondsInvalidArg(ExprValue arg) { FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); assertThrows(DateTimeException.class, () -> eval(expr)); } + private ExprValue eval(Expression expression) { return expression.valueOf(env); } diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index f90d90247c..8ff2ebcc20 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2479,7 +2479,7 @@ Example:: +---------------------------------+----------------------+ | TO_SECONDS(DATE '2008-10-07') | TO_SECONDS(950228) | |---------------------------------+----------------------| - | 63390556800 | | + | 63390556800 | 62961148800 | +---------------------------------+----------------------+ UNIX_TIMESTAMP From 0dcd3ce9b00dee691de7e2593ef28ad69931a912 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Wed, 1 Mar 2023 09:14:56 -0800 Subject: [PATCH 10/18] Changed DateTimeFormatter Priority Signed-off-by: GabeFernandez310 --- .../sql/expression/datetime/DateTimeFunction.java | 6 +++--- .../opensearch/sql/expression/datetime/ToSecondsTest.java | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 0408b9c38f..e7dd478f28 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -1658,13 +1658,13 @@ private ExprValue exprToSeconds(ExprValue date) { */ private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { try { - if (dateExpr.longValue() < 0 || dateExpr.longValue() > 999999) { + if (dateExpr.longValue() < 0 || dateExpr.longValue() > 99999999) { throw new DateTimeException("Integer argument was out of range"); } try { LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_SHORT_YEAR); + DATE_FORMATTER_LONG_YEAR); return new ExprLongValue( date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) @@ -1674,7 +1674,7 @@ private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { } LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_TIME_FORMATTER_LONG_YEAR); + DATE_FORMATTER_SHORT_YEAR); return new ExprLongValue( date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index fe9444e92b..53550af2c2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -48,6 +48,7 @@ class ToSecondsTest extends ExpressionTestBase { private static Stream getTestDataForToSeconds() { return Stream.of( Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), + Arguments.of(new ExprLongValue(19950501), new ExprLongValue(62966505600L)), Arguments.of(new ExprLongValue(9950501), ExprNullValue.of()), Arguments.of(new ExprStringValue("2009-11-29 00:00:00"), new ExprLongValue(63426672000L)), Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), From d4344d39079eeee62b20eaeb70f4f965ae944f73 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Wed, 1 Mar 2023 11:09:28 -0800 Subject: [PATCH 11/18] Added More Formatters Signed-off-by: GabeFernandez310 --- .../expression/datetime/DateTimeFunction.java | 38 ++++++++++++++++++- .../sql/utils/DateTimeFormatters.java | 24 ++++++++++++ .../expression/datetime/ToSecondsTest.java | 3 ++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index e7dd478f28..310cb7a034 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -27,7 +27,10 @@ import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_LONG_YEAR; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_NO_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SHORT_YEAR; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SINGLE_DIGIT_MONTH; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SINGLE_DIGIT_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_LONG_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_STRICT_WITH_TZ; @@ -1673,8 +1676,41 @@ private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { //ignore parse exception and try next format } + try { + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), + DATE_FORMATTER_SHORT_YEAR); + + return new ExprLongValue( + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } catch (DateTimeParseException ignored) { + //ignore parse exception and try next format + } + + try { + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), + DATE_FORMATTER_SINGLE_DIGIT_YEAR); + + return new ExprLongValue( + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } catch (DateTimeParseException ignored) { + //ignore parse exception and try next format + } + + try { + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), + DATE_FORMATTER_NO_YEAR); + + return new ExprLongValue( + date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + + DAYS_0000_TO_1970 * SECONDS_PER_DAY); + } catch (DateTimeParseException ignored) { + //ignore parse exception and try next format + } + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_SHORT_YEAR); + DATE_FORMATTER_SINGLE_DIGIT_MONTH); return new ExprLongValue( date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 2556aed8d8..7fd0c4bbfe 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -133,6 +133,30 @@ public class DateTimeFormatters { .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT); + // MMDD + public static final DateTimeFormatter DATE_FORMATTER_SINGLE_DIGIT_MONTH = + new DateTimeFormatterBuilder() + .parseDefaulting(YEAR, 2000) + .appendPattern("Mdd") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + + // MMDD + public static final DateTimeFormatter DATE_FORMATTER_NO_YEAR = + new DateTimeFormatterBuilder() + .parseDefaulting(YEAR, 2000) + .appendPattern("MMdd") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + + // YMMDD + public static final DateTimeFormatter DATE_FORMATTER_SINGLE_DIGIT_YEAR = + new DateTimeFormatterBuilder() + .appendValueReduced(YEAR, 1, 1, 2000) + .appendPattern("MMdd") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + // YYMMDD public static final DateTimeFormatter DATE_FORMATTER_SHORT_YEAR = new DateTimeFormatterBuilder() diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index 53550af2c2..6665e0d8d2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -47,6 +47,9 @@ class ToSecondsTest extends ExpressionTestBase { private static Stream getTestDataForToSeconds() { return Stream.of( + Arguments.of(new ExprLongValue(101), new ExprLongValue(63113904000L)), + Arguments.of(new ExprLongValue(1030), new ExprLongValue(63140083200L)), + Arguments.of(new ExprLongValue(50101), new ExprLongValue(63271756800L)), Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), Arguments.of(new ExprLongValue(19950501), new ExprLongValue(62966505600L)), Arguments.of(new ExprLongValue(9950501), ExprNullValue.of()), From 12b981c5edbd1d1ca0ec2d8e3d879b000c29b1e7 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Wed, 1 Mar 2023 14:47:22 -0800 Subject: [PATCH 12/18] Updated Docs Signed-off-by: GabeFernandez310 --- docs/user/dql/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 8ff2ebcc20..0b7b244a40 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2466,7 +2466,7 @@ Description >>>>>>>>>>> Usage: to_seconds(date) returns the number of seconds since the year 0 of the given value. Returns NULL if value is invalid. -An argument of a LONG type can be used. It must be formatted as YMMDD, YYMMDD, or YYYYMMDD. +An argument of a LONG type can be used. It must be formatted as YMMDD, YYMMDD, YYYMMDD or YYYYMMDD. Note that a LONG type argument cannot have leading 0s as it will be parsed using an octal numbering system. Argument type: STRING/LONG/DATE/DATETIME/TIME/TIMESTAMP From ec908addea8a1f53a8d220a3c32b52b528f385f7 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Thu, 2 Mar 2023 11:20:10 -0800 Subject: [PATCH 13/18] Reworked Implementation For Formatters Signed-off-by: GabeFernandez310 --- .../expression/datetime/DateTimeFunction.java | 94 +++++++++---------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 310cb7a034..e02ffdebc0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -1654,68 +1654,64 @@ private ExprValue exprToSeconds(ExprValue date) { } /** - * To_seconds implementation with an integer argument for ExprValue. + * Helper function to determine the correct formatter for date arguments passed in as integers. * - * @param dateExpr ExprValue of an Integer/Long formatted for a date (e.g., 950501 = 1995-05-01) - * @return ExprValue. + * @param dateAsInt is an integer formatted as one of YYYYMMDD, YYMMDD, YMMDD, MMDD, MDD + * @return is a DateTimeFormatter that can parse the input. */ - private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { - try { - if (dateExpr.longValue() < 0 || dateExpr.longValue() > 99999999) { - throw new DateTimeException("Integer argument was out of range"); - } + private DateTimeFormatter getFormatter(int dateAsInt) { + if (dateAsInt < 0 || dateAsInt > 99999999) { + throw new DateTimeException("Integer argument was out of range"); + } - try { - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_LONG_YEAR); + //Check below from YYYYMMDD - MMDD which format should be used - return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) - + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - } catch (DateTimeParseException ignored) { - //ignore parse exception and try next format - } + //Check if dateAsInt is at least 8 digits long + if (dateAsInt - 9999999 > 0) { + return DATE_FORMATTER_LONG_YEAR; + } - try { - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_SHORT_YEAR); + //Check if dateAsInt is at least 6 digits long + if (dateAsInt - 99999 > 0) { + return DATE_FORMATTER_SHORT_YEAR; + } - return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) - + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - } catch (DateTimeParseException ignored) { - //ignore parse exception and try next format - } + //Check if dateAsInt is at least 5 digits long + if (dateAsInt - 9999 > 0) { + return DATE_FORMATTER_SINGLE_DIGIT_YEAR; + } - try { - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_SINGLE_DIGIT_YEAR); + //Check if dateAsInt is at least 4 digits long + if (dateAsInt - 999 > 0) { + return DATE_FORMATTER_NO_YEAR; + } - return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) - + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - } catch (DateTimeParseException ignored) { - //ignore parse exception and try next format - } + //Check if dateAsInt is at least 3 digits long + if (dateAsInt - 99 > 0) { + return DATE_FORMATTER_SINGLE_DIGIT_MONTH; + } - try { - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_NO_YEAR); + throw new DateTimeException("No Matching Format"); - return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) - + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - } catch (DateTimeParseException ignored) { - //ignore parse exception and try next format - } + } + /** + * To_seconds implementation with an integer argument for ExprValue. + * + * @param dateExpr ExprValue of an Integer/Long formatted for a date (e.g., 950501 = 1995-05-01) + * @return ExprValue. + */ + private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { + try { + //Attempt to parse integer argument as date LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), - DATE_FORMATTER_SINGLE_DIGIT_MONTH); + getFormatter(dateExpr.integerValue())); + + return new ExprLongValue(date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - return new ExprLongValue( - date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) - + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - } catch (DateTimeException e) { + } catch (DateTimeParseException ignored) { + //Return null if parsing error return ExprNullValue.of(); } } From 8388705ffa522ae1eff6581c829dcf4b4ccc5b34 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Thu, 2 Mar 2023 12:58:26 -0800 Subject: [PATCH 14/18] Cleanup Signed-off-by: GabeFernandez310 --- .../sql/expression/datetime/DateTimeFunction.java | 6 ++++-- .../java/org/opensearch/sql/utils/DateTimeFormatters.java | 2 +- .../opensearch/sql/expression/datetime/ToSecondsTest.java | 5 +---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index e02ffdebc0..3769d98218 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -834,7 +834,9 @@ private DefaultFunctionResolver to_days() { } /** - * TO_SECONDS(STRING/DATE/DATETIME/TIMESTAMP/LONG). return the seconds number of the given date. + * TO_SECONDS(TIMESTAMP/LONG). return the seconds number of the given date. + * Arguments of type STRING/TIMESTAMP/LONG are also accepted. + * STRING/TIMESTAMP/LONG arguments are automatically cast to TIMESTAMP. */ private DefaultFunctionResolver to_seconds() { return define(BuiltinFunctionName.TO_SECONDS.getName(), @@ -1704,7 +1706,7 @@ private DateTimeFormatter getFormatter(int dateAsInt) { private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { try { //Attempt to parse integer argument as date - LocalDate date = LocalDate.parse(String.valueOf(dateExpr.longValue()), + LocalDate date = LocalDate.parse(String.valueOf(dateExpr.integerValue()), getFormatter(dateExpr.integerValue())); return new ExprLongValue(date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 7fd0c4bbfe..99344046e6 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -133,7 +133,7 @@ public class DateTimeFormatters { .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT); - // MMDD + // MDD public static final DateTimeFormatter DATE_FORMATTER_SINGLE_DIGIT_MONTH = new DateTimeFormatterBuilder() .parseDefaulting(YEAR, 2000) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index 6665e0d8d2..6f12878c85 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -40,9 +40,6 @@ @ExtendWith(MockitoExtension.class) class ToSecondsTest extends ExpressionTestBase { - @Mock - Environment env; - private static final long SECONDS_FROM_0001_01_01_TO_EPOCH_START = 62167219200L; private static Stream getTestDataForToSeconds() { @@ -97,6 +94,6 @@ public void testToSecondsInvalidArg(ExprValue arg) { } private ExprValue eval(Expression expression) { - return expression.valueOf(env); + return expression.valueOf(); } } From fb3c19b88ede07ecc2a25c97cb3efe8cbbf8b730 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Thu, 2 Mar 2023 14:22:25 -0800 Subject: [PATCH 15/18] Added Test Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/expression/DSL.java | 4 ++++ .../expression/datetime/DateTimeFunction.java | 5 +++-- .../expression/datetime/ToSecondsTest.java | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index cc7ade42ec..f4ad0fd178 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -306,6 +306,10 @@ public static FunctionExpression datetime(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DATETIME, expressions); } + public static FunctionExpression date_add(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.DATE_ADD, expressions); + } + public static FunctionExpression day(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DAY, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 3769d98218..3cc554f96e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -96,6 +96,9 @@ @UtilityClass @SuppressWarnings("unchecked") public class DateTimeFunction { + //The number of seconds per day + public static final long SECONDS_PER_DAY = 86400; + // The number of days from year zero to year 1970. private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L); @@ -106,8 +109,6 @@ public class DateTimeFunction { // Mode used for week/week_of_year function by default when no argument is provided private static final ExprIntegerValue DEFAULT_WEEK_OF_YEAR_MODE = new ExprIntegerValue(0); - //The number of seconds per day - private static final long SECONDS_PER_DAY = 86400; // Map used to determine format output for the get_format function private static final Table formats = diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index 6f12878c85..e7699d3e6e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -9,8 +9,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.expression.datetime.DateTimeFunction.SECONDS_PER_DAY; import java.time.DateTimeException; +import java.time.Duration; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneOffset; @@ -24,6 +26,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprIntervalValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; @@ -93,6 +96,22 @@ public void testToSecondsInvalidArg(ExprValue arg) { assertThrows(DateTimeException.class, () -> eval(expr)); } + @Test + public void testToSecondsWithDateAdd() { + LocalDate date = LocalDate.of(2000, 1, 1); + FunctionExpression dateExpr = DSL.to_seconds(DSL.literal(new ExprDateValue(date))); + long addedSeconds = SECONDS_PER_DAY; + long expected = eval(dateExpr).longValue() + addedSeconds; + + FunctionExpression dateAddExpr = DSL.date_add( + DSL.literal(new ExprDateValue(date)), + DSL.literal(new ExprIntervalValue(Duration.ofSeconds(addedSeconds)))); + + long result = eval(DSL.to_seconds(DSL.literal(eval(dateAddExpr)))).longValue(); + + assertEquals(expected, result); + } + private ExprValue eval(Expression expression) { return expression.valueOf(); } From d30f7490d7212fc9b66c139547ac87a793ea1fe4 Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Tue, 7 Mar 2023 09:39:50 -0800 Subject: [PATCH 16/18] Fixed Implementation And Code Coverage Signed-off-by: GabeFernandez310 --- .../sql/expression/datetime/DateTimeFunction.java | 2 +- .../sql/expression/datetime/ToSecondsTest.java | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 3cc554f96e..49c71fe4ef 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -1713,7 +1713,7 @@ private ExprValue exprToSecondsForIntType(ExprValue dateExpr) { return new ExprLongValue(date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY); - } catch (DateTimeParseException ignored) { + } catch (DateTimeException ignored) { //Return null if parsing error return ExprNullValue.of(); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java index e7699d3e6e..7847bc61ec 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java @@ -33,11 +33,11 @@ import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.env.Environment; @ExtendWith(MockitoExtension.class) @@ -53,6 +53,9 @@ private static Stream getTestDataForToSeconds() { Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)), Arguments.of(new ExprLongValue(19950501), new ExprLongValue(62966505600L)), Arguments.of(new ExprLongValue(9950501), ExprNullValue.of()), + Arguments.of(new ExprLongValue(-123L), ExprNullValue.of()), + Arguments.of(new ExprLongValue(1), ExprNullValue.of()), + Arguments.of(new ExprLongValue(919950501), ExprNullValue.of()), Arguments.of(new ExprStringValue("2009-11-29 00:00:00"), new ExprLongValue(63426672000L)), Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)), Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)), @@ -85,7 +88,12 @@ public void testToSecondsWithTimeType() { private static Stream getInvalidTestDataForToSeconds() { return Stream.of( - Arguments.of(new ExprLongValue(-123L)) + Arguments.of(new ExprStringValue("asdfasdf")), + Arguments.of(new ExprStringValue("2000-14-10")), + Arguments.of(new ExprStringValue("2000-10-45")), + Arguments.of(new ExprStringValue("2000-10-10 70:00:00")), + Arguments.of(new ExprStringValue("2000-10-10 00:70:00")), + Arguments.of(new ExprStringValue("2000-10-10 00:00:70")) ); } @@ -93,7 +101,7 @@ private static Stream getInvalidTestDataForToSeconds() { @MethodSource("getInvalidTestDataForToSeconds") public void testToSecondsInvalidArg(ExprValue arg) { FunctionExpression expr = DSL.to_seconds(DSL.literal(arg)); - assertThrows(DateTimeException.class, () -> eval(expr)); + assertThrows(SemanticCheckException.class, () -> eval(expr)); } @Test From 2b37b0d4090aca8b66dbb02ba9a47e0eda5e27eb Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Wed, 8 Mar 2023 08:34:31 -0800 Subject: [PATCH 17/18] Changed getFormatter Function Signed-off-by: GabeFernandez310 --- .../expression/datetime/DateTimeFunction.java | 47 ++++++++++--------- .../sql/utils/DateTimeFormatters.java | 6 +++ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 49c71fe4ef..e3cd56cdb9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -34,6 +34,11 @@ import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_LONG_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_STRICT_WITH_TZ; +import static org.opensearch.sql.utils.DateTimeFormatters.FULL_DATE_LENGTH; +import static org.opensearch.sql.utils.DateTimeFormatters.NO_YEAR_DATE_LENGTH; +import static org.opensearch.sql.utils.DateTimeFormatters.SHORT_DATE_LENGTH; +import static org.opensearch.sql.utils.DateTimeFormatters.SINGLE_DIGIT_MONTH_DATE_LENGTH; +import static org.opensearch.sql.utils.DateTimeFormatters.SINGLE_DIGIT_YEAR_DATE_LENGTH; import static org.opensearch.sql.utils.DateTimeUtils.extractDate; import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime; @@ -1663,39 +1668,39 @@ private ExprValue exprToSeconds(ExprValue date) { * @return is a DateTimeFormatter that can parse the input. */ private DateTimeFormatter getFormatter(int dateAsInt) { - if (dateAsInt < 0 || dateAsInt > 99999999) { + int length = String.format("%d", dateAsInt).length(); + + if (length > 8) { throw new DateTimeException("Integer argument was out of range"); } //Check below from YYYYMMDD - MMDD which format should be used + switch (length) { + //Check if dateAsInt is at least 8 digits long + case FULL_DATE_LENGTH: + return DATE_FORMATTER_LONG_YEAR; - //Check if dateAsInt is at least 8 digits long - if (dateAsInt - 9999999 > 0) { - return DATE_FORMATTER_LONG_YEAR; - } + //Check if dateAsInt is at least 6 digits long + case SHORT_DATE_LENGTH: + return DATE_FORMATTER_SHORT_YEAR; - //Check if dateAsInt is at least 6 digits long - if (dateAsInt - 99999 > 0) { - return DATE_FORMATTER_SHORT_YEAR; - } + //Check if dateAsInt is at least 5 digits long + case SINGLE_DIGIT_YEAR_DATE_LENGTH: + return DATE_FORMATTER_SINGLE_DIGIT_YEAR; - //Check if dateAsInt is at least 5 digits long - if (dateAsInt - 9999 > 0) { - return DATE_FORMATTER_SINGLE_DIGIT_YEAR; - } + //Check if dateAsInt is at least 4 digits long + case NO_YEAR_DATE_LENGTH: + return DATE_FORMATTER_NO_YEAR; - //Check if dateAsInt is at least 4 digits long - if (dateAsInt - 999 > 0) { - return DATE_FORMATTER_NO_YEAR; - } + //Check if dateAsInt is at least 3 digits long + case SINGLE_DIGIT_MONTH_DATE_LENGTH: + return DATE_FORMATTER_SINGLE_DIGIT_MONTH; - //Check if dateAsInt is at least 3 digits long - if (dateAsInt - 99 > 0) { - return DATE_FORMATTER_SINGLE_DIGIT_MONTH; + default: + break; } throw new DateTimeException("No Matching Format"); - } /** diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 99344046e6..3bee80b8a0 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -29,6 +29,12 @@ @UtilityClass public class DateTimeFormatters { + public static final int FULL_DATE_LENGTH = 8; + public static final int SHORT_DATE_LENGTH = 6; + public static final int SINGLE_DIGIT_YEAR_DATE_LENGTH = 5; + public static final int NO_YEAR_DATE_LENGTH = 4; + public static final int SINGLE_DIGIT_MONTH_DATE_LENGTH = 3; + public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() .appendOffset("+HHmm", "Z") From 8d2587e918d2ede9fb0b826ac9dd6c4faf002ebc Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Thu, 9 Mar 2023 09:02:31 -0800 Subject: [PATCH 18/18] Added Comments Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/utils/DateTimeFormatters.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 3bee80b8a0..a9ea53f142 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -29,10 +29,19 @@ @UtilityClass public class DateTimeFormatters { + //Length of a date formatted as YYYYMMDD. public static final int FULL_DATE_LENGTH = 8; + + //Length of a date formatted as YYMMDD. public static final int SHORT_DATE_LENGTH = 6; + + //Length of a date formatted as YMMDD. public static final int SINGLE_DIGIT_YEAR_DATE_LENGTH = 5; + + //Length of a date formatted as MMDD. public static final int NO_YEAR_DATE_LENGTH = 4; + + //Length of a date formatted as MDD. public static final int SINGLE_DIGIT_MONTH_DATE_LENGTH = 3; public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON =