Add STR_TO_DATE Function To The SQL Plugin (#1420) (#1444)

* Add `STR_TO_DATE` Function To The SQL Plugin. * Added Tests Signed-off-by: GabeFernandez310 <[email protected]> * Added Empty Implementation Signed-off-by: GabeFernandez310 <[email protected]> * Added Partially Working Implementation Signed-off-by: GabeFernandez310 <[email protected]> * Cleaned Up Implementation Signed-off-by: GabeFernandez310 <[email protected]> * Modified Implementation Signed-off-by: GabeFernandez310 <[email protected]> * Modified IT Test Signed-off-by: GabeFernandez310 <[email protected]> * Added Documentation Signed-off-by: GabeFernandez310 <[email protected]> * Added Unit Tests Signed-off-by: GabeFernandez310 <[email protected]> * Altered Implementation Signed-off-by: GabeFernandez310 <[email protected]> * Reworked Implementation To Always Return Datetime Signed-off-by: GabeFernandez310 <[email protected]> * Addressed PR Comments Signed-off-by: GabeFernandez310 <[email protected]> * Fixed Some Checkstyle Issues Signed-off-by: GabeFernandez310 <[email protected]> * Cleaned Implementation Signed-off-by: GabeFernandez310 <[email protected]> * Supported Function Properties Signed-off-by: GabeFernandez310 <[email protected]> * Fixed Checkstyle Signed-off-by: GabeFernandez310 <[email protected]> * Removed Unneeded Function Signed-off-by: GabeFernandez310 <[email protected]> * Cleaned Implementation and Added IT Test Signed-off-by: GabeFernandez310 <[email protected]> * Fixed Code Coverage and Implementation Signed-off-by: GabeFernandez310 <[email protected]> --------- Signed-off-by: GabeFernandez310 <[email protected]> * Cleaned Implementation Signed-off-by: GabeFernandez310 <[email protected]> --------- Signed-off-by: GabeFernandez310 <[email protected]> (cherry picked from commit 40336d4)
opensearch-project · Mar 16, 2023 · 5f1747d · 5f1747d
1 parent 29f005f
commit 5f1747d
Show file tree

Hide file tree

Showing 10 changed files with 453 additions and 8 deletions.
diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java
@@ -488,6 +488,12 @@ public static FunctionExpression module(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.MODULES, expressions);
   }
 
+
+  public static FunctionExpression str_to_date(FunctionProperties functionProperties,
+                                               Expression... expressions) {
+    return compile(functionProperties, BuiltinFunctionName.STR_TO_DATE, expressions);
+  }
+
   public static FunctionExpression sec_to_time(Expression... expressions) {
     return  compile(FunctionProperties.None, BuiltinFunctionName.SEC_TO_TIME, expressions);
   }

diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java
@@ -6,17 +6,26 @@
 package org.opensearch.sql.expression.datetime;
 
 import com.google.common.collect.ImmutableMap;
+import java.text.ParsePosition;
 import java.time.Clock;
+import java.time.DateTimeException;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
+import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.ResolverStyle;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
 import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import org.opensearch.sql.data.model.ExprDatetimeValue;
 import org.opensearch.sql.data.model.ExprNullValue;
 import org.opensearch.sql.data.model.ExprStringValue;
 import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.expression.function.FunctionProperties;
 
 /**
  * This class converts a SQL style DATE_FORMAT format specifier and converts it to a
@@ -28,6 +37,7 @@ class DateTimeFormatterUtil {
   private static final String SUFFIX_SPECIAL_TH = "th";
 
   private static final String NANO_SEC_FORMAT = "'%06d'";
+
   private static final Map<Integer, String> SUFFIX_CONVERTER =
       ImmutableMap.<Integer, String>builder()
       .put(1, "st").put(2, "nd").put(3, "rd").build();
@@ -122,6 +132,43 @@ interface DateTimeFormatHandler {
           .put("%x", (date) -> null)
           .build();
 
+  private static final Map<String, String> STR_TO_DATE_FORMATS =
+      ImmutableMap.<String, String>builder()
+          .put("%a", "EEE") // %a => EEE - Abbreviated weekday name (Sun..Sat)
+          .put("%b", "LLL") // %b => LLL - Abbreviated month name (Jan..Dec)
+          .put("%c", "M") // %c => MM - Month, numeric (0..12)
+          .put("%d", "d") // %d => dd - Day of the month, numeric (00..31)
+          .put("%e", "d") // %e => d - Day of the month, numeric (0..31)
+          .put("%H", "H") // %H => HH - (00..23)
+          .put("%h", "H") // %h => hh - (01..12)
+          .put("%I", "h") // %I => hh - (01..12)
+          .put("%i", "m") // %i => mm - Minutes, numeric (00..59)
+          .put("%j", "DDD") // %j => DDD - (001..366)
+          .put("%k", "H") // %k => H - (0..23)
+          .put("%l", "h") // %l => h - (1..12)
+          .put("%p", "a") // %p => a - AM or PM
+          .put("%M", "LLLL") // %M => LLLL - Month name (January..December)
+          .put("%m", "M") // %m => MM - Month, numeric (00..12)
+          .put("%r", "hh:mm:ss a") // %r => hh:mm:ss a - hh:mm:ss followed by AM or PM
+          .put("%S", "s") // %S => ss - Seconds (00..59)
+          .put("%s", "s") // %s => ss - Seconds (00..59)
+          .put("%T", "HH:mm:ss") // %T => HH:mm:ss
+          .put("%W", "EEEE") // %W => EEEE - Weekday name (Sunday..Saturday)
+          .put("%Y", "u") // %Y => yyyy - Year, numeric, 4 digits
+          .put("%y", "u") // %y => yy - Year, numeric, 2 digits
+          .put("%f", "n") // %f => n - Nanoseconds
+          //The following have been implemented but cannot be aligned with
+          // MySQL due to the limitations of the DatetimeFormatter
+          .put("%D", "d") // %w - Day of month with English suffix
+          .put("%w", "e") // %w - Day of week (0 indexed)
+          .put("%U", "w") // %U Week where Sunday is the first day - WEEK() mode 0
+          .put("%u", "w") // %u Week where Monday is the first day - WEEK() mode 1
+          .put("%V", "w") // %V Week where Sunday is the first day - WEEK() mode 2
+          .put("%v", "w") // %v Week where Monday is the first day - WEEK() mode 3
+          .put("%X", "u") // %X Year for week where Sunday is the first day
+          .put("%x", "u") // %x Year for week where Monday is the first day
+          .build();
+
   private static final Pattern pattern = Pattern.compile("%.");
   private static final Pattern CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
           = Pattern.compile("(?<!%)[a-zA-Z&&[^aydmshiHIMYDSEL]]+");
@@ -130,6 +177,19 @@ interface DateTimeFormatHandler {
   private DateTimeFormatterUtil() {
   }
 
+  static StringBuffer getCleanFormat(ExprValue formatExpr) {
+    final StringBuffer cleanFormat = new StringBuffer();
+    final Matcher m = CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
+        .matcher(formatExpr.stringValue());
+
+    while (m.find()) {
+      m.appendReplacement(cleanFormat,String.format("'%s'", m.group()));
+    }
+    m.appendTail(cleanFormat);
+
+    return cleanFormat;
+  }
+
   /**
    * Helper function to format a DATETIME according to a provided handler and matcher.
    * @param formatExpr ExprValue containing the format expression
@@ -140,14 +200,7 @@ private DateTimeFormatterUtil() {
   static ExprValue getFormattedString(ExprValue formatExpr,
                                       Map<String, DateTimeFormatHandler> handler,
                                       LocalDateTime datetime) {
-    final StringBuffer cleanFormat = new StringBuffer();
-    final Matcher m = CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
-        .matcher(formatExpr.stringValue());
-
-    while (m.find()) {
-      m.appendReplacement(cleanFormat,String.format("'%s'", m.group()));
-    }
-    m.appendTail(cleanFormat);
+    StringBuffer cleanFormat = getCleanFormat(formatExpr);
 
     final Matcher matcher = pattern.matcher(cleanFormat.toString());
     final StringBuffer format = new StringBuffer();
@@ -201,6 +254,84 @@ static ExprValue getFormattedTime(ExprValue timeExpr, ExprValue formatExpr) {
     return getFormattedString(formatExpr, TIME_HANDLERS, time);
   }
 
+  private static boolean canGetDate(TemporalAccessor ta) {
+    return (ta.isSupported(ChronoField.YEAR)
+        && ta.isSupported(ChronoField.MONTH_OF_YEAR)
+        && ta.isSupported(ChronoField.DAY_OF_MONTH));
+  }
+
+  private static boolean canGetTime(TemporalAccessor ta) {
+    return (ta.isSupported(ChronoField.HOUR_OF_DAY)
+        && ta.isSupported(ChronoField.MINUTE_OF_HOUR)
+        && ta.isSupported(ChronoField.SECOND_OF_MINUTE));
+  }
+
+  static ExprValue parseStringWithDateOrTime(FunctionProperties fp,
+                                             ExprValue datetimeStringExpr,
+                                             ExprValue formatExpr) {
+
+    //Replace patterns with % for Java DateTimeFormatter
+    StringBuffer cleanFormat = getCleanFormat(formatExpr);
+    final Matcher matcher = pattern.matcher(cleanFormat.toString());
+    final StringBuffer format = new StringBuffer();
+
+    while (matcher.find()) {
+      matcher.appendReplacement(format,
+          STR_TO_DATE_FORMATS.getOrDefault(matcher.group(),
+              String.format("'%s'", matcher.group().replaceFirst(MOD_LITERAL, ""))));
+    }
+    matcher.appendTail(format);
+
+    TemporalAccessor taWithMissingFields;
+    //Return NULL for invalid parse in string to align with MySQL
+    try {
+      //Get Temporal Accessor to initially parse string without default values
+      taWithMissingFields = new DateTimeFormatterBuilder()
+          .appendPattern(format.toString())
+          .toFormatter().withResolverStyle(ResolverStyle.STRICT)
+          .parseUnresolved(datetimeStringExpr.stringValue(), new ParsePosition(0));
+      if (taWithMissingFields == null) {
+        throw new DateTimeException("Input string could not be parsed properly.");
+      }
+      if (!canGetDate(taWithMissingFields) && !canGetTime(taWithMissingFields)) {
+        throw new DateTimeException("Not enough data to build a valid Date, Time, or Datetime.");
+      }
+    } catch (DateTimeException e) {
+      return ExprNullValue.of();
+    }
+
+    int year = taWithMissingFields.isSupported(ChronoField.YEAR)
+        ? taWithMissingFields.get(ChronoField.YEAR) : 2000;
+
+    int month = taWithMissingFields.isSupported(ChronoField.MONTH_OF_YEAR)
+        ? taWithMissingFields.get(ChronoField.MONTH_OF_YEAR) : 1;
+
+    int day = taWithMissingFields.isSupported(ChronoField.DAY_OF_MONTH)
+        ? taWithMissingFields.get(ChronoField.DAY_OF_MONTH) : 1;
+
+    int hour = taWithMissingFields.isSupported(ChronoField.HOUR_OF_DAY)
+        ? taWithMissingFields.get(ChronoField.HOUR_OF_DAY) : 0;
+
+    int minute = taWithMissingFields.isSupported(ChronoField.MINUTE_OF_HOUR)
+        ? taWithMissingFields.get(ChronoField.MINUTE_OF_HOUR) : 0;
+
+    int second = taWithMissingFields.isSupported(ChronoField.SECOND_OF_MINUTE)
+        ? taWithMissingFields.get(ChronoField.SECOND_OF_MINUTE) : 0;
+
+    //Fill returned datetime with current date if only Time information was parsed
+    LocalDateTime output;
+    if (!canGetDate(taWithMissingFields)) {
+      output = LocalDateTime.of(
+          LocalDate.now(fp.getQueryStartClock()),
+          LocalTime.of(hour, minute, second)
+      );
+    } else {
+      output = LocalDateTime.of(year, month, day, hour, minute, second);
+    }
+
+    return new ExprDatetimeValue(output);
+  }
+
   /**
    * Returns English suffix of incoming value.
    * @param val Incoming value.

diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java
@@ -212,6 +212,7 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(second(BuiltinFunctionName.SECOND_OF_MINUTE));
     repository.register(subdate());
     repository.register(subtime());
+    repository.register(str_to_date());
     repository.register(sysdate());
     repository.register(time());
     repository.register(time_format());
@@ -810,6 +811,18 @@ private DefaultFunctionResolver subtime() {
     );
   }
 
+  /**
+   * Extracts a date, time, or datetime from the given string.
+   * It accomplishes this using another string which specifies the input format.
+   */
+  private DefaultFunctionResolver str_to_date() {
+    return define(BuiltinFunctionName.STR_TO_DATE.getName(),
+        implWithProperties(
+            nullMissingHandlingWithProperties((functionProperties, arg, format)
+                -> DateTimeFunction.exprStrToDate(functionProperties, arg, format)),
+            DATETIME, STRING, STRING));
+  }
+
   /**
    * Extracts the time part of a date and time value.
    * Also to construct a time type. The supported signatures:
@@ -1718,6 +1731,12 @@ private ExprValue exprSubTime(FunctionProperties functionProperties,
     return exprApplyTime(functionProperties, temporal, temporalDelta, false);
   }
 
+  private ExprValue exprStrToDate(FunctionProperties fp,
+                                  ExprValue dateTimeExpr,
+                                  ExprValue formatStringExp) {
+    return DateTimeFormatterUtil.parseStringWithDateOrTime(fp, dateTimeExpr, formatStringExp);
+  }
+
   /**
    * Time implementation for ExprValue.
    *

diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
@@ -100,6 +100,7 @@ public enum BuiltinFunctionName {
   SEC_TO_TIME(FunctionName.of("sec_to_time")),
   SECOND(FunctionName.of("second")),
   SECOND_OF_MINUTE(FunctionName.of("second_of_minute")),
+  STR_TO_DATE(FunctionName.of("str_to_date")),
   SUBDATE(FunctionName.of("subdate")),
   SUBTIME(FunctionName.of("subtime")),
   TIME(FunctionName.of("time")),