Add "asMillis" option to "timeFormat" extractionFn.

docs/content/querying/dimensionspecs.md

@@ -180,13 +180,15 @@ For a regular dimension, it assumes the string is formatted in
 * `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](http://www.oracle.com/technetwork/java/javase/java8locales-2095355.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc.
 * `timeZone` : time zone to use in [IANA tz database format](http://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. `Europe/Berlin` (this can possibly be different than the aggregation time-zone)
 * `granularity` : [granularity](granularities.html) to apply before formatting, or omit to not apply any granularity.
+* `asMillis` : boolean value, set to true to treat input strings as millis rather than ISO8601 strings. Additionally, if `format` is null or not specified, output will be in millis rather than ISO8601.
 
 ```json
 { "type" : "timeFormat",
   "format" : <output_format> (optional),
   "timeZone" : <time_zone> (optional),
   "locale" : <locale> (optional),
-  "granularity" : <granularity> (optional) }
+  "granularity" : <granularity> (optional) },
+  "asMillis" : <true or false> (optional) }
 ```
 
 For example, the following dimension spec returns the day of the week for Montréal in French:

processing/src/main/java/io/druid/query/extraction/TimeFormatExtractionFn.java

@@ -20,6 +20,8 @@
 package io.druid.query.extraction;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import io.druid.common.guava.GuavaUtils;
 import io.druid.granularity.QueryGranularities;
 import io.druid.granularity.QueryGranularity;
 import io.druid.java.util.common.StringUtils;
@@ -38,22 +40,33 @@ public class TimeFormatExtractionFn implements ExtractionFn
   private final DateTimeZone tz;
   private final Locale locale;
   private final QueryGranularity granularity;
+  private final boolean asMillis;
   private final DateTimeFormatter formatter;
 
   public TimeFormatExtractionFn(
       @JsonProperty("format") String format,
       @JsonProperty("timeZone") DateTimeZone tz,
       @JsonProperty("locale") String localeString,
-      @JsonProperty("granularity") QueryGranularity granularity
+      @JsonProperty("granularity") QueryGranularity granularity,
+      @JsonProperty("asMillis") boolean asMillis
   )
   {
     this.format = format;
     this.tz = tz;
     this.locale = localeString == null ? null : Locale.forLanguageTag(localeString);
     this.granularity = granularity == null ? QueryGranularities.NONE : granularity;
-    this.formatter = (format == null ? ISODateTimeFormat.dateTime() : DateTimeFormat.forPattern(format))
-        .withZone(tz == null ? DateTimeZone.UTC : tz)
-        .withLocale(locale);
+
+    if (asMillis && format == null) {
+      Preconditions.checkArgument(tz == null, "timeZone requires a format");
+      Preconditions.checkArgument(localeString == null, "locale requires a format");
+      this.formatter = null;
+    } else {
+      this.formatter = (format == null ? ISODateTimeFormat.dateTime() : DateTimeFormat.forPattern(format))
+          .withZone(tz == null ? DateTimeZone.UTC : tz)
+          .withLocale(locale);
+    }
+
+    this.asMillis = asMillis;
   }
 
   @JsonProperty
@@ -84,29 +97,43 @@ public QueryGranularity getGranularity()
     return granularity;
   }
 
+  @JsonProperty
+  public boolean isAsMillis()
+  {
+    return asMillis;
+  }
+
   @Override
   public byte[] getCacheKey()
   {
     final byte[] exprBytes = StringUtils.toUtf8(format + "\u0001" + tz.getID() + "\u0001" + locale.toLanguageTag());
     final byte[] granularityCacheKey = granularity.cacheKey();
-    return ByteBuffer.allocate(2 + exprBytes.length + granularityCacheKey.length)
+    return ByteBuffer.allocate(4 + exprBytes.length + granularityCacheKey.length)
                      .put(ExtractionCacheHelper.CACHE_TYPE_ID_TIME_FORMAT)
                      .put(exprBytes)
                      .put((byte) 0xFF)
                      .put(granularityCacheKey)
+                     .put((byte) 0xFF)
+                     .put(asMillis ? (byte) 1 : (byte) 0)
                      .array();
   }
 
   @Override
   public String apply(long value)
   {
-    return formatter.print(granularity.truncate(value));
+    final long truncated = granularity.truncate(value);
+    return formatter == null ? String.valueOf(truncated) : formatter.print(truncated);
   }
 
   @Override
   public String apply(Object value)
   {
-    return apply(new DateTime(value).getMillis());
+    if (asMillis && value instanceof String) {
+      final Long theLong = GuavaUtils.tryParseLong((String) value);
+      return theLong == null ? apply(new DateTime(value).getMillis()) : apply(theLong.longValue());
+    } else {
+      return apply(new DateTime(value).getMillis());
+    }
   }
 
   @Override
@@ -139,32 +166,36 @@ public boolean equals(Object o)
 
     TimeFormatExtractionFn that = (TimeFormatExtractionFn) o;
 
-    if (tz != null ? !tz.equals(that.tz) : that.tz != null) {
+    if (asMillis != that.asMillis) {
       return false;
     }
     if (format != null ? !format.equals(that.format) : that.format != null) {
       return false;
     }
+    if (tz != null ? !tz.equals(that.tz) : that.tz != null) {
+      return false;
+    }
     if (locale != null ? !locale.equals(that.locale) : that.locale != null) {
       return false;
     }
-    return granularity.equals(that.granularity);
+    return granularity != null ? granularity.equals(that.granularity) : that.granularity == null;
 
   }
 
   @Override
   public int hashCode()
   {
-    int result = tz != null ? tz.hashCode() : 0;
-    result = 31 * result + (format != null ? format.hashCode() : 0);
+    int result = format != null ? format.hashCode() : 0;
+    result = 31 * result + (tz != null ? tz.hashCode() : 0);
     result = 31 * result + (locale != null ? locale.hashCode() : 0);
-    result = 31 * result + granularity.hashCode();
+    result = 31 * result + (granularity != null ? granularity.hashCode() : 0);
+    result = 31 * result + (asMillis ? 1 : 0);
     return result;
   }
 
   @Override
   public String toString()
   {
-    return String.format("timeFormat(\"%s\", %s, %s, %s)", format, tz, locale, granularity);
+    return String.format("timeFormat(\"%s\", %s, %s, %s, %s)", format, tz, locale, granularity, asMillis);
   }
 }

processing/src/test/java/io/druid/query/extraction/TimeFormatExtractionFnTest.java

@@ -28,6 +28,8 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.Arrays;
+
 public class TimeFormatExtractionFnTest
 {
 
@@ -43,7 +45,7 @@ public class TimeFormatExtractionFnTest
   @Test
   public void testDayOfWeekExtraction() throws Exception
   {
-    TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, null, null);
+    TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, null, null, false);
     Assert.assertEquals("Thursday",  fn.apply(timestamps[0]));
     Assert.assertEquals("Friday",    fn.apply(timestamps[1]));
     Assert.assertEquals("Tuesday",   fn.apply(timestamps[2]));
@@ -57,7 +59,7 @@ public void testDayOfWeekExtraction() throws Exception
   @Test
   public void testLocalizedExtraction() throws Exception
   {
-    TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, "is", null);
+    TimeFormatExtractionFn fn = new TimeFormatExtractionFn("EEEE", null, "is", null, false);
     Assert.assertEquals("fimmtudagur",  fn.apply(timestamps[0]));
     Assert.assertEquals("föstudagur",   fn.apply(timestamps[1]));
     Assert.assertEquals("þriðjudagur",  fn.apply(timestamps[2]));
@@ -71,7 +73,7 @@ public void testLocalizedExtraction() throws Exception
   @Test
   public void testGranularExtractionWithNullPattern() throws Exception
   {
-    TimeFormatExtractionFn fn = new TimeFormatExtractionFn(null, null, null, QueryGranularities.DAY);
+    TimeFormatExtractionFn fn = new TimeFormatExtractionFn(null, null, null, QueryGranularities.DAY, false);
     Assert.assertEquals("2015-01-01T00:00:00.000Z", fn.apply(timestamps[0]));
     Assert.assertEquals("2015-01-02T00:00:00.000Z", fn.apply(timestamps[1]));
     Assert.assertEquals("2015-03-03T00:00:00.000Z", fn.apply(timestamps[2]));
@@ -89,7 +91,8 @@ public void testTimeZoneExtraction() throws Exception
         "'In Berlin ist es schon 'EEEE",
         DateTimeZone.forID("Europe/Berlin"),
         "de",
-        null
+        null,
+        false
     );
     Assert.assertEquals("In Berlin ist es schon Freitag",    fn.apply(timestamps[0]));
     Assert.assertEquals("In Berlin ist es schon Samstag",    fn.apply(timestamps[1]));
@@ -140,4 +143,35 @@ public void testSerdeFromJson() throws Exception
         )
     );
   }
+
+  @Test
+  public void testCacheKey()
+  {
+    TimeFormatExtractionFn fn = new TimeFormatExtractionFn(
+        "'In Berlin ist es schon 'EEEE",
+        DateTimeZone.forID("Europe/Berlin"),
+        "de",
+        null,
+        false
+    );
+
+    TimeFormatExtractionFn fn2 = new TimeFormatExtractionFn(
+        "'In Berlin ist es schon 'EEEE",
+        DateTimeZone.forID("Europe/Berlin"),
+        "de",
+        null,
+        true
+    );
+
+    TimeFormatExtractionFn fn3 = new TimeFormatExtractionFn(
+        "'In Berlin ist es schon 'EEEE",
+        DateTimeZone.forID("Europe/Berlin"),
+        "de",
+        null,
+        true
+    );
+
+    Assert.assertFalse(Arrays.equals(fn.getCacheKey(), fn2.getCacheKey()));
+    Assert.assertTrue(Arrays.equals(fn2.getCacheKey(), fn3.getCacheKey()));
+  }
 }

processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java

@@ -4120,7 +4120,7 @@ public void testGroupByTimeExtractionNamedUnderUnderTime()
                 new ExtractionDimensionSpec(
                     Column.TIME_COLUMN_NAME,
                     Column.TIME_COLUMN_NAME,
-                    new TimeFormatExtractionFn("EEEE", null, null, null),
+                    new TimeFormatExtractionFn("EEEE", null, null, null, false),
                     null
                 )
             )
@@ -4994,8 +4994,8 @@ public void testSubqueryWithOuterTimeFilter()
         .setGranularity(QueryRunnerTestHelper.dayGran)
         .build();
 
-    final DimFilter fridayFilter = new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Friday", new TimeFormatExtractionFn("EEEE", null, null, null));
-    final DimFilter firstDaysFilter = new InDimFilter(Column.TIME_COLUMN_NAME, ImmutableList.of("1", "2", "3"),  new TimeFormatExtractionFn("d", null, null, null));
+    final DimFilter fridayFilter = new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Friday", new TimeFormatExtractionFn("EEEE", null, null, null, false));
+    final DimFilter firstDaysFilter = new InDimFilter(Column.TIME_COLUMN_NAME, ImmutableList.of("1", "2", "3"),  new TimeFormatExtractionFn("d", null, null, null, false));
     final GroupByQuery query = GroupByQuery
         .builder()
         .setDataSource(subquery)
@@ -5452,7 +5452,7 @@ public void testGroupByTimeExtraction()
                 new ExtractionDimensionSpec(
                     Column.TIME_COLUMN_NAME,
                     "dayOfWeek",
-                    new TimeFormatExtractionFn("EEEE", null, null, null),
+                    new TimeFormatExtractionFn("EEEE", null, null, null, false),
                     null
                 )
             )

processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java

@@ -3063,7 +3063,7 @@ public void testTopNTimeExtraction()
             new ExtractionDimensionSpec(
                 Column.TIME_COLUMN_NAME,
                 "dayOfWeek",
-                new TimeFormatExtractionFn("EEEE", null, null, null),
+                new TimeFormatExtractionFn("EEEE", null, null, null, false),
                 null
             )
         )

processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java

@@ -225,7 +225,7 @@ public void testTimeFilterWithExtractionFn()
   @Test
   public void testTimeFilterWithTimeFormatExtractionFn()
   {
-    ExtractionFn exfn = new TimeFormatExtractionFn("EEEE", DateTimeZone.forID("America/New_York"), "en", null);
+    ExtractionFn exfn = new TimeFormatExtractionFn("EEEE", DateTimeZone.forID("America/New_York"), "en", null, false);
     assertFilterMatches(
         new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Wednesday", exfn),
         ImmutableList.<String>of("0", "1", "2", "3", "4", "5")