Add long/float ColumnSelectorStrategy implementations

docs/content/querying/dimensionspecs.md

@@ -23,9 +23,9 @@ Returns dimension values as is and optionally renames the dimension.
 }
 ```
 
-When specifying a DimensionSpec on a numeric column, the user should include the type of the column in the `outputType` field. This is necessary as it is possible for a column with given name to have different value types in different segments; results will be converted to the type specified by `outputType` before merging.
+When specifying a DimensionSpec on a numeric column, the user should include the type of the column in the `outputType` field. If left unspecified, the `outputType` defaults to STRING.
 
-If left unspecified, the `outputType` defaults to STRING.
+Please refer to the [Output Types](#output-types) section for more details.
 
 ### Extraction DimensionSpec
 
@@ -36,10 +36,15 @@ Returns dimension values transformed using the given [extraction function](#extr
   "type" : "extraction",
   "dimension" : <dimension>,
   "outputName" :  <output_name>,
+  "outputType": <"STRING"|"LONG"|"FLOAT">,
   "extractionFn" : <extraction_function>
 }
 ```
 
+`outputType` may also be specified in an ExtractionDimensionSpec to apply type conversion to results before merging. If left unspecified, the `outputType` defaults to STRING.
+
+Please refer to the [Output Types](#output-types) section for more details.
+
 ### Filtered DimensionSpecs
 
 These are only useful for multi-value dimensions. If you have a row in druid that has a multi-value dimension with values ["v1", "v2", "v3"] and you send a groupBy/topN query grouping by that dimension with [query filter](filters.html) for value "v1". In the response you will get 3 rows containing "v1", "v2" and "v3". This behavior might be unintuitive for some use cases.
@@ -61,6 +66,8 @@ Following filtered dimension spec retains only the values matching regex. Note t
 { "type" : "regexFiltered", "delegate" : <dimensionSpec>, "pattern": <java regex pattern> }
 ```
 
+Note that filtered DimensionSpecs are not supported when applied to a numeric column (when the base column in a segment contains long or float values, not related to what `outputType` specifies).
+
 For more details and examples, see [multi-value dimensions](multi-value-dimensions.html).
 
 ### Lookup DimensionSpecs
@@ -105,6 +112,21 @@ The second kind where it is not possible to pass at query time due to their size
 }
 ```
 
+## Output Types
+
+The dimension specs provide an option to specify the output type of a column's values. This is necessary as it is possible for a column with given name to have different value types in different segments; results will be converted to the type specified by `outputType` before merging.
+
+Note that not all use cases for DimensionSpec currently support `outputType`, the table below shows which use cases support this option:
+
+|Query Type|Supported?|
+|--------|---------|
+|GroupBy (v1)|no|
+|GroupBy (v2)|yes|
+|TopN|yes|
+|Search|no|
+|Select|no|
+|Cardinality Aggregator|no|
+
 ## Extraction Functions
 
 Extraction functions define the transformation applied to each dimension value.

docs/content/querying/sql.md

@@ -160,8 +160,6 @@ Druid does not support all SQL features. Most of these are due to missing featur
 language. Some unsupported SQL features include:
 
 - Grouping on functions of multiple columns, like concatenation: `SELECT COUNT(*) FROM data_source GROUP BY dim1 || ' ' || dim2`
-- Grouping on long and float columns.
-- Filtering on float columns.
 - Filtering on non-boolean interactions between columns, like two columns equaling each other: `SELECT COUNT(*) FROM data_source WHERE dim1 = dim2`.
 - A number of miscellaneous functions, like `TRIM`.
 - Joins, other than semi-joins as described above.

processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java

@@ -49,6 +49,7 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.StorageAdapter;
 import io.druid.segment.VirtualColumns;
+import io.druid.segment.column.ValueType;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.Filters;
 import org.joda.time.DateTime;
@@ -326,13 +327,18 @@ public RowIterator(GroupByQuery query, final Cursor cursor, ByteBuffer metricsBu
 
       for (int i = 0; i < dimensionSpecs.size(); ++i) {
         final DimensionSpec dimSpec = dimensionSpecs.get(i);
+        if ( dimSpec.getOutputType() != ValueType.STRING) {
+          throw new UnsupportedOperationException(
+              "GroupBy v1 only supports dimensions with an outputType of STRING."
+          );
+        }
+
         final DimensionSelector selector = cursor.makeDimensionSelector(dimSpec);
         if (selector != null) {
           if (selector.getValueCardinality() == DimensionSelector.CARDINALITY_UNKNOWN) {
             throw new UnsupportedOperationException(
                 "GroupBy v1 does not support dimension selectors with unknown cardinality.");
           }
-
           dimensions.add(selector);
           dimNames.add(dimSpec.getOutputName());
         }

processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java

@@ -49,10 +49,10 @@ public class DictionaryBuildingStringGroupByColumnSelectorStrategy extends Strin
   public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map<String, Object> resultMap)
   {
     final int id = key.getInt(selectorPlus.getKeyBufferPosition());
-    final String value = dictionary.get(id);
 
     // GROUP_BY_MISSING_VALUE is used to indicate empty rows, which are omitted from the result map.
     if (id != GROUP_BY_MISSING_VALUE) {
+      final String value = dictionary.get(id);
       resultMap.put(
           selectorPlus.getOutputName(),
           value

processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java

@@ -117,8 +117,11 @@ public Function<Sequence<Row>, Sequence<Row>> build(
         final StringComparator naturalComparator;
         if (columnType == ValueType.STRING) {
           naturalComparator = StringComparators.LEXICOGRAPHIC;
-        } else {
+        } else if (columnType == ValueType.LONG || columnType == ValueType.FLOAT) {
           naturalComparator = StringComparators.NUMERIC;
+        } else {
+          sortingNeeded = true;
+          break;
         }
 
         if (columnSpec.getDirection() != OrderByColumnSpec.Direction.ASCENDING

processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java

@@ -133,9 +133,8 @@ public static <ColumnSelectorStrategyClass extends ColumnSelectorStrategy> Colum
       );
       ColumnSelectorStrategyClass strategy = makeStrategy(
           strategyFactory,
-          dimName,
+          dimSpec,
           cursor.getColumnCapabilities(dimSpec.getDimension()),
-          dimSpec.getExtractionFn() != null,
           selector
       );
       final ColumnSelectorPlus<ColumnSelectorStrategyClass> selectorPlus = new ColumnSelectorPlus<>(
@@ -156,19 +155,13 @@ public static ColumnValueSelector getColumnValueSelectorFromDimensionSpec(
   {
     String dimName = dimSpec.getDimension();
     ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(dimName);
-    capabilities = getEffectiveCapabilities(dimName, capabilities, dimSpec.getExtractionFn() != null);
+    capabilities = getEffectiveCapabilities(dimSpec, capabilities);
     switch (capabilities.getType()) {
       case STRING:
         return columnSelectorFactory.makeDimensionSelector(dimSpec);
       case LONG:
-        if (dimSpec instanceof BaseFilteredDimensionSpec) {
-          throw new UnsupportedOperationException("Filtered dimension specs are not supported on numeric columns.");
-        }
         return columnSelectorFactory.makeLongColumnSelector(dimSpec.getDimension());
       case FLOAT:
-        if (dimSpec instanceof BaseFilteredDimensionSpec) {
-          throw new UnsupportedOperationException("Filtered dimension specs are not supported on numeric columns.");
-        }
         return columnSelectorFactory.makeFloatColumnSelector(dimSpec.getDimension());
       default:
         return null;
@@ -179,9 +172,8 @@ public static ColumnValueSelector getColumnValueSelectorFromDimensionSpec(
   // adjusts the capabilities for columns that cannot be handled as-is to manageable defaults
   // (e.g., treating missing columns as empty String columns)
   private static ColumnCapabilities getEffectiveCapabilities(
-      String dimName,
-      ColumnCapabilities capabilities,
-      boolean hasExFn
+      DimensionSpec dimSpec,
+      ColumnCapabilities capabilities
   )
   {
     if (capabilities == null) {
@@ -195,22 +187,29 @@ private static ColumnCapabilities getEffectiveCapabilities(
 
     // Currently, all extractionFns output Strings, so the column will return String values via a
     // DimensionSelector if an extractionFn is present.
-    if (hasExFn) {
+    if (dimSpec.getExtractionFn() != null) {
       capabilities = DEFAULT_STRING_CAPABILITIES;
     }
 
+    // Filtered dimension specs are not supported on numerics, the numeric column
+    // will be treated as a null String column in that case
+    if (capabilities.getType() == ValueType.LONG || capabilities.getType() == ValueType.FLOAT) {
+      if (dimSpec instanceof BaseFilteredDimensionSpec) {
+        capabilities = DEFAULT_STRING_CAPABILITIES;
+      }
+    }
+
     return capabilities;
   }
 
   private static <ColumnSelectorStrategyClass extends ColumnSelectorStrategy> ColumnSelectorStrategyClass makeStrategy(
       ColumnSelectorStrategyFactory<ColumnSelectorStrategyClass> strategyFactory,
-      String dimName,
+      DimensionSpec dimSpec,
       ColumnCapabilities capabilities,
-      boolean hasExFn,
       ColumnValueSelector selector
   )
   {
-    capabilities = getEffectiveCapabilities(dimName, capabilities, hasExFn);
+    capabilities = getEffectiveCapabilities(dimSpec, capabilities);
     return strategyFactory.makeColumnSelectorStrategy(capabilities, selector);
   }
 

processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java

@@ -32,6 +32,7 @@
 import io.druid.java.util.common.guava.Sequence;
 import io.druid.java.util.common.guava.Sequences;
 import io.druid.query.QueryInterruptedException;
+import io.druid.query.dimension.BaseFilteredDimensionSpec;
 import io.druid.query.dimension.DimensionSpec;
 import io.druid.query.extraction.ExtractionFn;
 import io.druid.query.filter.BooleanFilter;
@@ -455,10 +456,17 @@ private DimensionSelector makeDimensionSelectorUndecorated(
                       }
 
                       if (columnDesc.getCapabilities().getType() == ValueType.LONG) {
+                        // Filtered dimension specs are not supported on numerics
+                        if (dimensionSpec instanceof BaseFilteredDimensionSpec) {
+                          return NullDimensionSelector.instance();
+                        }
                         return new LongWrappingDimensionSelector(makeLongColumnSelector(dimension), extractionFn);
                       }
 
                       if (columnDesc.getCapabilities().getType() == ValueType.FLOAT) {
+                        if (dimensionSpec instanceof BaseFilteredDimensionSpec) {
+                          return NullDimensionSelector.instance();
+                        }
                         return new FloatWrappingDimensionSelector(makeFloatColumnSelector(dimension), extractionFn);
                       }
 

processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java

@@ -27,6 +27,7 @@
 import io.druid.java.util.common.guava.Sequence;
 import io.druid.java.util.common.guava.Sequences;
 import io.druid.query.QueryInterruptedException;
+import io.druid.query.dimension.BaseFilteredDimensionSpec;
 import io.druid.query.dimension.DefaultDimensionSpec;
 import io.druid.query.dimension.DimensionSpec;
 import io.druid.query.extraction.ExtractionFn;
@@ -359,9 +360,16 @@ public DimensionSelector makeDimensionSelector(
                 ColumnCapabilities capabilities = getColumnCapabilities(dimension);
                 if (capabilities != null) {
                   if (capabilities.getType() == ValueType.LONG) {
+                    if (dimensionSpec instanceof BaseFilteredDimensionSpec) {
+                      // Filtered dimension specs are not supported on numerics
+                      return dimensionSpec.decorate(NullDimensionSelector.instance());
+                    }
                     return new LongWrappingDimensionSelector(makeLongColumnSelector(dimension), extractionFn);
                   }
                   if (capabilities.getType() == ValueType.FLOAT) {
+                    if (dimensionSpec instanceof BaseFilteredDimensionSpec) {
+                      return dimensionSpec.decorate(NullDimensionSelector.instance());
+                    }
                     return new FloatWrappingDimensionSelector(makeFloatColumnSelector(dimension), extractionFn);
                   }
                 }