add backwards compat mode for frontCoded stringEncodingStrategy

benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java

@@ -139,15 +139,15 @@ public void createIndex() throws IOException
         new OnHeapMemorySegmentWriteOutMedium(),
         ByteOrder.nativeOrder(),
         "front-coded-4".equals(indexType) ? 4 : 16,
-        false
+        FrontCodedIndexed.V0
     );
     frontCodedIndexedWriter.open();
 
     FrontCodedIndexedWriter frontCodedIndexedWriterIncrementalBuckets = new FrontCodedIndexedWriter(
         new OnHeapMemorySegmentWriteOutMedium(),
         ByteOrder.nativeOrder(),
         "front-coded-incremental-buckets-4".equals(indexType) ? 4 : 16,
-        true
+        FrontCodedIndexed.V1
     );
     frontCodedIndexedWriterIncrementalBuckets.open();
 

docs/ingestion/ingestion-spec.md

@@ -477,7 +477,7 @@ The `indexSpec` object can include the following properties:
 |-----|-----------|-------|
 |bitmap|Compression format for bitmap indexes. Should be a JSON object with `type` set to `roaring` or `concise`.|`{"type": "roaring"}`|
 |dimensionCompression|Compression format for dimension columns. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
-|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt; columns. <br /><br />Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br /><br />See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`|
+|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX&lt;json&gt; columns. <br /><br />Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br /> `formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`. Defaults to `1`<br /><br />See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`|
 |metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`|
 |longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`|
 |jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
@@ -488,7 +488,7 @@ Front coding is an experimental feature starting in version 25.0. Front coding i
 
 You can enable front coding with all types of ingestion. For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters).
 
-> Front coding is new to Druid 25.0 so the current recommendation is to enable it in a staging environment and fully test your use case before using in production. Segments created with front coding enabled are not compatible with Druid versions older than 25.0.
+> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are not backwards compatible with Druid 25.0, and those created with Druid 25.0 are not compatible with Druid versions older than 25.0. If using front coding in Druid 25.0 and upgrading to Druid 26.0, the `formatVersion` can be specified as `0` keep writing out the older format to enable seamless downgrades to Druid 25.0, and then later changed to `1` (or removed) once determined that rollback is not necessary.
 
 Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each
 [ingestion method](./index.md#ingestion-methods) for details.

processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java

@@ -58,7 +58,7 @@ public static DictionaryWriter<String> getStringDictionaryWriter(
             writeoutMedium,
             IndexIO.BYTE_ORDER,
             strategy.getBucketSize(),
-            true
+            strategy.getFormatVersion()
         );
       } else {
         throw new ISE("Unknown encoding strategy: %s", encodingStrategy.getType());

processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java

@@ -24,6 +24,7 @@
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.segment.data.FrontCodedIndexed;
 
 import javax.annotation.Nullable;
 import java.util.Objects;
@@ -89,15 +90,25 @@ class FrontCoded implements StringEncodingStrategy
     @JsonProperty
     private final int bucketSize;
 
+    @JsonProperty
+    private final byte formatVersion;
+
     @JsonCreator
     public FrontCoded(
-        @JsonProperty("bucketSize") @Nullable Integer bucketSize
+        @JsonProperty("bucketSize") @Nullable Integer bucketSize,
+        @JsonProperty("formatVersion") @Nullable Byte version
     )
     {
       this.bucketSize = bucketSize == null ? DEFAULT_BUCKET_SIZE : bucketSize;
       if (Integer.bitCount(this.bucketSize) != 1) {
         throw new ISE("bucketSize must be a power of two but was[%,d]", bucketSize);
       }
+      this.formatVersion = version == null ? FrontCodedIndexed.V1 : FrontCodedIndexed.validateVersion(version);
+    }
+
+    public FrontCoded(@Nullable Integer bucketSize)
+    {
+      this(bucketSize, null);
     }
 
     @JsonProperty
@@ -106,6 +117,12 @@ public int getBucketSize()
       return bucketSize;
     }
 
+    @JsonProperty
+    public byte getFormatVersion()
+    {
+      return formatVersion;
+    }
+
     @Override
     public String getType()
     {
@@ -128,20 +145,21 @@ public boolean equals(Object o)
         return false;
       }
       FrontCoded that = (FrontCoded) o;
-      return bucketSize == that.bucketSize;
+      return bucketSize == that.bucketSize && formatVersion == that.formatVersion;
     }
 
     @Override
     public int hashCode()
     {
-      return Objects.hash(bucketSize);
+      return Objects.hash(bucketSize, formatVersion);
     }
 
     @Override
     public String toString()
     {
       return "FrontCoded{" +
              "bucketSize=" + bucketSize +
+             "formatVersion=" + formatVersion +
              '}';
     }
   }

processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java

@@ -23,6 +23,7 @@
 import com.google.common.base.Supplier;
 import org.apache.druid.annotations.SuppressFBWarnings;
 import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
@@ -76,11 +77,26 @@
  */
 public final class FrontCodedIndexed implements Indexed<ByteBuffer>
 {
+  public static byte V0 = 0;
+  public static byte V1 = 1;
+
+  public static byte validateVersion(byte version)
+  {
+    if (version != FrontCodedIndexed.V0 && version != FrontCodedIndexed.V1) {
+      throw new IAE(
+          "Unknown format version for FrontCodedIndexed [%s], must be [%s] or [%s]",
+          version,
+          FrontCodedIndexed.V0,
+          FrontCodedIndexed.V1
+      );
+    }
+    return version;
+  }
   public static Supplier<FrontCodedIndexed> read(ByteBuffer buffer, ByteOrder ordering)
   {
     final ByteBuffer orderedBuffer = buffer.asReadOnlyBuffer().order(ordering);
     final byte version = orderedBuffer.get();
-    Preconditions.checkArgument(version == 0 || version == 1, "only V0 and V1 exist, encountered " + version);
+    Preconditions.checkArgument(version == V0 || version == V1, "only V0 and V1 exist, encountered " + version);
     final int bucketSize = Byte.toUnsignedInt(orderedBuffer.get());
     final boolean hasNull = NullHandling.IS_NULL_BYTE == orderedBuffer.get();
     final int numValues = VByte.readInt(orderedBuffer);

processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java

@@ -59,7 +59,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
   private final byte[][] bucketBuffer;
   private final ByteBuffer getOffsetBuffer;
   private final int div;
-  private final boolean useIncrementalBuckets;
+  private final byte version;
 
   @Nullable
   private byte[] prevObject = null;
@@ -78,7 +78,7 @@ public FrontCodedIndexedWriter(
       SegmentWriteOutMedium segmentWriteOutMedium,
       ByteOrder byteOrder,
       int bucketSize,
-      boolean useIncrementalBuckets
+      byte version
   )
   {
     if (Integer.bitCount(bucketSize) != 1 || bucketSize < 1 || bucketSize > 128) {
@@ -91,7 +91,7 @@ public FrontCodedIndexedWriter(
     this.bucketBuffer = new byte[bucketSize][];
     this.getOffsetBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder);
     this.div = Integer.numberOfTrailingZeros(bucketSize);
-    this.useIncrementalBuckets = useIncrementalBuckets;
+    this.version = FrontCodedIndexed.validateVersion(version);
   }
 
   @Override
@@ -124,9 +124,9 @@ public void write(@Nullable byte[] value) throws IOException
       int written;
       // write the bucket, growing scratch buffer as necessary
       do {
-        written = useIncrementalBuckets
-                  ? writeIncrementalBucket(scratch, bucketBuffer, bucketSize)
-                  : writeBucket(scratch, bucketBuffer, bucketSize);
+        written = version == FrontCodedIndexed.V1
+                  ? writeBucketV1(scratch, bucketBuffer, bucketSize)
+                  : writeBucketV0(scratch, bucketBuffer, bucketSize);
         if (written < 0) {
           growScratch();
         }
@@ -170,14 +170,7 @@ public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws I
       flush();
     }
     resetScratch();
-
-    if (useIncrementalBuckets) {
-      // version 1 is incremental buckets
-      scratch.put((byte) 1);
-    } else {
-      // version 0 all values are prefixed on first bucket value
-      scratch.put((byte) 0);
-    }
+    scratch.put(version);
     scratch.put((byte) bucketSize);
     scratch.put(hasNulls ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE);
     VByte.writeInt(scratch, numWritten);
@@ -222,7 +215,7 @@ public byte[] get(int index) throws IOException
       final ByteBuffer bucketBuffer = ByteBuffer.allocate(bucketBytesSize).order(byteOrder);
       valuesOut.readFully(startOffset, bucketBuffer);
       bucketBuffer.clear();
-      final ByteBuffer valueBuffer = useIncrementalBuckets
+      final ByteBuffer valueBuffer = version == FrontCodedIndexed.V1
                                      ? getFromBucketV1(bucketBuffer, relativeIndex, bucketSize)
                                      : FrontCodedIndexed.getFromBucketV0(bucketBuffer, relativeIndex);
       final byte[] valueBytes = new byte[valueBuffer.limit() - valueBuffer.position()];
@@ -248,9 +241,9 @@ private void flush() throws IOException
     int written;
     do {
       int flushSize = remainder == 0 ? bucketSize : remainder;
-      written = useIncrementalBuckets
-                ? writeIncrementalBucket(scratch, bucketBuffer, flushSize)
-                : writeBucket(scratch, bucketBuffer, flushSize);
+      written = version == FrontCodedIndexed.V1
+                ? writeBucketV1(scratch, bucketBuffer, flushSize)
+                : writeBucketV0(scratch, bucketBuffer, flushSize);
       if (written < 0) {
         growScratch();
       }
@@ -283,7 +276,7 @@ private void growScratch()
    *
    * Uses {@link VByte} encoded integers to indicate prefix length and value length.
    */
-  public static int writeBucket(ByteBuffer buffer, byte[][] values, int numValues)
+  public static int writeBucketV0(ByteBuffer buffer, byte[][] values, int numValues)
   {
     int written = 0;
     byte[] first = null;
@@ -333,7 +326,7 @@ public static int writeBucket(ByteBuffer buffer, byte[][] values, int numValues)
    *
    * Uses {@link VByte} encoded integers to indicate prefix length and value length.
    */
-  public static int writeIncrementalBucket(ByteBuffer buffer, byte[][] values, int numValues)
+  public static int writeBucketV1(ByteBuffer buffer, byte[][] values, int numValues)
   {
     int written = 0;
     byte[] prev = null;

processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.column;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.data.FrontCodedIndexed;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StringEncodingStrategyTest
+{
+  private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
+
+
+  @Test
+  public void testUtf8Serde() throws JsonProcessingException
+  {
+    StringEncodingStrategy utf8 = new StringEncodingStrategy.Utf8();
+    String there = JSON_MAPPER.writeValueAsString(utf8);
+    StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+    Assert.assertEquals(utf8, andBackAgain);
+  }
+
+  @Test
+  public void testFrontCodedDefaultSerde() throws JsonProcessingException
+  {
+    StringEncodingStrategy frontCoded = new StringEncodingStrategy.FrontCoded(null, null);
+    String there = JSON_MAPPER.writeValueAsString(frontCoded);
+    StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+    Assert.assertEquals(frontCoded, andBackAgain);
+  }
+
+  @Test
+  public void testFrontCodedFormatSerde() throws JsonProcessingException
+  {
+    StringEncodingStrategy frontCoded = new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0);
+    String there = JSON_MAPPER.writeValueAsString(frontCoded);
+    StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+    Assert.assertEquals(frontCoded, andBackAgain);
+  }
+
+  @Test
+  public void testEqualsAndHashcode()
+  {
+    EqualsVerifier.forClass(StringEncodingStrategy.Utf8.class).usingGetClass().verify();
+    EqualsVerifier.forClass(StringEncodingStrategy.FrontCoded.class).usingGetClass().verify();
+  }
+}