diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java index 00be92025323..fe0e717ab18e 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java @@ -139,7 +139,7 @@ public void createIndex() throws IOException new OnHeapMemorySegmentWriteOutMedium(), ByteOrder.nativeOrder(), "front-coded-4".equals(indexType) ? 4 : 16, - false + FrontCodedIndexed.V0 ); frontCodedIndexedWriter.open(); @@ -147,7 +147,7 @@ public void createIndex() throws IOException new OnHeapMemorySegmentWriteOutMedium(), ByteOrder.nativeOrder(), "front-coded-incremental-buckets-4".equals(indexType) ? 4 : 16, - true + FrontCodedIndexed.V1 ); frontCodedIndexedWriterIncrementalBuckets.open(); diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md index d0ae34eec366..22e4d167bb19 100644 --- a/docs/ingestion/ingestion-spec.md +++ b/docs/ingestion/ingestion-spec.md @@ -477,7 +477,7 @@ The `indexSpec` object can include the following properties: |-----|-----------|-------| |bitmap|Compression format for bitmap indexes. Should be a JSON object with `type` set to `roaring` or `concise`.|`{"type": "roaring"}`| |dimensionCompression|Compression format for dimension columns. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`| -|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns.

Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`
`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.

See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`| +|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns.

Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`
`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.
`formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`. Defaults to `1`

See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`| |metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`| |longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`| |jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`| @@ -488,7 +488,7 @@ Front coding is an experimental feature starting in version 25.0. Front coding i You can enable front coding with all types of ingestion. For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters). -> Front coding is new to Druid 25.0 so the current recommendation is to enable it in a staging environment and fully test your use case before using in production. Segments created with front coding enabled are not compatible with Druid versions older than 25.0. +> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are not backwards compatible with Druid 25.0, and those created with Druid 25.0 are not compatible with Druid versions older than 25.0. If using front coding in Druid 25.0 and upgrading to Druid 26.0, the `formatVersion` can be specified as `0` keep writing out the older format to enable seamless downgrades to Druid 25.0, and then later changed to `1` (or removed) once determined that rollback is not necessary. Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each [ingestion method](./index.md#ingestion-methods) for details. diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java index 0c04f0288c88..c4a9b0661a93 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java +++ b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java @@ -58,7 +58,7 @@ public static DictionaryWriter getStringDictionaryWriter( writeoutMedium, IndexIO.BYTE_ORDER, strategy.getBucketSize(), - true + strategy.getFormatVersion() ); } else { throw new ISE("Unknown encoding strategy: %s", encodingStrategy.getType()); diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java index e850b0c362ab..a8246c9d75d1 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java +++ b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.segment.data.FrontCodedIndexed; import javax.annotation.Nullable; import java.util.Objects; @@ -84,20 +85,31 @@ public String toString() class FrontCoded implements StringEncodingStrategy { - public static final int DEFAULT_BUCKET_SIZE = 4; @JsonProperty private final int bucketSize; + @JsonProperty + private final byte formatVersion; + @JsonCreator public FrontCoded( - @JsonProperty("bucketSize") @Nullable Integer bucketSize + @JsonProperty("bucketSize") @Nullable Integer bucketSize, + @JsonProperty("formatVersion") @Nullable Byte version ) { - this.bucketSize = bucketSize == null ? DEFAULT_BUCKET_SIZE : bucketSize; + this.bucketSize = bucketSize == null ? FrontCodedIndexed.DEFAULT_BUCKET_SIZE : bucketSize; if (Integer.bitCount(this.bucketSize) != 1) { throw new ISE("bucketSize must be a power of two but was[%,d]", bucketSize); } + this.formatVersion = version == null + ? FrontCodedIndexed.DEFAULT_VERSION + : FrontCodedIndexed.validateVersion(version); + } + + public FrontCoded(@Nullable Integer bucketSize) + { + this(bucketSize, null); } @JsonProperty @@ -106,6 +118,12 @@ public int getBucketSize() return bucketSize; } + @JsonProperty + public byte getFormatVersion() + { + return formatVersion; + } + @Override public String getType() { @@ -128,13 +146,13 @@ public boolean equals(Object o) return false; } FrontCoded that = (FrontCoded) o; - return bucketSize == that.bucketSize; + return bucketSize == that.bucketSize && formatVersion == that.formatVersion; } @Override public int hashCode() { - return Objects.hash(bucketSize); + return Objects.hash(bucketSize, formatVersion); } @Override @@ -142,6 +160,7 @@ public String toString() { return "FrontCoded{" + "bucketSize=" + bucketSize + + ", formatVersion=" + formatVersion + '}'; } } diff --git a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java index 27a84b5df310..bb135b0b9a24 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java +++ b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java @@ -23,6 +23,7 @@ import com.google.common.base.Supplier; import org.apache.druid.annotations.SuppressFBWarnings; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -76,11 +77,28 @@ */ public final class FrontCodedIndexed implements Indexed { + public static final byte V0 = 0; + public static final byte V1 = 1; + public static final byte DEFAULT_VERSION = V1; + public static final int DEFAULT_BUCKET_SIZE = 4; + + public static byte validateVersion(byte version) + { + if (version != FrontCodedIndexed.V0 && version != FrontCodedIndexed.V1) { + throw new IAE( + "Unknown format version for FrontCodedIndexed [%s], must be [%s] or [%s]", + version, + FrontCodedIndexed.V0, + FrontCodedIndexed.V1 + ); + } + return version; + } public static Supplier read(ByteBuffer buffer, ByteOrder ordering) { final ByteBuffer orderedBuffer = buffer.asReadOnlyBuffer().order(ordering); final byte version = orderedBuffer.get(); - Preconditions.checkArgument(version == 0 || version == 1, "only V0 and V1 exist, encountered " + version); + Preconditions.checkArgument(version == V0 || version == V1, "only V0 and V1 exist, encountered " + version); final int bucketSize = Byte.toUnsignedInt(orderedBuffer.get()); final boolean hasNull = NullHandling.IS_NULL_BYTE == orderedBuffer.get(); final int numValues = VByte.readInt(orderedBuffer); diff --git a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java index c5a26f3a597c..8f711df610d0 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java +++ b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java @@ -59,7 +59,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter private final byte[][] bucketBuffer; private final ByteBuffer getOffsetBuffer; private final int div; - private final boolean useIncrementalBuckets; + private final byte version; @Nullable private byte[] prevObject = null; @@ -78,7 +78,7 @@ public FrontCodedIndexedWriter( SegmentWriteOutMedium segmentWriteOutMedium, ByteOrder byteOrder, int bucketSize, - boolean useIncrementalBuckets + byte version ) { if (Integer.bitCount(bucketSize) != 1 || bucketSize < 1 || bucketSize > 128) { @@ -91,7 +91,7 @@ public FrontCodedIndexedWriter( this.bucketBuffer = new byte[bucketSize][]; this.getOffsetBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder); this.div = Integer.numberOfTrailingZeros(bucketSize); - this.useIncrementalBuckets = useIncrementalBuckets; + this.version = FrontCodedIndexed.validateVersion(version); } @Override @@ -124,9 +124,9 @@ public void write(@Nullable byte[] value) throws IOException int written; // write the bucket, growing scratch buffer as necessary do { - written = useIncrementalBuckets - ? writeIncrementalBucket(scratch, bucketBuffer, bucketSize) - : writeBucket(scratch, bucketBuffer, bucketSize); + written = version == FrontCodedIndexed.V1 + ? writeBucketV1(scratch, bucketBuffer, bucketSize) + : writeBucketV0(scratch, bucketBuffer, bucketSize); if (written < 0) { growScratch(); } @@ -170,14 +170,7 @@ public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws I flush(); } resetScratch(); - - if (useIncrementalBuckets) { - // version 1 is incremental buckets - scratch.put((byte) 1); - } else { - // version 0 all values are prefixed on first bucket value - scratch.put((byte) 0); - } + scratch.put(version); scratch.put((byte) bucketSize); scratch.put(hasNulls ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE); VByte.writeInt(scratch, numWritten); @@ -222,7 +215,7 @@ public byte[] get(int index) throws IOException final ByteBuffer bucketBuffer = ByteBuffer.allocate(bucketBytesSize).order(byteOrder); valuesOut.readFully(startOffset, bucketBuffer); bucketBuffer.clear(); - final ByteBuffer valueBuffer = useIncrementalBuckets + final ByteBuffer valueBuffer = version == FrontCodedIndexed.V1 ? getFromBucketV1(bucketBuffer, relativeIndex, bucketSize) : FrontCodedIndexed.getFromBucketV0(bucketBuffer, relativeIndex); final byte[] valueBytes = new byte[valueBuffer.limit() - valueBuffer.position()]; @@ -248,9 +241,9 @@ private void flush() throws IOException int written; do { int flushSize = remainder == 0 ? bucketSize : remainder; - written = useIncrementalBuckets - ? writeIncrementalBucket(scratch, bucketBuffer, flushSize) - : writeBucket(scratch, bucketBuffer, flushSize); + written = version == FrontCodedIndexed.V1 + ? writeBucketV1(scratch, bucketBuffer, flushSize) + : writeBucketV0(scratch, bucketBuffer, flushSize); if (written < 0) { growScratch(); } @@ -283,7 +276,7 @@ private void growScratch() * * Uses {@link VByte} encoded integers to indicate prefix length and value length. */ - public static int writeBucket(ByteBuffer buffer, byte[][] values, int numValues) + public static int writeBucketV0(ByteBuffer buffer, byte[][] values, int numValues) { int written = 0; byte[] first = null; @@ -333,7 +326,7 @@ public static int writeBucket(ByteBuffer buffer, byte[][] values, int numValues) * * Uses {@link VByte} encoded integers to indicate prefix length and value length. */ - public static int writeIncrementalBucket(ByteBuffer buffer, byte[][] values, int numValues) + public static int writeBucketV1(ByteBuffer buffer, byte[][] values, int numValues) { int written = 0; byte[] prev = null; diff --git a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java new file mode 100644 index 000000000000..e2b219c0525d --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.column; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.segment.data.FrontCodedIndexed; +import org.junit.Assert; +import org.junit.Test; + +public class StringEncodingStrategyTest +{ + private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper(); + + + @Test + public void testUtf8Serde() throws JsonProcessingException + { + StringEncodingStrategy utf8 = new StringEncodingStrategy.Utf8(); + String there = JSON_MAPPER.writeValueAsString(utf8); + StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class); + Assert.assertEquals(utf8, andBackAgain); + } + + @Test + public void testFrontCodedDefaultSerde() throws JsonProcessingException + { + StringEncodingStrategy frontCoded = new StringEncodingStrategy.FrontCoded(null, null); + String there = JSON_MAPPER.writeValueAsString(frontCoded); + StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class); + Assert.assertEquals(frontCoded, andBackAgain); + Assert.assertEquals(FrontCodedIndexed.DEFAULT_BUCKET_SIZE, ((StringEncodingStrategy.FrontCoded) andBackAgain).getBucketSize()); + Assert.assertEquals(FrontCodedIndexed.DEFAULT_VERSION, ((StringEncodingStrategy.FrontCoded) andBackAgain).getFormatVersion()); + + // this next assert seems silly, but its a sanity check to make us think hard before changing the default version, + // to make us think of the backwards compatibility implications, as new versions of segment format stuff cannot be + // downgraded to older versions of Druid and still read + Assert.assertEquals(FrontCodedIndexed.V1, FrontCodedIndexed.DEFAULT_VERSION); + } + + @Test + public void testFrontCodedFormatSerde() throws JsonProcessingException + { + StringEncodingStrategy frontCodedV0 = new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0); + String there = JSON_MAPPER.writeValueAsString(frontCodedV0); + StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class); + Assert.assertEquals(frontCodedV0, andBackAgain); + + StringEncodingStrategy frontCodedV1 = new StringEncodingStrategy.FrontCoded(8, FrontCodedIndexed.V1); + there = JSON_MAPPER.writeValueAsString(frontCodedV1); + andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class); + Assert.assertEquals(frontCodedV1, andBackAgain); + } + + @Test + public void testEqualsAndHashcode() + { + EqualsVerifier.forClass(StringEncodingStrategy.Utf8.class).usingGetClass().verify(); + EqualsVerifier.forClass(StringEncodingStrategy.FrontCoded.class).usingGetClass().verify(); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java b/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java index 4c1c51231aaf..98e6b4bbbf04 100644 --- a/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java +++ b/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java @@ -47,20 +47,20 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest public static Collection constructorFeeder() { return ImmutableList.of( - new Object[]{ByteOrder.LITTLE_ENDIAN, true}, - new Object[]{ByteOrder.LITTLE_ENDIAN, false}, - new Object[]{ByteOrder.BIG_ENDIAN, true}, - new Object[]{ByteOrder.BIG_ENDIAN, false} + new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V1}, + new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V0}, + new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V1}, + new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V0} ); } private final ByteOrder order; - private final boolean useIncrementalBuckets; + private final byte version; - public FrontCodedIndexedTest(ByteOrder byteOrder, boolean useIncrementalBuckets) + public FrontCodedIndexedTest(ByteOrder byteOrder, byte version) { this.order = byteOrder; - this.useIncrementalBuckets = useIncrementalBuckets; + this.version = version; } @Test @@ -68,7 +68,7 @@ public void testFrontCodedIndexed() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order); List theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy"); - persistToBuffer(buffer, theList, 4, useIncrementalBuckets); + persistToBuffer(buffer, theList, 4, version); buffer.position(0); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( @@ -99,7 +99,7 @@ public void testFrontCodedIndexedSingleBucket() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order); List theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy"); - persistToBuffer(buffer, theList, 16, useIncrementalBuckets); + persistToBuffer(buffer, theList, 16, version); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( buffer, @@ -137,7 +137,7 @@ public void testFrontCodedIndexedBigger() throws IOException for (int i = 0; i < sizeBase + sizeAdjust; i++) { values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId()); } - persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets); + persistToBuffer(buffer, values, bucketSize, version); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( buffer, @@ -173,7 +173,7 @@ public void testFrontCodedIndexedBiggerWithNulls() throws IOException for (int i = 0; i < sizeBase + sizeAdjust; i++) { values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId()); } - persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets); + persistToBuffer(buffer, values, bucketSize, version); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( buffer, @@ -207,7 +207,7 @@ public void testFrontCodedIndexedIndexOf() throws IOException ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order); List theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy"); - persistToBuffer(buffer, theList, 4, useIncrementalBuckets); + persistToBuffer(buffer, theList, 4, version); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( buffer, @@ -231,7 +231,7 @@ public void testFrontCodedIndexedIndexOfWithNull() throws IOException TreeSet values = new TreeSet<>(GenericIndexed.STRING_STRATEGY); values.add(null); values.addAll(theList); - persistToBuffer(buffer, values, 4, useIncrementalBuckets); + persistToBuffer(buffer, values, 4, version); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( buffer, @@ -254,7 +254,7 @@ public void testFrontCodedIndexedUnicodes() throws IOException // "\uD83D\uDCA9" and "（請參見已被刪除版本）" are a regression test for https://github.com/apache/druid/pull/13364 List theList = ImmutableList.of("Győ-Moson-Sopron", "Győr", "\uD83D\uDCA9", "（請參見已被刪除版本）"); - persistToBuffer(buffer, theList, 4, useIncrementalBuckets); + persistToBuffer(buffer, theList, 4, version); buffer.position(0); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( @@ -282,7 +282,7 @@ public void testFrontCodedOnlyNull() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order); List theList = Collections.singletonList(null); - persistToBuffer(buffer, theList, 4, useIncrementalBuckets); + persistToBuffer(buffer, theList, 4, version); buffer.position(0); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( @@ -308,7 +308,7 @@ public void testFrontCodedEmpty() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1 << 6).order(order); List theList = Collections.emptyList(); - persistToBuffer(buffer, theList, 4, useIncrementalBuckets); + persistToBuffer(buffer, theList, 4, version); buffer.position(0); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( @@ -351,7 +351,7 @@ public void testBucketSizes() throws IOException values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId()); } for (int bucketSize : bucketSizes) { - persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets); + persistToBuffer(buffer, values, bucketSize, version); FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read( buffer, buffer.order() @@ -389,7 +389,7 @@ public void testBadBucketSize() medium, ByteOrder.nativeOrder(), 0, - useIncrementalBuckets + version ) ); @@ -399,7 +399,7 @@ public void testBadBucketSize() medium, ByteOrder.nativeOrder(), 15, - useIncrementalBuckets + version ) ); @@ -409,12 +409,17 @@ public void testBadBucketSize() medium, ByteOrder.nativeOrder(), 256, - useIncrementalBuckets + version ) ); } - private static long persistToBuffer(ByteBuffer buffer, Iterable sortedIterable, int bucketSize, boolean useIncrementalBuckets) throws IOException + private static long persistToBuffer( + ByteBuffer buffer, + Iterable sortedIterable, + int bucketSize, + byte version + ) throws IOException { Iterator sortedStrings = sortedIterable.iterator(); buffer.position(0); @@ -424,7 +429,7 @@ private static long persistToBuffer(ByteBuffer buffer, Iterable sortedIt medium, buffer.order(), bucketSize, - useIncrementalBuckets + version ); writer.open(); int index = 0;