[Minor] Small refactor of raw index creator constructor to be more cl…

…ear (#13093)
apache · May 7, 2024 · b4dfd04 · b4dfd04
1 parent e07b576
commit b4dfd04
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 31 deletions.
diff --git a/...ache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java b/...ache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
@@ -77,16 +77,17 @@ public MultiValueFixedByteRawIndexCreator(File indexFile, ChunkCompressionType c
       throws IOException {
     // Store the length followed by the values
     int totalMaxLength = Integer.BYTES + (maxNumberOfMultiValueElements * valueType.getStoredType().size());
-    int numDocsPerChunk = deriveNumDocsPerChunk ? Math.max(
-        targetMaxChunkSizeBytes / (totalMaxLength + VarByteChunkForwardIndexWriter.CHUNK_HEADER_ENTRY_ROW_OFFSET_SIZE),
-        1) : targetDocsPerChunk;
-    // For columns with very small max value, target chunk size should also be capped to reduce memory during read
-    int dynamicTargetChunkSize =
-        ForwardIndexUtils.getDynamicTargetChunkSize(totalMaxLength, targetDocsPerChunk, targetMaxChunkSizeBytes);
-    _indexWriter =
-        writerVersion < VarByteChunkForwardIndexWriterV4.VERSION ? new VarByteChunkForwardIndexWriter(indexFile,
-            compressionType, totalDocs, numDocsPerChunk, totalMaxLength, writerVersion)
-            : new VarByteChunkForwardIndexWriterV4(indexFile, compressionType, dynamicTargetChunkSize);
+    if (writerVersion < VarByteChunkForwardIndexWriterV4.VERSION) {
+      int numDocsPerChunk = deriveNumDocsPerChunk ? Math.max(targetMaxChunkSizeBytes / (totalMaxLength
+          + VarByteChunkForwardIndexWriter.CHUNK_HEADER_ENTRY_ROW_OFFSET_SIZE), 1) : targetDocsPerChunk;
+      _indexWriter =
+          new VarByteChunkForwardIndexWriter(indexFile, compressionType, totalDocs, numDocsPerChunk, totalMaxLength,
+              writerVersion);
+    } else {
+      int chunkSize =
+          ForwardIndexUtils.getDynamicTargetChunkSize(totalMaxLength, targetDocsPerChunk, targetMaxChunkSizeBytes);
+      _indexWriter = new VarByteChunkForwardIndexWriterV4(indexFile, compressionType, chunkSize);
+    }
     _valueType = valueType;
   }
 

diff --git a/...apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java b/...apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
@@ -75,19 +75,20 @@ public MultiValueVarByteRawIndexCreator(File baseIndexDir, ChunkCompressionType
       int totalDocs, DataType valueType, int writerVersion, int maxRowLengthInBytes, int maxNumberOfElements,
       int targetMaxChunkSizeBytes, int targetDocsPerChunk)
       throws IOException {
-    //we will prepend the actual content with numElements and length array containing length of each element
-    int totalMaxLength = getTotalRowStorageBytes(maxNumberOfElements, maxRowLengthInBytes);
-
     File file = new File(baseIndexDir, column + Indexes.RAW_MV_FORWARD_INDEX_FILE_EXTENSION);
-    int numDocsPerChunk = Math.max(
-        targetMaxChunkSizeBytes / (totalMaxLength + VarByteChunkForwardIndexWriter.CHUNK_HEADER_ENTRY_ROW_OFFSET_SIZE),
-        1);
-    // For columns with very small max value, target chunk size should also be capped to reduce memory during read
-    int dynamicTargetChunkSize =
-        ForwardIndexUtils.getDynamicTargetChunkSize(totalMaxLength, targetDocsPerChunk, targetMaxChunkSizeBytes);
-    _indexWriter = writerVersion < VarByteChunkForwardIndexWriterV4.VERSION ? new VarByteChunkForwardIndexWriter(file,
-        compressionType, totalDocs, numDocsPerChunk, totalMaxLength, writerVersion)
-        : new VarByteChunkForwardIndexWriterV4(file, compressionType, dynamicTargetChunkSize);
+    // We will prepend the actual content with numElements and length array containing length of each element
+    int totalMaxLength = getTotalRowStorageBytes(maxNumberOfElements, maxRowLengthInBytes);
+    if (writerVersion < VarByteChunkForwardIndexWriterV4.VERSION) {
+      int numDocsPerChunk = Math.max(targetMaxChunkSizeBytes / (totalMaxLength
+          + VarByteChunkForwardIndexWriter.CHUNK_HEADER_ENTRY_ROW_OFFSET_SIZE), 1);
+      _indexWriter =
+          new VarByteChunkForwardIndexWriter(file, compressionType, totalDocs, numDocsPerChunk, totalMaxLength,
+              writerVersion);
+    } else {
+      int chunkSize =
+          ForwardIndexUtils.getDynamicTargetChunkSize(totalMaxLength, targetDocsPerChunk, targetMaxChunkSizeBytes);
+      _indexWriter = new VarByteChunkForwardIndexWriterV4(file, compressionType, chunkSize);
+    }
     _valueType = valueType;
   }
 

diff --git a/...pache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java b/...pache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
@@ -78,15 +78,16 @@ public SingleValueVarByteRawIndexCreator(File baseIndexDir, ChunkCompressionType
       int targetMaxChunkSizeBytes, int targetDocsPerChunk)
       throws IOException {
     File file = new File(baseIndexDir, column + V1Constants.Indexes.RAW_SV_FORWARD_INDEX_FILE_EXTENSION);
-    int numDocsPerChunk =
-        deriveNumDocsPerChunk ? getNumDocsPerChunk(maxLength, targetMaxChunkSizeBytes) : targetDocsPerChunk;
-
-    // For columns with very small max value, target chunk size should also be capped to reduce memory during read
-    int dynamicTargetChunkSize =
-        ForwardIndexUtils.getDynamicTargetChunkSize(maxLength, targetDocsPerChunk, targetMaxChunkSizeBytes);
-    _indexWriter = writerVersion < VarByteChunkForwardIndexWriterV4.VERSION ? new VarByteChunkForwardIndexWriter(file,
-        compressionType, totalDocs, numDocsPerChunk, maxLength, writerVersion)
-        : new VarByteChunkForwardIndexWriterV4(file, compressionType, dynamicTargetChunkSize);
+    if (writerVersion < VarByteChunkForwardIndexWriterV4.VERSION) {
+      int numDocsPerChunk =
+          deriveNumDocsPerChunk ? getNumDocsPerChunk(maxLength, targetMaxChunkSizeBytes) : targetDocsPerChunk;
+      _indexWriter = new VarByteChunkForwardIndexWriter(file, compressionType, totalDocs, numDocsPerChunk, maxLength,
+          writerVersion);
+    } else {
+      int chunkSize =
+          ForwardIndexUtils.getDynamicTargetChunkSize(maxLength, targetDocsPerChunk, targetMaxChunkSizeBytes);
+      _indexWriter = new VarByteChunkForwardIndexWriterV4(file, compressionType, chunkSize);
+    }
     _valueType = valueType;
   }