apacheGH-3040: DictionaryFilter.canDrop may return false positive res…

…ult when dict size exceeds 8k

parquet-common/src/main/java/org/apache/parquet/bytes/BytesInput.java

@@ -26,6 +26,7 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.util.Arrays;
 import java.util.List;
@@ -376,7 +377,11 @@ void writeInto(ByteBuffer buffer) {
         ByteBuffer workBuf = buffer.duplicate();
         int pos = buffer.position();
         workBuf.limit(pos + byteCount);
-        Channels.newChannel(in).read(workBuf);
+        ReadableByteChannel channel = Channels.newChannel(in);
+        int remaining = byteCount;
+        while (remaining > 0) {
+          remaining -= channel.read(workBuf);
+        }
         buffer.position(pos + byteCount);
       } catch (IOException e) {
         new RuntimeException("Exception occurred during reading input stream", e);

parquet-common/src/test/java/org/apache/parquet/bytes/AvailableAgnosticInputStream.java

@@ -0,0 +1,35 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.parquet.bytes;
+
+import java.io.ByteArrayInputStream;
+
+public class AvailableAgnosticInputStream extends ByteArrayInputStream {
+
+  public AvailableAgnosticInputStream(byte[] buf) {
+    super(buf);
+  }
+
+  // In practice, there are some implementations always return 0 even if they has more data
+  @Override
+  public synchronized int available() {
+    return 0;
+  }
+}

parquet-common/src/test/java/org/apache/parquet/bytes/TestBytesInput.java

@@ -37,6 +37,7 @@
 import java.util.Random;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
+import org.apache.parquet.io.DelegatingSeekableInputStream;
 import org.apache.parquet.util.AutoCloseables;
 import org.junit.After;
 import org.junit.Before;
@@ -140,6 +141,20 @@ public void testFromInputStream() throws IOException {
     validate(data, factory);
   }
 
+  @Test
+  public void testFromLargeAvailableAgnosticInputStream() throws IOException {
+    // allocate a bytes that large than
+    // java.nio.channel.Channels.ReadableByteChannelImpl.TRANSFER_SIZE = 8192
+    byte[] data = new byte[9 * 1024];
+    RANDOM.nextBytes(data);
+    byte[] input = new byte[data.length + 10];
+    RANDOM.nextBytes(input);
+    System.arraycopy(data, 0, input, 0, data.length);
+    Supplier<BytesInput> factory = () -> BytesInput.from(new AvailableAgnosticInputStream(input), 9 * 1024);
+
+    validate(data, factory);
+  }
+
   @Test
   public void testFromByteArrayOutputStream() throws IOException {
     byte[] data = new byte[1000];