Skip to content

Commit

Permalink
Fix velox_parquet_e2e_filter_test CI
Browse files Browse the repository at this point in the history
Co-authored-by: youxiduo <[email protected]>
  • Loading branch information
2 people authored and zhejiangxiaomai committed Jul 4, 2023
1 parent 07d191d commit 3071e92
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 0 deletions.
4 changes: 4 additions & 0 deletions velox/dwio/parquet/reader/ParquetReader.h
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ class ParquetReader : public dwio::common::Reader {
return readerBase_->schemaWithId();
}

size_t numberOfRowGroups() const {
return readerBase_->fileMetaData().row_groups.size();
}

std::unique_ptr<dwio::common::RowReader> createRowReader(
const dwio::common::RowReaderOptions& options = {}) const override;

Expand Down
22 changes: 22 additions & 0 deletions velox/dwio/parquet/tests/reader/E2EFilterTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,10 @@ TEST_F(E2EFilterTest, list) {
}

TEST_F(E2EFilterTest, metadataFilter) {
// Follow the batch size in `E2EFiltersTestBase`,
// so that each batch can produce a row group.
writerProperties_ =
::parquet::WriterProperties::Builder().max_row_group_length(10)->build();
testMetadataFilter();
}

Expand Down Expand Up @@ -568,6 +572,24 @@ TEST_F(E2EFilterTest, date) {
20);
}

TEST_F(E2EFilterTest, combineRowGroup) {
rowType_ = ROW({INTEGER()});
std::vector<RowVectorPtr> batches;
for (int i = 0; i < 5; i++) {
batches.push_back(std::static_pointer_cast<RowVector>(
test::BatchMaker::createBatch(rowType_, 1, *leafPool_, nullptr, 0)));
}
writeToMemory(rowType_, batches, false);
std::string_view data(sinkPtr_->getData(), sinkPtr_->size());
dwio::common::ReaderOptions readerOpts{leafPool_.get()};
auto input = std::make_unique<BufferedInput>(
std::make_shared<InMemoryReadFile>(data), readerOpts.getMemoryPool());
auto reader = makeReader(readerOpts, std::move(input));
auto parquetReader = dynamic_cast<ParquetReader&>(*reader.get());
EXPECT_EQ(parquetReader.numberOfRowGroups(), 1);
EXPECT_EQ(parquetReader.numberOfRows(), 5);
}

// Define main so that gflags get processed.
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
Expand Down

0 comments on commit 3071e92

Please sign in to comment.