Skip to content

Commit

Permalink
[native] Add support for ORC reader and add orc native tests
Browse files Browse the repository at this point in the history
  • Loading branch information
wypb committed Feb 12, 2025
1 parent 7882d83 commit 18c6209
Show file tree
Hide file tree
Showing 14 changed files with 661 additions and 265 deletions.
2 changes: 2 additions & 0 deletions presto-docs/src/main/sphinx/presto-cpp.rst
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,6 @@ Only specific connectors are supported in the Presto C++ evaluation engine.

* Iceberg connector supports both V1 and V2 tables, including tables with delete files.

* Supports reading and writing of DWRF and PARQUET file formats, ORC only supports reading.

* TPCH connector, with ``tpch.naming=standard`` catalog property.
2 changes: 2 additions & 0 deletions presto-native-execution/presto_cpp/main/PrestoServer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
#include "velox/connectors/tpch/TpchConnector.h"
#include "velox/dwio/dwrf/RegisterDwrfReader.h"
#include "velox/dwio/dwrf/RegisterDwrfWriter.h"
#include "velox/dwio/orc/reader/OrcReader.h"
#include "velox/dwio/parquet/RegisterParquetReader.h"
#include "velox/dwio/parquet/RegisterParquetWriter.h"
#include "velox/exec/OutputBufferManager.h"
Expand Down Expand Up @@ -1391,6 +1392,7 @@ void PrestoServer::registerMemoryArbitrators() {
void PrestoServer::registerFileReadersAndWriters() {
velox::dwrf::registerDwrfReaderFactory();
velox::dwrf::registerDwrfWriterFactory();
velox::orc::registerOrcReaderFactory();
velox::parquet::registerParquetReaderFactory();
velox::parquet::registerParquetWriterFactory();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ dwio::common::FileFormat toVeloxFileFormat(
const presto::protocol::hive::StorageFormat& format) {
if (format.inputFormat == "com.facebook.hive.orc.OrcInputFormat") {
return dwio::common::FileFormat::DWRF;
} else if (
format.inputFormat == "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat") {
return dwio::common::FileFormat::ORC;
} else if (
format.inputFormat ==
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat") {
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ public static void createNation(QueryRunner queryRunner)
if (!queryRunner.tableExists(queryRunner.getDefaultSession(), "nation")) {
queryRunner.execute("CREATE TABLE nation AS SELECT * FROM tpch.tiny.nation");
}
if (!queryRunner.tableExists(queryRunner.getDefaultSession(), "nation")) {
queryRunner.execute("CREATE TABLE nation WITH (FORMAT = 'ORC') AS SELECT * FROM tpch.tiny.nation");
}
if (!queryRunner.tableExists(queryRunner.getDefaultSession(), "nation_json")) {
queryRunner.execute("CREATE TABLE nation_json WITH (FORMAT = 'JSON') AS SELECT * FROM tpch.tiny.nation");
}
Expand All @@ -214,7 +217,7 @@ public static void createNationWithFormat(Session session, QueryRunner queryRunn
}

if (storageFormat.equals("ORC") && !queryRunner.tableExists(session, "nation")) {
queryRunner.execute(session, "CREATE TABLE nation AS SELECT * FROM tpch.tiny.nation");
queryRunner.execute(session, "CREATE TABLE nation WITH (FORMAT = 'ORC') AS SELECT * FROM tpch.tiny.nation");
}

if (storageFormat.equals("JSON") && !queryRunner.tableExists(session, "nation_json")) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.nativeworker;

import com.facebook.presto.testing.ExpectedQueryRunner;
import com.facebook.presto.testing.QueryRunner;

import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner;
import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner;

public class TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift
extends AbstractTestNativeIcebergPositionDeleteQueries
{
private final String storageFormat = "ORC";

@Override
protected QueryRunner createQueryRunner()
throws Exception
{
return createNativeIcebergQueryRunner(true, storageFormat);
}

@Override
protected ExpectedQueryRunner createExpectedQueryRunner()
throws Exception
{
return createJavaIcebergQueryRunner(storageFormat);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner;
import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner;

public class TestPrestoNativeIcebergPositionDeleteQueriesUsingThrift
public class TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift
extends AbstractTestNativeIcebergPositionDeleteQueries
{
private final String storageFormat = "PARQUET";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.nativeworker;

import com.facebook.presto.testing.ExpectedQueryRunner;
import com.facebook.presto.testing.QueryRunner;
import org.testng.annotations.Test;

@Test(groups = {"orc"})
public class TestPrestoNativeIcebergTpcdsQueriesOrcUsingThrift
extends AbstractTestNativeTpcdsQueries
{
@Override
protected QueryRunner createQueryRunner()
throws Exception
{
this.storageFormat = "ORC";
return PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(true, "ORC");
}

@Override
protected ExpectedQueryRunner createExpectedQueryRunner()
throws Exception
{
this.storageFormat = "ORC";
return PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner("ORC");
}

protected void runAllQueries() throws Exception
{
testTpcdsQ1();
testTpcdsQ2();
testTpcdsQ3();
testTpcdsQ4();
testTpcdsQ5();
testTpcdsQ6();
testTpcdsQ7();
testTpcdsQ8();
testTpcdsQ9();
testTpcdsQ10();
testTpcdsQ11();
testTpcdsQ12();
testTpcdsQ13();
testTpcdsQ14_1();
testTpcdsQ14_2();
testTpcdsQ15();
testTpcdsQ16();
testTpcdsQ17();
testTpcdsQ18();
testTpcdsQ19();
testTpcdsQ20();
// testTpcdsQ21();
testTpcdsQ22();
testTpcdsQ23_1();
testTpcdsQ23_2();
testTpcdsQ24_1();
testTpcdsQ24_2();
testTpcdsQ25();
testTpcdsQ26();
testTpcdsQ27();
testTpcdsQ28();
testTpcdsQ29();
testTpcdsQ30();
testTpcdsQ31();
testTpcdsQ32();
// testTpcdsQ33();
testTpcdsQ34();
testTpcdsQ35();
testTpcdsQ36();
// testTpcdsQ37();
testTpcdsQ38();
testTpcdsQ39_1();
testTpcdsQ39_2();
// testTpcdsQ40();
testTpcdsQ41();
testTpcdsQ42();
// testTpcdsQ43();
testTpcdsQ44();
testTpcdsQ45();
testTpcdsQ46();
testTpcdsQ47();
testTpcdsQ48();
// testTpcdsQ49();
testTpcdsQ50();
testTpcdsQ51();
testTpcdsQ52();
testTpcdsQ53();
testTpcdsQ54();
testTpcdsQ55();
// testTpcdsQ56();
testTpcdsQ57();
testTpcdsQ58();
testTpcdsQ59();
// testTpcdsQ60();
// testTpcdsQ61();
testTpcdsQ62();
testTpcdsQ63();
testTpcdsQ65();
testTpcdsQ66();
testTpcdsQ67();
testTpcdsQ68();
testTpcdsQ69();
testTpcdsQ70();
testTpcdsQ71();
testTpcdsQ72();
testTpcdsQ73();
testTpcdsQ74();
testTpcdsQ75();
testTpcdsQ76();
testTpcdsQ77();
testTpcdsQ78();
testTpcdsQ79();
// testTpcdsQ80();
testTpcdsQ81();
// testTpcdsQ82();
testTpcdsQ83();
testTpcdsQ84();
testTpcdsQ85();
testTpcdsQ86();
testTpcdsQ87();
testTpcdsQ88();
testTpcdsQ89();
testTpcdsQ90();
// testTpcdsQ91();
testTpcdsQ92();
testTpcdsQ93();
testTpcdsQ94();
testTpcdsQ95();
testTpcdsQ96();
testTpcdsQ97();
testTpcdsQ98();
testTpcdsQ99();
}

@Test
public void doDeletesAndQuery() throws Exception
{
doDeletes();
verifyDeletes();
runAllQueries();
}
}
Loading

0 comments on commit 18c6209

Please sign in to comment.