diff --git a/native-sql-engine/cpp/CMakeLists.txt b/cpp/CMakeLists.txt similarity index 100% rename from native-sql-engine/cpp/CMakeLists.txt rename to cpp/CMakeLists.txt diff --git a/native-sql-engine/cpp/compile.sh b/cpp/compile.sh similarity index 100% rename from native-sql-engine/cpp/compile.sh rename to cpp/compile.sh diff --git a/native-sql-engine/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt similarity index 99% rename from native-sql-engine/cpp/src/CMakeLists.txt rename to cpp/src/CMakeLists.txt index 2a51616c6608..064ed8d89996 100644 --- a/native-sql-engine/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -80,7 +80,7 @@ set(JEMALLOC_BUILD_VERSION "5.2.1") find_package(JNI REQUIRED) set(source_root_directory ${CMAKE_CURRENT_SOURCE_DIR}) -set(substrait_proto_directory ${CMAKE_CURRENT_SOURCE_DIR}/../../core/src/main/java/com/intel/oap/substrait/binary) +set(substrait_proto_directory ${CMAKE_CURRENT_SOURCE_DIR}/../../jvm/src/main/java/com/intel/oap/substrait/binary) message(STATUS "Set Substrait Directory in ${substrait_proto_directory}") find_program(CCACHE_FOUND ccache) diff --git a/native-sql-engine/cpp/src/common/result_iterator.h b/cpp/src/common/result_iterator.h similarity index 100% rename from native-sql-engine/cpp/src/common/result_iterator.h rename to cpp/src/common/result_iterator.h diff --git a/native-sql-engine/cpp/src/jni/concurrent_map.h b/cpp/src/jni/concurrent_map.h similarity index 100% rename from native-sql-engine/cpp/src/jni/concurrent_map.h rename to cpp/src/jni/concurrent_map.h diff --git a/native-sql-engine/cpp/src/jni/jni_common.h b/cpp/src/jni/jni_common.h similarity index 100% rename from native-sql-engine/cpp/src/jni/jni_common.h rename to cpp/src/jni/jni_common.h diff --git a/native-sql-engine/cpp/src/jni/jni_wrapper.cc b/cpp/src/jni/jni_wrapper.cc similarity index 100% rename from native-sql-engine/cpp/src/jni/jni_wrapper.cc rename to cpp/src/jni/jni_wrapper.cc diff --git a/native-sql-engine/cpp/src/operators/columnar_to_row_converter.cc b/cpp/src/operators/columnar_to_row_converter.cc similarity index 100% rename from native-sql-engine/cpp/src/operators/columnar_to_row_converter.cc rename to cpp/src/operators/columnar_to_row_converter.cc diff --git a/native-sql-engine/cpp/src/operators/columnar_to_row_converter.h b/cpp/src/operators/columnar_to_row_converter.h similarity index 100% rename from native-sql-engine/cpp/src/operators/columnar_to_row_converter.h rename to cpp/src/operators/columnar_to_row_converter.h diff --git a/native-sql-engine/cpp/src/proto/protobuf_utils.cc b/cpp/src/proto/protobuf_utils.cc similarity index 100% rename from native-sql-engine/cpp/src/proto/protobuf_utils.cc rename to cpp/src/proto/protobuf_utils.cc diff --git a/native-sql-engine/cpp/src/proto/protobuf_utils.h b/cpp/src/proto/protobuf_utils.h similarity index 100% rename from native-sql-engine/cpp/src/proto/protobuf_utils.h rename to cpp/src/proto/protobuf_utils.h diff --git a/native-sql-engine/cpp/src/proto/substrait_utils.cc b/cpp/src/proto/substrait_utils.cc similarity index 100% rename from native-sql-engine/cpp/src/proto/substrait_utils.cc rename to cpp/src/proto/substrait_utils.cc diff --git a/native-sql-engine/cpp/src/proto/substrait_utils.h b/cpp/src/proto/substrait_utils.h similarity index 100% rename from native-sql-engine/cpp/src/proto/substrait_utils.h rename to cpp/src/proto/substrait_utils.h diff --git a/native-sql-engine/cpp/src/resources/libhdfs.so b/cpp/src/resources/libhdfs.so similarity index 100% rename from native-sql-engine/cpp/src/resources/libhdfs.so rename to cpp/src/resources/libhdfs.so diff --git a/native-sql-engine/cpp/src/resources/libhdfs3.so b/cpp/src/resources/libhdfs3.so similarity index 100% rename from native-sql-engine/cpp/src/resources/libhdfs3.so rename to cpp/src/resources/libhdfs3.so diff --git a/native-sql-engine/cpp/src/resources/libprotobuf.so.13 b/cpp/src/resources/libprotobuf.so.13 similarity index 100% rename from native-sql-engine/cpp/src/resources/libprotobuf.so.13 rename to cpp/src/resources/libprotobuf.so.13 diff --git a/native-sql-engine/core/pom.xml b/jvm/pom.xml similarity index 98% rename from native-sql-engine/core/pom.xml rename to jvm/pom.xml index c059b6e5cb16..c31ef3c44c5c 100644 --- a/native-sql-engine/core/pom.xml +++ b/jvm/pom.xml @@ -16,16 +16,15 @@ 4.0.0 com.intel.oap - native-sql-engine-parent + gazelle-jni-parent 1.2.0-snapshot - ../../pom.xml + ../pom.xml - com.intel.oap - spark-columnar-core + gazelle-jni-jvm 1.2.0-snapshot jar - OAP Project Spark Columnar Plugin + Gazelle Jni ../cpp/ diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/execution/ColumnarNativeIterator.java b/jvm/src/main/java/com/intel/oap/execution/ColumnarNativeIterator.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/execution/ColumnarNativeIterator.java rename to jvm/src/main/java/com/intel/oap/execution/ColumnarNativeIterator.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java b/jvm/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java rename to jvm/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java b/jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java rename to jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java b/jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java rename to jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java b/jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java rename to jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java b/jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java rename to jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java b/jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java rename to jvm/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/Exprs.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/Exprs.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/Exprs.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/Exprs.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/expression.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/expression.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/expression.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/expression.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/extensions.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/extensions.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/extensions.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/extensions.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/function.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/function.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/function.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/function.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/parameterized_types.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/parameterized_types.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/parameterized_types.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/parameterized_types.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/plan.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/plan.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/plan.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/plan.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/relations.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/relations.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/relations.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/relations.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/selection.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/selection.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/selection.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/selection.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/type.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/type.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/type.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/type.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/type_expressions.proto b/jvm/src/main/java/com/intel/oap/substrait/binary/type_expressions.proto similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/binary/type_expressions.proto rename to jvm/src/main/java/com/intel/oap/substrait/binary/type_expressions.proto diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/BinaryOPNode.java b/jvm/src/main/java/com/intel/oap/substrait/derivation/BinaryOPNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/BinaryOPNode.java rename to jvm/src/main/java/com/intel/oap/substrait/derivation/BinaryOPNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionBuilder.java b/jvm/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionBuilder.java rename to jvm/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionNode.java b/jvm/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionNode.java rename to jvm/src/main/java/com/intel/oap/substrait/derivation/DerivationExpressionNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/DerivationFP64TypeNode.java b/jvm/src/main/java/com/intel/oap/substrait/derivation/DerivationFP64TypeNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/derivation/DerivationFP64TypeNode.java rename to jvm/src/main/java/com/intel/oap/substrait/derivation/DerivationFP64TypeNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/AggregateFunctionNode.java b/jvm/src/main/java/com/intel/oap/substrait/expression/AggregateFunctionNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/AggregateFunctionNode.java rename to jvm/src/main/java/com/intel/oap/substrait/expression/AggregateFunctionNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/DoubleLiteralNode.java b/jvm/src/main/java/com/intel/oap/substrait/expression/DoubleLiteralNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/DoubleLiteralNode.java rename to jvm/src/main/java/com/intel/oap/substrait/expression/DoubleLiteralNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/ExpressionBuilder.java b/jvm/src/main/java/com/intel/oap/substrait/expression/ExpressionBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/ExpressionBuilder.java rename to jvm/src/main/java/com/intel/oap/substrait/expression/ExpressionBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/ExpressionNode.java b/jvm/src/main/java/com/intel/oap/substrait/expression/ExpressionNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/ExpressionNode.java rename to jvm/src/main/java/com/intel/oap/substrait/expression/ExpressionNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/ScalarFunctionNode.java b/jvm/src/main/java/com/intel/oap/substrait/expression/ScalarFunctionNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/ScalarFunctionNode.java rename to jvm/src/main/java/com/intel/oap/substrait/expression/ScalarFunctionNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/SelectionNode.java b/jvm/src/main/java/com/intel/oap/substrait/expression/SelectionNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/expression/SelectionNode.java rename to jvm/src/main/java/com/intel/oap/substrait/expression/SelectionNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/extensions/FunctionMappingNode.java b/jvm/src/main/java/com/intel/oap/substrait/extensions/FunctionMappingNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/extensions/FunctionMappingNode.java rename to jvm/src/main/java/com/intel/oap/substrait/extensions/FunctionMappingNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/extensions/MappingBuilder.java b/jvm/src/main/java/com/intel/oap/substrait/extensions/MappingBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/extensions/MappingBuilder.java rename to jvm/src/main/java/com/intel/oap/substrait/extensions/MappingBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/extensions/MappingNode.java b/jvm/src/main/java/com/intel/oap/substrait/extensions/MappingNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/extensions/MappingNode.java rename to jvm/src/main/java/com/intel/oap/substrait/extensions/MappingNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/plan/PlanBuilder.java b/jvm/src/main/java/com/intel/oap/substrait/plan/PlanBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/plan/PlanBuilder.java rename to jvm/src/main/java/com/intel/oap/substrait/plan/PlanBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/plan/PlanNode.java b/jvm/src/main/java/com/intel/oap/substrait/plan/PlanNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/plan/PlanNode.java rename to jvm/src/main/java/com/intel/oap/substrait/plan/PlanNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/AggregateRelNode.java b/jvm/src/main/java/com/intel/oap/substrait/rel/AggregateRelNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/AggregateRelNode.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/AggregateRelNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/FilterRelNode.java b/jvm/src/main/java/com/intel/oap/substrait/rel/FilterRelNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/FilterRelNode.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/FilterRelNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/LocalFilesBuilder.java b/jvm/src/main/java/com/intel/oap/substrait/rel/LocalFilesBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/LocalFilesBuilder.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/LocalFilesBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/LocalFilesNode.java b/jvm/src/main/java/com/intel/oap/substrait/rel/LocalFilesNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/LocalFilesNode.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/LocalFilesNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/ProjectRelNode.java b/jvm/src/main/java/com/intel/oap/substrait/rel/ProjectRelNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/ProjectRelNode.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/ProjectRelNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/ReadRelNode.java b/jvm/src/main/java/com/intel/oap/substrait/rel/ReadRelNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/ReadRelNode.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/ReadRelNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/RelBuilder.java b/jvm/src/main/java/com/intel/oap/substrait/rel/RelBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/RelBuilder.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/RelBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/RelNode.java b/jvm/src/main/java/com/intel/oap/substrait/rel/RelNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/rel/RelNode.java rename to jvm/src/main/java/com/intel/oap/substrait/rel/RelNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/BooleanTypeNode.java b/jvm/src/main/java/com/intel/oap/substrait/type/BooleanTypeNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/BooleanTypeNode.java rename to jvm/src/main/java/com/intel/oap/substrait/type/BooleanTypeNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/FP64TypeNode.java b/jvm/src/main/java/com/intel/oap/substrait/type/FP64TypeNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/FP64TypeNode.java rename to jvm/src/main/java/com/intel/oap/substrait/type/FP64TypeNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/StringTypeNode.java b/jvm/src/main/java/com/intel/oap/substrait/type/StringTypeNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/StringTypeNode.java rename to jvm/src/main/java/com/intel/oap/substrait/type/StringTypeNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/StructNode.java b/jvm/src/main/java/com/intel/oap/substrait/type/StructNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/StructNode.java rename to jvm/src/main/java/com/intel/oap/substrait/type/StructNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/TypeBuiler.java b/jvm/src/main/java/com/intel/oap/substrait/type/TypeBuiler.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/TypeBuiler.java rename to jvm/src/main/java/com/intel/oap/substrait/type/TypeBuiler.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/TypeNode.java b/jvm/src/main/java/com/intel/oap/substrait/type/TypeNode.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/substrait/type/TypeNode.java rename to jvm/src/main/java/com/intel/oap/substrait/type/TypeNode.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java b/jvm/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java rename to jvm/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowInfo.java b/jvm/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowInfo.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowInfo.java rename to jvm/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowInfo.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowJniWrapper.java b/jvm/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowJniWrapper.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowJniWrapper.java rename to jvm/src/main/java/com/intel/oap/vectorized/ArrowColumnarToRowJniWrapper.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java b/jvm/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java rename to jvm/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java b/jvm/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java rename to jvm/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java b/jvm/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java rename to jvm/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/BatchIterator.java b/jvm/src/main/java/com/intel/oap/vectorized/BatchIterator.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/BatchIterator.java rename to jvm/src/main/java/com/intel/oap/vectorized/BatchIterator.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java b/jvm/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java rename to jvm/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java b/jvm/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java rename to jvm/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java b/jvm/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java rename to jvm/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java b/jvm/src/main/java/com/intel/oap/vectorized/JniUtils.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java rename to jvm/src/main/java/com/intel/oap/vectorized/JniUtils.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/MetricsObject.java b/jvm/src/main/java/com/intel/oap/vectorized/MetricsObject.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/MetricsObject.java rename to jvm/src/main/java/com/intel/oap/vectorized/MetricsObject.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativePartitioning.java b/jvm/src/main/java/com/intel/oap/vectorized/NativePartitioning.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativePartitioning.java rename to jvm/src/main/java/com/intel/oap/vectorized/NativePartitioning.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java b/jvm/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java rename to jvm/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java b/jvm/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java rename to jvm/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SerializableObject.java b/jvm/src/main/java/com/intel/oap/vectorized/SerializableObject.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SerializableObject.java rename to jvm/src/main/java/com/intel/oap/vectorized/SerializableObject.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java b/jvm/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java rename to jvm/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java b/jvm/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java rename to jvm/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SplitResult.java b/jvm/src/main/java/com/intel/oap/vectorized/SplitResult.java similarity index 100% rename from native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SplitResult.java rename to jvm/src/main/java/com/intel/oap/vectorized/SplitResult.java diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/GazellePlugin.scala b/jvm/src/main/scala/com/intel/oap/GazellePlugin.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/GazellePlugin.scala rename to jvm/src/main/scala/com/intel/oap/GazellePlugin.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/GazellePluginConfig.scala b/jvm/src/main/scala/com/intel/oap/GazellePluginConfig.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/GazellePluginConfig.scala rename to jvm/src/main/scala/com/intel/oap/GazellePluginConfig.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ArrowColumnarToRowExec.scala b/jvm/src/main/scala/com/intel/oap/execution/ArrowColumnarToRowExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/ArrowColumnarToRowExec.scala rename to jvm/src/main/scala/com/intel/oap/execution/ArrowColumnarToRowExec.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/BasicPhysicalOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/BasicPhysicalOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/BasicPhysicalOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/BasicPhysicalOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/BatchScanExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/BatchScanExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/BatchScanExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/BatchScanExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala b/jvm/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala rename to jvm/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/BroadcastHashJoinExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/BroadcastHashJoinExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/BroadcastHashJoinExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/BroadcastHashJoinExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala b/jvm/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala rename to jvm/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/CoalesceExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/CoalesceExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/CoalesceExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/CoalesceExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarInMemoryTableScanExec.scala b/jvm/src/main/scala/com/intel/oap/execution/ColumnarInMemoryTableScanExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarInMemoryTableScanExec.scala rename to jvm/src/main/scala/com/intel/oap/execution/ColumnarInMemoryTableScanExec.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala b/jvm/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala rename to jvm/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ExpandExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/ExpandExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/ExpandExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/ExpandExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/HashAggregateExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/HashAggregateExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/HashAggregateExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/HashAggregateExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala b/jvm/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala rename to jvm/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ShuffledHashJoinExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/ShuffledHashJoinExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/ShuffledHashJoinExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/ShuffledHashJoinExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/SortExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/SortExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/SortExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/SortExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/SortMergeJoinExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/SortMergeJoinExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/SortMergeJoinExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/SortMergeJoinExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/WholeStageTransformerExec.scala b/jvm/src/main/scala/com/intel/oap/execution/WholeStageTransformerExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/WholeStageTransformerExec.scala rename to jvm/src/main/scala/com/intel/oap/execution/WholeStageTransformerExec.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/WholestageColumnarRDD.scala b/jvm/src/main/scala/com/intel/oap/execution/WholestageColumnarRDD.scala similarity index 97% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/WholestageColumnarRDD.scala rename to jvm/src/main/scala/com/intel/oap/execution/WholestageColumnarRDD.scala index 461d35edbd13..c55e154687f2 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/WholestageColumnarRDD.scala +++ b/jvm/src/main/scala/com/intel/oap/execution/WholestageColumnarRDD.scala @@ -31,8 +31,6 @@ import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.read.{InputPartition, PartitionReaderFactory} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.datasources.FilePartition -import org.apache.spark.sql.execution.datasources.v2.VectorizedFilePartitionReaderHandler -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetPartitionReaderFactory import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.util.ArrowUtils import org.apache.spark.sql.util.OASPackageBridge._ diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/WindowExecTransformer.scala b/jvm/src/main/scala/com/intel/oap/execution/WindowExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/execution/WindowExecTransformer.scala rename to jvm/src/main/scala/com/intel/oap/execution/WindowExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ArithmeticTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/ArithmeticTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/ArithmeticTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/ArithmeticTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/BinaryExpressionTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/BinaryExpressionTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/BinaryExpressionTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/BinaryExpressionTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/BinaryOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/BinaryOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/BinaryOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/BinaryOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/BoundReferenceTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/BoundReferenceTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/BoundReferenceTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/BoundReferenceTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/CaseWhenOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/CaseWhenOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/CaseWhenOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/CaseWhenOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/CoalesceOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/CoalesceOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/CoalesceOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/CoalesceOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/CodeGeneration.scala b/jvm/src/main/scala/com/intel/oap/expression/CodeGeneration.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/CodeGeneration.scala rename to jvm/src/main/scala/com/intel/oap/expression/CodeGeneration.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConcatOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/ConcatOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConcatOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/ConcatOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala b/jvm/src/main/scala/com/intel/oap/expression/ConverterUtils.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala rename to jvm/src/main/scala/com/intel/oap/expression/ConverterUtils.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/DateTimeExpressionsTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/DateTimeExpressionsTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/DateTimeExpressionsTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/DateTimeExpressionsTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ExpressionConverter.scala b/jvm/src/main/scala/com/intel/oap/expression/ExpressionConverter.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/ExpressionConverter.scala rename to jvm/src/main/scala/com/intel/oap/expression/ExpressionConverter.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ExpressionTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/ExpressionTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/ExpressionTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/ExpressionTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/IfOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/IfOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/IfOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/IfOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/InOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/InOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/InOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/InOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/InSetOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/InSetOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/InSetOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/InSetOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/LiteralTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/LiteralTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/LiteralTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/LiteralTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/NamedExpressionsTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/NamedExpressionsTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/NamedExpressionsTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/NamedExpressionsTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/RoundOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/RoundOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/RoundOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/RoundOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ScalarSubqueryTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/ScalarSubqueryTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/ScalarSubqueryTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/ScalarSubqueryTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/TernaryOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/TernaryOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/TernaryOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/TernaryOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UnaryOperatorTransformer.scala b/jvm/src/main/scala/com/intel/oap/expression/UnaryOperatorTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/UnaryOperatorTransformer.scala rename to jvm/src/main/scala/com/intel/oap/expression/UnaryOperatorTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/extension/ColumnarOverrides.scala b/jvm/src/main/scala/com/intel/oap/extension/ColumnarOverrides.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/extension/ColumnarOverrides.scala rename to jvm/src/main/scala/com/intel/oap/extension/ColumnarOverrides.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/extension/StrategyOverrides.scala b/jvm/src/main/scala/com/intel/oap/extension/StrategyOverrides.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/extension/StrategyOverrides.scala rename to jvm/src/main/scala/com/intel/oap/extension/StrategyOverrides.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/extension/columnar/ColumnarGuardRule.scala b/jvm/src/main/scala/com/intel/oap/extension/columnar/ColumnarGuardRule.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/extension/columnar/ColumnarGuardRule.scala rename to jvm/src/main/scala/com/intel/oap/extension/columnar/ColumnarGuardRule.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala b/jvm/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala rename to jvm/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala b/jvm/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala rename to jvm/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala b/jvm/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala rename to jvm/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/jvm/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala rename to jvm/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala b/jvm/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala rename to jvm/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarInMemoryRelation.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarInMemoryRelation.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarInMemoryRelation.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarInMemoryRelation.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkVectorUtils.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkVectorUtils.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkVectorUtils.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkVectorUtils.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExecTransformer.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExecTransformer.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExecTransformer.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExecTransformer.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/python/ColumnarArrowPythonRunner.scala b/jvm/src/main/scala/org/apache/spark/sql/execution/python/ColumnarArrowPythonRunner.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/python/ColumnarArrowPythonRunner.scala rename to jvm/src/main/scala/org/apache/spark/sql/execution/python/ColumnarArrowPythonRunner.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala b/jvm/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala rename to jvm/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala b/jvm/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala rename to jvm/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala b/jvm/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala rename to jvm/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/jvm/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala rename to jvm/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/util/ExecutorManager.scala b/jvm/src/main/scala/org/apache/spark/util/ExecutorManager.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/util/ExecutorManager.scala rename to jvm/src/main/scala/org/apache/spark/util/ExecutorManager.scala diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala b/jvm/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala similarity index 100% rename from native-sql-engine/core/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala rename to jvm/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java deleted file mode 100644 index ff238fc32a41..000000000000 --- a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intel.oap.datasource; - -import com.intel.oap.vectorized.ArrowWritableColumnVector; - -import java.io.IOException; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -import org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader; -import com.intel.oap.datasource.parquet.ParquetReader; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.parquet.HadoopReadOptions; -import org.apache.parquet.ParquetReadOptions; -import org.apache.parquet.hadoop.ParquetFileReader; -import org.apache.parquet.hadoop.ParquetInputSplit; -import org.apache.parquet.hadoop.metadata.BlockMetaData; -import org.apache.parquet.hadoop.util.ContextUtil; -import org.apache.parquet.hadoop.util.HadoopInputFile; -import org.apache.spark.sql.catalyst.InternalRow; -import org.apache.spark.sql.types.StructType; -import org.apache.spark.sql.vectorized.ColumnarBatch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class VectorizedParquetArrowReader extends VectorizedParquetRecordReader { - private static final Logger LOG = - LoggerFactory.getLogger(VectorizedParquetArrowReader.class); - private ParquetReader reader = null; - private String path; - private long capacity; - private VectorSchemaRoot schemaRoot = null; - - private long lastReadLength = 0; - private int numLoaded = 0; - private int numReaded = 0; - private long totalLength; - private String tmp_dir; - - private ArrowRecordBatch next_batch; - // private ColumnarBatch last_columnar_batch; - - private StructType sourceSchema; - private StructType readDataSchema; - - private Schema schema = null; - - public VectorizedParquetArrowReader(String path, ZoneId convertTz, boolean useOffHeap, - int capacity, StructType sourceSchema, StructType readDataSchema, String tmp_dir) { - super(convertTz, "CORRECTED", "LEGACY", useOffHeap, capacity); - this.capacity = capacity; - this.path = path; - this.tmp_dir = tmp_dir; - - this.sourceSchema = sourceSchema; - this.readDataSchema = readDataSchema; - } - - @Override - public void initBatch(StructType partitionColumns, InternalRow partitionValues) {} - - @Override - public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) - throws IOException, InterruptedException, UnsupportedOperationException { - final ParquetInputSplit parquetInputSplit = toParquetSplit(inputSplit); - final Configuration configuration = ContextUtil.getConfiguration(taskAttemptContext); - initialize(parquetInputSplit, configuration); - } - - public void initialize(ParquetInputSplit inputSplit, Configuration configuration) - throws IOException, InterruptedException, UnsupportedOperationException { - this.totalLength = inputSplit.getLength(); - - int ordinal = 0; - int cur_index = 0; - - int[] column_indices = new int[readDataSchema.size()]; - List targetSchema = Arrays.asList(readDataSchema.names()); - for (String fieldName : sourceSchema.names()) { - if (targetSchema.contains(fieldName)) { - column_indices[cur_index++] = ordinal; - } - ordinal++; - } - - final int[] rowGroupIndices = filterRowGroups(inputSplit, configuration); - String uriPath = this.path; - if (uriPath.contains("hdfs")) { - uriPath = this.path + "?user=root&replication=1"; - } - ParquetInputSplit split = (ParquetInputSplit) inputSplit; - LOG.info("ParquetReader uri path is " + uriPath + ", rowGroupIndices is " - + Arrays.toString(rowGroupIndices) + ", column_indices is " - + Arrays.toString(column_indices)); - this.reader = new ParquetReader(uriPath, split.getStart(), split.getEnd(), - column_indices, capacity, ArrowWritableColumnVector.getAllocator(), tmp_dir); - } - - @Override - public void initialize(String path, List columns) - throws IOException, UnsupportedOperationException {} - - @Override - public boolean nextKeyValue() throws IOException { - return nextBatch(); - } - - @Override - public boolean nextBatch() throws IOException { - next_batch = reader.readNext(); - if (schema == null) { - schema = reader.getSchema(); - } - if (next_batch == null) { - lastReadLength = 0; - return false; - } - lastReadLength = next_batch.getLength(); - numLoaded += lastReadLength; - - return true; - } - - @Override - public Object getCurrentValue() { - if (numReaded == numLoaded) { - return null; - } - numReaded += lastReadLength; - ArrowWritableColumnVector[] columnVectors = - ArrowWritableColumnVector.loadColumns(next_batch.getLength(), schema, next_batch); - next_batch.close(); - return new ColumnarBatch(columnVectors, next_batch.getLength()); - } - - @Override - public void close() throws IOException { - if (reader != null) { - reader.close(); - reader = null; - } - } - - @Override - public float getProgress() { - return (float) (numReaded / totalLength); - } - - private int[] filterRowGroups(ParquetInputSplit parquetInputSplit, - Configuration configuration) throws IOException { - final long[] rowGroupOffsets = parquetInputSplit.getRowGroupOffsets(); - if (rowGroupOffsets != null) { - throw new UnsupportedOperationException(); - } - - final Path path = parquetInputSplit.getPath(); - - final List filteredRowGroups; - final List unfilteredRowGroups; - - try (ParquetFileReader reader = - ParquetFileReader.open(HadoopInputFile.fromPath(path, configuration), - createOptions(parquetInputSplit, configuration))) { - unfilteredRowGroups = reader.getFooter().getBlocks(); - filteredRowGroups = reader.getRowGroups(); - } - - final int[] acc = {0}; - final Map dict = unfilteredRowGroups.stream().collect( - Collectors.toMap(BlockMetaDataWrapper::wrap, b -> acc[0]++)); - return filteredRowGroups.stream() - .map(BlockMetaDataWrapper::wrap) - .map(b -> { - if (!dict.containsKey(b)) { - // This should not happen - throw new IllegalStateException("Unrecognizable filtered row group: " + b); - } - return dict.get(b); - }) - .mapToInt(n -> n) - .toArray(); - } - - private ParquetReadOptions createOptions( - ParquetInputSplit split, Configuration configuration) { - return HadoopReadOptions.builder(configuration) - .withRange(split.getStart(), split.getEnd()) - .build(); - } - - private ParquetInputSplit toParquetSplit(InputSplit split) throws IOException { - if (split instanceof ParquetInputSplit) { - return (ParquetInputSplit) split; - } else { - throw new IllegalArgumentException( - "Invalid split (not a ParquetInputSplit): " + split); - } - } - - // ID for BlockMetaData, to prevent from resulting in mutable BlockMetaData instances - // after being filtered - private static class BlockMetaDataWrapper { - private BlockMetaData m; - - private BlockMetaDataWrapper(BlockMetaData m) { - this.m = m; - } - - public static BlockMetaDataWrapper wrap(BlockMetaData m) { - return new BlockMetaDataWrapper(m); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - BlockMetaDataWrapper that = (BlockMetaDataWrapper) o; - return equals(m, that.m); - } - - private boolean equals(BlockMetaData one, BlockMetaData other) { - return Objects.equals(one.getStartingPos(), other.getStartingPos()); - } - - @Override - public int hashCode() { - return hash(m); - } - - private int hash(BlockMetaData m) { - return Objects.hash(m.getStartingPos()); - } - } -} diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java deleted file mode 100644 index e98dd0acf4a8..000000000000 --- a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intel.oap.datasource.parquet; - -import java.io.IOException; -import java.util.List; - -import org.apache.arrow.dataset.jni.UnsafeRecordBatchSerializer; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.VectorLoader; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.ReadChannel; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.ipc.message.MessageChannelReader; -import org.apache.arrow.vector.ipc.message.MessageResult; -import org.apache.arrow.vector.ipc.message.MessageSerializer; -import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; - -/** Parquet Reader Class. */ -public class ParquetReader implements AutoCloseable { - /** reference to native reader instance. */ - private long nativeInstanceId; - - /** last readed length of a record batch. */ - private long lastReadLength; - - private BufferAllocator allocator; - private ParquetReaderJniWrapper jniWrapper; - - /** - * Create an instance for ParquetReader. - * - * @param path Parquet Reader File Path. - * @param rowGroupIndices An array to indicate which rowGroup to read. - * @param columnIndices An array to indicate which columns to read. - * @param batchSize number of rows expected to be read in one batch. - * @param allocator A BufferAllocator reference. - * @throws IOException throws io exception in case of native failure. - */ - public ParquetReader(String path, int[] rowGroupIndices, int[] columnIndices, - long batchSize, BufferAllocator allocator, String tmp_dir) throws IOException { - this.jniWrapper = new ParquetReaderJniWrapper(tmp_dir); - this.allocator = allocator; - this.nativeInstanceId = jniWrapper.nativeOpenParquetReader(path, batchSize); - jniWrapper.nativeInitParquetReader(nativeInstanceId, columnIndices, rowGroupIndices); - } - - /** - * Create an instance for ParquetReader. - * - * @param path Parquet Reader File Path. - * @param startPos A start pos to indicate which rowGroup to read. - * @param endPos An end pos indicate which rowGroup to read. - * @param columnIndices An array to indicate which columns to read. - * @param batchSize number of rows expected to be read in one batch. - * @param allocator A BufferAllocator reference. - * @throws IOException throws io exception in case of native failure. - */ - public ParquetReader(String path, long startPos, long endPos, int[] columnIndices, - long batchSize, BufferAllocator allocator, String tmp_dir) throws IOException { - this.jniWrapper = new ParquetReaderJniWrapper(tmp_dir); - this.allocator = allocator; - this.nativeInstanceId = jniWrapper.nativeOpenParquetReader(path, batchSize); - jniWrapper.nativeInitParquetReader2( - nativeInstanceId, columnIndices, startPos, endPos); - } - - /** - * Get Arrow Schema from ParquetReader. - * - * @return Schema of parquet file - * @throws IOException throws io exception in case of native failure - */ - public Schema getSchema() throws IOException { - byte[] schemaBytes = jniWrapper.nativeGetSchema(nativeInstanceId); - - try (MessageChannelReader schemaReader = new MessageChannelReader( - new ReadChannel(new ByteArrayReadableSeekableByteChannel(schemaBytes)), - allocator)) { - MessageResult result = schemaReader.readNext(); - if (result == null) { - throw new IOException("Unexpected end of input. Missing schema."); - } - - return MessageSerializer.deserializeSchema(result.getMessage()); - } - } - - /** - * Read Next ArrowRecordBatch from ParquetReader. - * - * @return One ArrowRecordBatch readed from parquet file reader - * @throws IOException throws io exception in case of native failure - */ - public ArrowRecordBatch readNext() throws IOException { - byte[] serializedBatch = - jniWrapper.nativeReadNext(nativeInstanceId); - if (serializedBatch == null) { - return null; - } - ArrowRecordBatch batch = UnsafeRecordBatchSerializer.deserializeUnsafe(allocator, - serializedBatch); - if (batch == null) { - throw new IllegalArgumentException("failed to build record batch"); - } - this.lastReadLength = batch.getLength(); - return batch; - } - - /** - * Read Next ValueVectorList from ParquetReader. - * - * @return Next ValueVectorList readed from parquet file. - * @throws IOException throws io exception in case of native failure - */ - public List readNextVectors(VectorSchemaRoot root) throws IOException { - ArrowRecordBatch batch = readNext(); - if (batch == null) { - return null; - } - VectorLoader loader = new VectorLoader(root); - loader.load(batch); - batch.close(); - return root.getFieldVectors(); - } - - /** - * Get last readed ArrowRecordBatch Length. - * - * @return lastReadLength. - */ - public long lastReadLength() { - return lastReadLength; - } - - @Override - public void close() { - jniWrapper.nativeCloseParquetReader(nativeInstanceId); - } -} diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java deleted file mode 100644 index 9e7ca944d446..000000000000 --- a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intel.oap.datasource.parquet; - -import com.intel.oap.vectorized.JniUtils; - -import java.io.IOException; - -/** Wrapper for Parquet Reader native API. */ -public class ParquetReaderJniWrapper { - /** Construct a Jni Instance. */ - ParquetReaderJniWrapper(String tmp_dir) throws IOException { - JniUtils.getInstance(tmp_dir); - } - - /** - * Construct a parquet file reader over the target file name. - * - * @param path absolute file path of target file - * @param batchSize number of rows of one readed batch - * @return long id of the parquet reader instance - * @throws IOException throws exception in case of any io exception in native codes - */ - public native long nativeOpenParquetReader(String path, long batchSize) - throws IOException; - - /** - * Init a parquet file reader by specifying columns and rowgroups. - * - * @param id parquet reader instance number - * @param columnIndices a array of indexes indicate which columns to be read - * @param rowGroupIndices a array of indexes indicate which row groups to be read - * @throws IOException throws exception in case of any io exception in native codes - */ - public native void nativeInitParquetReader( - long id, int[] columnIndices, int[] rowGroupIndices) throws IOException; - - /** - * Init a parquet file reader by specifying columns and rowgroups. - * - * @param id parquet reader instance number - * @param columnIndices a array of indexes indicate which columns to be read - * @param startPos a start pos to indicate which row group to be read - * @param endPos a end pos to indicate which row group to be read - * @throws IOException throws exception in case of any io exception in native codes - */ - public native void nativeInitParquetReader2( - long id, int[] columnIndices, long startPos, long endPos) throws IOException; - - /** - * Close a parquet file reader. - * - * @param id parquet reader instance number - */ - public native void nativeCloseParquetReader(long id); - - /** - * Read next record batch from parquet file reader. - * - * @param id parquet reader instance number - * @throws IOException throws exception in case of any io exception in native codes - */ - public native byte[] nativeReadNext(long id) throws IOException; - - /** - * Get schema from parquet file reader. - * - * @param id parquet reader instance number - * @throws IOException throws exception in case of any io exception in native codes - */ - public native byte[] nativeGetSchema(long id) throws IOException; -} diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java deleted file mode 100644 index fcdec4814cee..000000000000 --- a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intel.oap.datasource.parquet; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.channels.Channels; -import java.util.List; - -import org.apache.arrow.vector.ipc.WriteChannel; -import org.apache.arrow.vector.ipc.message.ArrowBuffer; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.ipc.message.MessageSerializer; -import org.apache.arrow.vector.types.pojo.Schema; - -import org.apache.arrow.memory.ArrowBuf; - -/** Wrapper for Parquet Writer native API. */ -public class ParquetWriter implements AutoCloseable { - - /** reference to native reader instance. */ - private long nativeInstanceId; - - private ParquetWriterJniWrapper jniWrapper; - - /** - * Open native ParquetWriter Instance. - * - * @param path Parquet File Path to write. - * @param schema arrow schema to initialize Parquet file. - * @throws IOException throws io exception in case of native failure. - */ - public ParquetWriter(String path, Schema schema) - throws IOException { - this.jniWrapper = new ParquetWriterJniWrapper(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), schema); - byte[] schemaBytes = out.toByteArray(); - this.nativeInstanceId = jniWrapper.nativeOpenParquetWriter(path, schemaBytes); - } - - /** - * Write Next ArrowRecordBatch to ParquetWriter. - * - * @param recordBatch next ArrowRecordBatch to write. - * @throws IOException throws exception in case of io issues. - */ - public void writeNext(ArrowRecordBatch recordBatch) throws IOException { - // convert ArrowRecordBatch to buffer List - int numRows = recordBatch.getLength(); - List buffers = recordBatch.getBuffers(); - List buffersLayout = recordBatch.getBuffersLayout(); - - long[] bufAddrs = new long[buffers.size()]; - long[] bufSizes = new long[buffers.size()]; - - int idx = 0; - for (ArrowBuf buf : buffers) { - bufAddrs[idx++] = buf.memoryAddress(); - } - - idx = 0; - for (ArrowBuffer bufLayout : buffersLayout) { - bufSizes[idx++] = bufLayout.getSize(); - } - jniWrapper.nativeWriteNext(nativeInstanceId, numRows, bufAddrs, bufSizes); - } - - @Override - public void close() throws IOException { - jniWrapper.nativeCloseParquetWriter(nativeInstanceId); - } -} diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java deleted file mode 100644 index 5b8e2be14100..000000000000 --- a/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intel.oap.datasource.parquet; - -import com.intel.oap.vectorized.JniUtils; -import java.io.IOException; - -/** Wrapper for Parquet Writer native API. */ -public class ParquetWriterJniWrapper { - - /** Construct a Jni Instance. */ - public ParquetWriterJniWrapper() throws IOException { - JniUtils.getInstance(); - } - - /** - * Construct a parquet file reader over the target file name. - * - * @param path absolute file path of target file - * @param schemaBytes a byte array of Schema serialized output - * @return long id of the parquet writer instance - * @throws IOException throws exception in case of any io exception in native codes - */ - public native long nativeOpenParquetWriter(String path, byte[] schemaBytes); - - /** - * Close a parquet file writer. - * - * @param id parquet writer instance number - */ - public native void nativeCloseParquetWriter(long id); - - /** - * Write next record batch to parquet file writer. - * - * @param id parquet writer instance number - * @param numRows number of Rows in this batch - * @param bufAddrs a array of buffers address of this batch - * @param bufSizes a array of buffers size of this batch - * @throws IOException throws exception in case of any io exception in native codes - */ - public native void nativeWriteNext(long id, int numRows, long[] bufAddrs, long[] bufSizes); -} diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java b/native-sql-engine/core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java deleted file mode 100644 index 620e8f0804d2..000000000000 --- a/native-sql-engine/core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intel.oap.expression; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import org.apache.arrow.memory.ArrowBuf; -import com.google.common.collect.Lists; - -import org.apache.arrow.gandiva.evaluator.*; -import org.apache.arrow.gandiva.exceptions.GandivaException; -import org.apache.arrow.gandiva.expression.*; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ipc.message.ArrowFieldNode; -import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.FloatingPointPrecision; -import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.spark.sql.execution.datasources.v2.arrow.SparkMemoryUtils; - -public class ColumnarArithmeticWithGandiva implements AutoCloseable { - protected static ArrowType int32 = new ArrowType.Int(32, true); - protected static ArrowType float32 = new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); - protected static long make_time = 0; - protected static long evaluate_time = 0; - - @Override - public void close() throws IOException { - } - - public static void columnarAdd( - Schema schema, - List exprs, - int numRowsInBatch, - List inputVectors, - List outputVectors) - throws GandivaException, Exception { - long start = System.nanoTime(); - Projector projector = Projector.make(schema, exprs); - make_time += System.nanoTime() - start; - - start = System.nanoTime(); - List fieldNodes = new ArrayList(); - List inputData = new ArrayList(); - for (int i = 0; i < inputVectors.size(); i++) { - fieldNodes.add(new ArrowFieldNode(numRowsInBatch, inputVectors.get(i).getNullCount())); - inputData.add(inputVectors.get(i).getValidityBuffer()); - inputData.add(inputVectors.get(i).getDataBuffer()); - } - ArrowRecordBatch inputRecordBatch = new ArrowRecordBatch(numRowsInBatch, fieldNodes, inputData); - projector.evaluate(inputRecordBatch, outputVectors); - evaluate_time += System.nanoTime() - start; - - projector.close(); - } - - /** ******* For Test ******** */ - public static void columnarBatchAdd( - Schema schema, - List exprs, - List inputBatchs, - List outputBatchs) - throws GandivaException, Exception { - // start test - for (int i = 0; i < inputBatchs.size(); i++) { - columnarAdd( - schema, - exprs, - inputBatchs.get(i).numRowsInBatch, - inputBatchs.get(i).valueVectors, - outputBatchs.get(i).valueVectors); - } - long make_elapsedTime = TimeUnit.NANOSECONDS.toMillis(make_time); - long evaluate_elapsedTime = TimeUnit.NANOSECONDS.toMillis(evaluate_time); - // long elapsedTime = finish - start; - System.out.println( - "ColumnarAdd process time is: " - + evaluate_elapsedTime - + " ms. And JIT time is " - + make_elapsedTime - + " ms."); - } - - private static ExpressionTree produceAddExpressionTree( - List dataType, List ops, Field result) { - if (dataType.size() < 2) { - System.out.println("dataType size is less than 2"); - return null; - } - - TreeNode last_node = - TreeBuilder.makeFunction( - ops.get(0), - Lists.newArrayList( - TreeBuilder.makeField(dataType.get(0)), TreeBuilder.makeField(dataType.get(1))), - float32); - TreeNode cur_node = last_node; - - for (int i = 2; i < dataType.size(); i++) { - cur_node = - TreeBuilder.makeFunction( - ops.get(i - 1), - Lists.newArrayList(last_node, TreeBuilder.makeField(dataType.get(i))), - float32); - last_node = cur_node; - } - return TreeBuilder.makeExpression(cur_node, result); - } - - public static ArrowBuf arrowBufWithAllValid(int size) { - int bufLen = (size + 7) / 8; - BufferAllocator allocator = SparkMemoryUtils.contextAllocator(); - ArrowBuf buffer = allocator.buffer(bufLen); - for (int i = 0; i < bufLen; i++) { - buffer.writeByte(255); - } - - return buffer; - } - - public static void releaseRecordBatch(ArrowRecordBatch recordBatch) { - // There are 2 references to the buffers - // One in the recordBatch - release that by calling close() - // One in the allocator - release that explicitly - List buffers = recordBatch.getBuffers(); - recordBatch.close(); - for (ArrowBuf buf : buffers) { - buf.getReferenceManager().release(); - } - } - - public static void main(String[] args) { - /** ***** start preparation ****** */ - // prepare expr tree - List dataType = new ArrayList(); - List ops = new ArrayList(); - for (int i = 0; i < 10; i++) { - // dataType.add(Field.nullable("n"+i, int32)); - dataType.add(Field.nullable("n" + i, float32)); - if (i > 0) { - ops.add("add"); - } - } - Schema schema = new Schema(dataType); - // Field result = Field.nullable("result", int32); - Field result = Field.nullable("result", float32); - ExpressionTree expr = produceAddExpressionTree(dataType, ops, result); - List exprs = Lists.newArrayList(expr); - - // set data scale - int numRows = 200 * 1024 * 1024; - int maxRowsInBatch = 16 * 1024; - // int inputFieldSize = 4; - int inputFieldSize = 4; - - // prepara data - List inputArrowColumnarBatchs = new ArrayList(); - List outputArrowColumnarBatchs = new ArrayList(); - - int numRemaining = numRows; - Random rand = new Random(); - while (numRemaining > 0) { - int numRowsInBatch = maxRowsInBatch; - if (numRowsInBatch > numRemaining) { - numRowsInBatch = numRemaining; - } - - List inputVectors = new ArrayList<>(); - List outputVectors = new ArrayList<>(); - for (int i = 0; i < dataType.size(); i++) { - // init input vector - // IntVector inputVector = new IntVector("input_"+i, allocator); - BufferAllocator allocator = SparkMemoryUtils.contextAllocator(); - Float4Vector inputVector = new Float4Vector("input_" + i, allocator); - inputVector.allocateNew(numRowsInBatch); - for (int j = 0; j < numRowsInBatch; j++) { - inputVector.setSafe(j, rand.nextFloat()); - } - inputVectors.add(inputVector); - } - inputArrowColumnarBatchs.add( - new ArrowColumnarBatch(inputVectors, dataType.size(), numRowsInBatch)); - - // init output vector - // IntVector outputVector = new IntVector("result", allocator); - BufferAllocator allocator = SparkMemoryUtils.contextAllocator(); - Float4Vector outputVector = new Float4Vector("result", allocator); - outputVector.allocateNew(numRowsInBatch); - outputVectors.add(outputVector); - outputArrowColumnarBatchs.add(new ArrowColumnarBatch(outputVectors, 1, numRowsInBatch)); - - // fix numRemaining - numRemaining -= numRowsInBatch; - } - - /** ***** end preparation ****** */ - try { - columnarBatchAdd(schema, exprs, inputArrowColumnarBatchs, outputArrowColumnarBatchs); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala deleted file mode 100644 index c2ef1b9a1821..000000000000 --- a/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.v2 - -import com.intel.oap.datasource.VectorizedParquetArrowReader - -import java.net.URI -import java.time.ZoneId - -import org.apache.hadoop.mapreduce._ -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -import org.apache.hadoop.fs.Path - -import org.apache.spark.sql.connector.read.{ - InputPartition, - PartitionReaderFactory, - PartitionReader -} -import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile} -import org.apache.spark.sql.execution.datasources.v2.PartitionedFileReader -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetPartitionReaderFactory -import org.apache.spark.sql.execution.datasources.v2.FilePartitionReader -import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} - -object VectorizedFilePartitionReaderHandler { - def get( - inputPartition: InputPartition, - parquetReaderFactory: ParquetPartitionReaderFactory, - tmpDir: String): FilePartitionReader[ColumnarBatch] = { - val iter: Iterator[PartitionedFileReader[ColumnarBatch]] = - inputPartition.asInstanceOf[FilePartition].files.toIterator.map { file => - val filePath = new Path(new URI(file.filePath)) - val split = - new org.apache.parquet.hadoop.ParquetInputSplit( - filePath, - file.start, - file.start + file.length, - file.length, - Array.empty, - null) - //val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion - /*val convertTz = - if (timestampConversion && !isCreatedByParquetMr) { - Some(DateTimeUtils.getZoneId(conf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) - } else { - None - }*/ - val capacity = 4096 - //partitionReaderFactory.createColumnarReader(inputPartition) - val dataSchema = parquetReaderFactory.dataSchema - val readDataSchema = parquetReaderFactory.readDataSchema - - val conf = parquetReaderFactory.broadcastedConf.value.value - val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) - val hadoopAttemptContext = new TaskAttemptContextImpl(conf, attemptId) - - val vectorizedReader = new VectorizedParquetArrowReader( - split.getPath().toString(), - null, - false, - capacity, - dataSchema, - readDataSchema, - tmpDir) - vectorizedReader.initialize(split, hadoopAttemptContext) - val partitionReader = new PartitionReader[ColumnarBatch] { - override def next(): Boolean = vectorizedReader.nextKeyValue() - override def get(): ColumnarBatch = - vectorizedReader.getCurrentValue.asInstanceOf[ColumnarBatch] - override def close(): Unit = vectorizedReader.close() - } - - PartitionedFileReader(file, partitionReader) - } - new FilePartitionReader[ColumnarBatch](iter) - } -} diff --git a/pom.xml b/pom.xml index 95692576b804..62b67fb5682d 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.intel.oap - native-sql-engine-parent + gazelle-jni-parent 1.2.0-snapshot pom @@ -30,7 +30,7 @@ - native-sql-engine/core + jvm shims @@ -127,7 +127,7 @@ ON ON spark-sql-columnar - OAP Project Spark Columnar Plugin + Gazelle Jni 3.1.1 diff --git a/shims/pom.xml b/shims/pom.xml index aa1ff1029abe..af15adc4b8aa 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -19,7 +19,7 @@ com.intel.oap - native-sql-engine-parent + gazelle-jni-parent 1.2.0-snapshot ../pom.xml diff --git a/native-sql-engine/tools/formatcppcode.sh b/tools/formatcppcode.sh similarity index 100% rename from native-sql-engine/tools/formatcppcode.sh rename to tools/formatcppcode.sh diff --git a/native-sql-engine/tools/gitdiff_AdaptiveSparkPlanExec.patch b/tools/gitdiff_AdaptiveSparkPlanExec.patch similarity index 100% rename from native-sql-engine/tools/gitdiff_AdaptiveSparkPlanExec.patch rename to tools/gitdiff_AdaptiveSparkPlanExec.patch