diff --git a/.clang-format b/.clang-format deleted file mode 100644 index 06453dfbb25b..000000000000 --- a/.clang-format +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -BasedOnStyle: Google -DerivePointerAlignment: false -ColumnLimit: 90 diff --git a/.github/workflows/report_ram_log.yml b/.github/workflows/report_ram_log.yml index 2fef9911cf26..6d7bc0dc5d3d 100644 --- a/.github/workflows/report_ram_log.yml +++ b/.github/workflows/report_ram_log.yml @@ -61,7 +61,7 @@ jobs: mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests - name: Run Maven tests run: | - cd core/ + cd native-sql-engine/core/ mvn test -B -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true env: MAVEN_OPTS: "-Xmx2048m" diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 8f6908d77f57..903af77f07ec 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -56,7 +56,7 @@ jobs: mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - name: Run Maven tests - BHJ run: | - cd core/ + cd native-sql-engine/core/ mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" env: MALLOC_ARENA_MAX: "4" @@ -66,7 +66,7 @@ jobs: ENABLE_TPCH_TESTS: "true" - name: Run Maven tests - SMJ run: | - cd core/ + cd native-sql-engine/core/ mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" env: MALLOC_ARENA_MAX: "4" diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 0cdf3f470f40..83020c70983e 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -51,7 +51,7 @@ jobs: sudo make install - name: Run unit tests run: | - cd cpp/ + cd native-sql-engine/cpp/ mkdir -p build cd build cmake .. -DTESTS=1 diff --git a/.gitignore b/.gitignore index 58c6c9d4a501..fc167d012534 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,48 @@ # IDEA config files .idea/ *.iml +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +.idea/ +target/ +*.iml + +**/target/** + +# logs +*.log + +thirdparty/*.tar* +CMakeFiles/ +CMakeCache.txt +CTestTestfile.cmake +Makefile +cmake_install.cmake +build/ +*-build/ +Testing/ +cmake-build-debug/ +cmake-build-release/ + +# Editor temporary/working/backup files # +.#* +*\#*\# +[#]*# +*~ +*$ +*.bak +*flymake* +*.kdev4 +*.swp +scalastyle-output.xml diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2d9d38326436..000000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "dep/arrow-data-source"] - path = dep/arrow-data-source - url = https://github.com/oap-project/arrow-data-source diff --git a/arrow-data-source/.github/PULL_REQUEST_TEMPLATE b/arrow-data-source/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 000000000000..989e95ccd013 --- /dev/null +++ b/arrow-data-source/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,12 @@ +## What changes were proposed in this pull request? + +(Please fill in changes proposed in this fix) + + +## How was this patch tested? + +(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) + + +(If this patch involves UI changes, please attach a screenshot; otherwise, remove this) + diff --git a/arrow-data-source/.github/workflows/dev_cron.yml b/arrow-data-source/.github/workflows/dev_cron.yml new file mode 100644 index 000000000000..ab2840ebd896 --- /dev/null +++ b/arrow-data-source/.github/workflows/dev_cron.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Dev PR + +on: + + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + process: + name: Process + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Comment Issues link + if: | + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || + github.event.action == 'edited') + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_cron/issues_link.js`); + script({github, context}); + + - name: Check title + if: | + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || + github.event.action == 'edited') + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_cron/title_check.js`); + script({github, context}); + diff --git a/arrow-data-source/.github/workflows/dev_cron/issues_link.js b/arrow-data-source/.github/workflows/dev_cron/issues_link.js new file mode 100644 index 000000000000..1794b2566b97 --- /dev/null +++ b/arrow-data-source/.github/workflows/dev_cron/issues_link.js @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +function detectISSUESID(title) { + if (!title) { + return null; + } + const matched = /^\[NSE-\d+\]/.exec(title); + if (!matched) { + return null; + } + const issues_number = matched[0].replace(/[^0-9]/ig,""); + return issues_number; +} + +async function haveComment(github, context, pullRequestNumber, body) { + const options = { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + page: 1 + }; + while (true) { + const response = await github.issues.listComments(options); + if (response.data.some(comment => comment.body === body)) { + return true; + } + if (!/;\s*rel="next"/.test(response.headers.link || "")) { + break; + } + options.page++; + } + return false; +} + +async function commentISSUESURL(github, context, pullRequestNumber, issuesID) { + const issuesURL = `https://github.com/oap-project/native-sql-engine/issues/${issuesID}`; + if (await haveComment(github, context, pullRequestNumber, issuesURL)) { + return; + } + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: issuesURL + }); +} + +module.exports = async ({github, context}) => { + const pullRequestNumber = context.payload.number; + const title = context.payload.pull_request.title; + const issuesID = detectISSUESID(title); + if (issuesID) { + await commentISSUESURL(github, context, pullRequestNumber, issuesID); + } +}; diff --git a/arrow-data-source/.github/workflows/dev_cron/title_check.js b/arrow-data-source/.github/workflows/dev_cron/title_check.js new file mode 100644 index 000000000000..9380a434445e --- /dev/null +++ b/arrow-data-source/.github/workflows/dev_cron/title_check.js @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const fs = require("fs"); + +function haveISSUESID(title) { + if (!title) { + return false; + } + return /^\[NSE-\d+\]/.test(title); +} + +async function commentOpenISSUESIssue(github, context, pullRequestNumber) { + const {data: comments} = await github.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 1 + }); + if (comments.length > 0) { + return; + } + const commentPath = ".github/workflows/dev_cron/title_check.md"; + const comment = fs.readFileSync(commentPath).toString(); + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: comment + }); +} + +module.exports = async ({github, context}) => { + const pullRequestNumber = context.payload.number; + const title = context.payload.pull_request.title; + if (!haveISSUESID(title)) { + await commentOpenISSUESIssue(github, context, pullRequestNumber); + } +}; diff --git a/arrow-data-source/.github/workflows/dev_cron/title_check.md b/arrow-data-source/.github/workflows/dev_cron/title_check.md new file mode 100644 index 000000000000..928a8fc4b8f9 --- /dev/null +++ b/arrow-data-source/.github/workflows/dev_cron/title_check.md @@ -0,0 +1,33 @@ + + +Thanks for opening a pull request! + +Could you open an issue for this pull request on Github Issues? + +https://github.com/oap-project/native-sql-engine/issues + +Then could you also rename ***commit message*** and ***pull request title*** in the following format? + + [NSE-${ISSUES_ID}] ${detailed message} + +See also: + + * [Other pull requests](https://github.com/oap-project/native-sql-engine/pulls/) + diff --git a/arrow-data-source/.github/workflows/report_ram_log.yml b/arrow-data-source/.github/workflows/report_ram_log.yml new file mode 100644 index 000000000000..2fef9911cf26 --- /dev/null +++ b/arrow-data-source/.github/workflows/report_ram_log.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Report RAM Log + +on: + workflow_run: + workflows: ["Native SQL Engine TPC-H Suite"] + types: + - completed + +jobs: + comment-on-pr: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Download log + uses: dawidd6/action-download-artifact@v2 + with: + workflow: tpch.yml + run_id: ${{ github.event.workflow_run.id }} + name: comment_content + path: /tmp/ + - name: Download previous event payload + uses: dawidd6/action-download-artifact@v2 + with: + workflow: tpch.yml + run_id: ${{ github.event.workflow_run.id }} + name: pr_event + path: /tmp/ + - name: Install OAP optimized Arrow + run: | + cd /tmp + git clone -b arrow-3.0.0-oap https://github.com/oap-project/arrow.git + cd arrow/java + mvn clean install -B -P arrow-jni -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip + - name: Install Arrow datasource + run: | + cd /tmp + git clone https://github.com/oap-project/arrow-data-source.git + cd arrow-data-source + mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests + - name: Run Maven tests + run: | + cd core/ + mvn test -B -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true + env: + MAVEN_OPTS: "-Xmx2048m" + COMMENT_CONTENT_PATH: "/tmp/comment.md" + PREVIOUS_EVENT_PATH: "/tmp/event.json" + GITHUB_TOKEN: ${{ github.token }} + ENABLE_TPCH_TESTS: "true" + diff --git a/arrow-data-source/.github/workflows/tpch.yml b/arrow-data-source/.github/workflows/tpch.yml new file mode 100644 index 000000000000..8f6908d77f57 --- /dev/null +++ b/arrow-data-source/.github/workflows/tpch.yml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Native SQL Engine TPC-H Suite + +on: + pull_request + +jobs: + ram-usage-test: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'No RAM Report') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - uses: iterative/setup-cml@v1 + with: + version: latest + - run: sudo swapoff -a + - run: free + - run: sudo apt-get update + - run: sudo apt-get install cmake + - run: sudo apt-get install libboost-all-dev + - name: Install OAP optimized Arrow + run: | + cd /tmp + git clone https://github.com/oap-project/arrow.git + cd arrow && git checkout arrow-3.0.0-oap && cd cpp + mkdir build && cd build + cmake .. -DARROW_JNI=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_FILESYSTEM=ON -DARROW_WITH_SNAPPY=ON -DARROW_JSON=ON -DARROW_DATASET=ON -DARROW_WITH_LZ4=ON -DARROW_JEMALLOC=OFF && make -j2 + sudo make install + cd ../../java + mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip + - name: Install Arrow datasource + run: | + cd /tmp + git clone https://github.com/oap-project/arrow-data-source.git + cd arrow-data-source + mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + - name: Run Maven tests - BHJ + run: | + cd core/ + mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" + env: + MALLOC_ARENA_MAX: "4" + MAVEN_OPTS: "-Xmx1G" + COMMENT_TEXT_OUTPUT_PATH: "/tmp/comment_text_1.txt" + COMMENT_IMAGE_OUTPUT_PATH: "/tmp/comment_image_1.png" + ENABLE_TPCH_TESTS: "true" + - name: Run Maven tests - SMJ + run: | + cd core/ + mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" + env: + MALLOC_ARENA_MAX: "4" + MAVEN_OPTS: "-Xmx1G" + COMMENT_TEXT_OUTPUT_PATH: "/tmp/comment_text_2.txt" + COMMENT_IMAGE_OUTPUT_PATH: "/tmp/comment_image_2.png" + ENABLE_TPCH_TESTS: "true" + - run: | + cml-publish /tmp/comment_image_1.png --md > /tmp/comment.md + cml-publish /tmp/comment_image_2.png --md >> /tmp/comment.md + - run: echo "::set-output name=event_path::${GITHUB_EVENT_PATH}" + id: output-envs + - uses: actions/upload-artifact@v2 + with: + name: comment_content + path: /tmp/comment.md + - uses: actions/upload-artifact@v2 + with: + name: pr_event + path: ${{steps.output-envs.outputs.event_path}} + diff --git a/arrow-data-source/.github/workflows/unittests.yml b/arrow-data-source/.github/workflows/unittests.yml new file mode 100644 index 000000000000..0cdf3f470f40 --- /dev/null +++ b/arrow-data-source/.github/workflows/unittests.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Native SQL Engine Unit Tests Suite + +on: + pull_request + +jobs: + cpp-unit-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - run: sudo swapoff -a + - run: free + - run: sudo apt-get update + - run: sudo apt-get install cmake + - run: sudo apt-get install libboost-all-dev + - name: Install Googletest + run: | + sudo apt-get install libgtest-dev + cd /usr/src/gtest + sudo cmake CMakeLists.txt -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local + sudo make + sudo apt-get install google-mock + - name: Install OAP optimized Arrow (C++ libs) + run: | + cd /tmp + git clone https://github.com/oap-project/arrow.git + cd arrow && git checkout arrow-3.0.0-oap && cd cpp + mkdir build && cd build + cmake .. -DARROW_JNI=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_FILESYSTEM=ON -DARROW_WITH_SNAPPY=ON -DARROW_JSON=ON -DARROW_DATASET=ON -DARROW_WITH_LZ4=ON -DGTEST_ROOT=/usr/src/gtest && make -j2 + sudo make install + - name: Run unit tests + run: | + cd cpp/ + mkdir -p build + cd build + cmake .. -DTESTS=1 + make + cd src + ctest -R + diff --git a/.travis.yml b/arrow-data-source/.travis.yml similarity index 100% rename from .travis.yml rename to arrow-data-source/.travis.yml diff --git a/CHANGELOG.md b/arrow-data-source/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to arrow-data-source/CHANGELOG.md diff --git a/LICENSE.txt b/arrow-data-source/LICENSE.txt similarity index 100% rename from LICENSE.txt rename to arrow-data-source/LICENSE.txt diff --git a/arrow-data-source/README.md b/arrow-data-source/README.md new file mode 100644 index 000000000000..0409f46c9e07 --- /dev/null +++ b/arrow-data-source/README.md @@ -0,0 +1,235 @@ +# Arrow Data Source + +A Spark DataSource implementation for reading files into Arrow compatible columnar vectors. + +## Note + +The development of this library is still in progress. As a result some of the functionality may not be constantly stable for being used in production environments that have not been fully considered due to the limited testing capabilities so far. + +## Online Documentation + +You can find the all the Native SQL Engine documents on the [project web page](https://oap-project.github.io/arrow-data-source/). + +## Build + +### Prerequisite + +There are some requirements before you build the project. +Please make sure you have already installed the software in your system. + +1. gcc 9.3 or higher version +2. java8 OpenJDK -> yum install java-1.8.0-openjdk +3. cmake 3.2 or higher version +4. maven 3.1.1 or higher version +5. Hadoop 2.7.5 or higher version +6. Spark 3.0.0 or higher version +7. Intel Optimized Arrow 3.0.0 + +### Building by Conda + +If you already have a working Hadoop Spark Cluster, we provide a Conda package which will automatically install dependencies needed by OAP, you can refer to [OAP-Installation-Guide](./docs/OAP-Installation-Guide.md) for more information. Once finished [OAP-Installation-Guide](./docs/OAP-Installation-Guide.md), you can find built `spark-arrow-datasource-standard--jar-with-dependencies.jar` under `$HOME/miniconda2/envs/oapenv/oap_jars`. +Then you can just skip steps below and jump to [Get Started](#get-started). + +### cmake installation + +If you are facing some trouble when installing cmake, please follow below steps to install cmake. + +``` +// installing cmake 3.2 +sudo yum install cmake3 + +// If you have an existing cmake, you can use below command to set it as an option within alternatives command +sudo alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake 10 --slave /usr/local/bin/ctest ctest /usr/bin/ctest --slave /usr/local/bin/cpack cpack /usr/bin/cpack --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake --family cmake + +// Set cmake3 as an option within alternatives command +sudo alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 --slave /usr/local/bin/ctest ctest /usr/bin/ctest3 --slave /usr/local/bin/cpack cpack /usr/bin/cpack3 --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 --family cmake + +// Use alternatives to choose cmake version +sudo alternatives --config cmake +``` + +### maven installation + +If you are facing some trouble when installing maven, please follow below steps to install maven + +``` +// installing maven 3.6.3 +Go to https://maven.apache.org/download.cgi and download the specific version of maven + +// Below command use maven 3.6.3 as an example +wget https://ftp.wayne.edu/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz +tar xzf apache-maven-3.6.3-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.6.3/ /usr/local/maven/ + +// Set maven 3.6.3 as an option within alternatives command +sudo alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.6.3/bin/mvn 1 + +// Use alternatives to choose mvn version +sudo alternatives --config mvn +``` + +### Hadoop Native Library(Default) + +Please make sure you have set up Hadoop directory properly with Hadoop Native Libraries +By default, Apache Arrow would scan `$HADOOP_HOME` and find the native Hadoop library `libhdfs.so`(under `$HADOOP_HOME/lib/native` directory) to be used for Hadoop client. + +You can also use `ARROW_LIBHDFS_DIR` to configure the location of `libhdfs.so` if it is installed in other directory than `$HADOOP_HOME/lib/native` + +If your SPARK and HADOOP are separated in different nodes, please find `libhdfs.so` in your Hadoop cluster and copy it to SPARK cluster, then use one of the above methods to set it properly. + +For more information, please check +Arrow HDFS interface [documentation](https://github.com/apache/arrow/blob/master/cpp/apidoc/HDFS.md) +Hadoop Native Library, please read the official Hadoop website [documentation](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/NativeLibraries.html) + +### Use libhdfs3 library for better performance(Optional) + +For better performance ArrowDataSource reads HDFS files using the third-party library `libhdfs3`. The library must be pre-installed on machines Spark Executor nodes are running on. + +To install the library, use of [Conda](https://docs.conda.io/en/latest/) is recommended. + +``` +// installing libhdfs3 +conda install -c conda-forge libhdfs3 + +// check the installed library file +ll ~/miniconda/envs/$(YOUR_ENV_NAME)/lib/libhdfs3.so +``` + +To set up libhdfs3, there are two different ways: +Option1: Overwrite the soft link for libhdfs.so +To install libhdfs3.so, you have to create a soft link for libhdfs.so in your Hadoop directory(`$HADOOP_HOME/lib/native` by default). + +``` +ln -f -s libhdfs3.so libhdfs.so +``` + +Option2: +Add env variable to the system +``` +export ARROW_LIBHDFS3_DIR="PATH_TO_LIBHDFS3_DIR/" +``` + +Add following Spark configuration options before running the DataSource to make the library to be recognized: +* `spark.executorEnv.ARROW_LIBHDFS3_DIR = "PATH_TO_LIBHDFS3_DIR/"` +* `spark.executorEnv.LD_LIBRARY_PATH = "PATH_TO_LIBHDFS3_DEPENDENCIES_DIR/"` + +Please notes: If you choose to use libhdfs3.so, there are some other dependency libraries you have to installed such as libprotobuf or libcrypto. + +### Build and install Intel® Optimized Arrow with Datasets Java API +You have to use a customized Arrow to support for our datasets Java API. + +``` +// build arrow-cpp +git clone -b https://github.com/Intel-bigdata/arrow.git +cd arrow/cpp +mkdir build +cd build +cmake -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JNI=ON -DARROW_DATASET=ON -DARROW_WITH_PROTOBUF=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_LZ4=ON -DARROW_FILESYSTEM=ON -DARROW_JSON=ON .. +make + +// build and install arrow jvm library +cd ../../java +mvn clean install -P arrow-jni -am -Darrow.cpp.build.dir=$PATH_TO_ARROW_SOURCE_CODE/arrow/cpp/build/release +``` + +### Build Arrow Data Source Library + +``` +// Download Arrow Data Source Code +git clone -b https://github.com/oap-project/arrow-data-source.git + +// Go to the directory +cd arrow-data-source + +// build +mvn clean -DskipTests package + +// check built jar library +readlink -f standard/target/spark-arrow-datasource-standard--jar-with-dependencies.jar +``` + +### Download Spark 3.0.0 + +Currently ArrowDataSource works on the Spark 3.0.0 version. + +``` +wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz +tar -xf ./spark-3.0.0-bin-hadoop2.7.tgz +export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7 +``` + +If you are new to Apache Spark, please go though [Spark's official deploying guide](https://spark.apache.org/docs/latest/cluster-overview.html) before getting started with ArrowDataSource. + +## Get started +### Add extra class pathes to Spark + +To enable ArrowDataSource, the previous built jar `spark-arrow-datasource-standard--jar-with-dependencies.jar` should be added to Spark configuration. Typically the options are: + +* `spark.driver.extraClassPath` : Set to load jar file to driver. +* `spark.executor.extraClassPath` : Set to load jar file to executor. +* `jars` : Set to copy jar file to the executors when using yarn cluster mode. +* `spark.executorEnv.ARROW_LIBHDFS3_DIR` : Optional if you are using a custom libhdfs3.so. +* `spark.executorEnv.LD_LIBRARY_PATH` : Optional if you are using a custom libhdfs3.so. + +For Spark Standalone Mode, please set the above value as relative path to the jar file. +For Spark Yarn Cluster Mode, please set the above value as absolute path to the jar file. + +Example to run Spark Shell with ArrowDataSource jar file +``` +${SPARK_HOME}/bin/spark-shell \ + --verbose \ + --master yarn \ + --driver-memory 10G \ + --conf spark.driver.extraClassPath=$PATH_TO_DATASOURCE_DIR/spark-arrow-datasource-standard--jar-with-dependencies.jar \ + --conf spark.executor.extraClassPath=$PATH_TO_DATASOURCE_DIR/spark-arrow-datasource-standard--jar-with-dependencies.jar \ + --conf spark.driver.cores=1 \ + --conf spark.executor.instances=12 \ + --conf spark.executor.cores=6 \ + --conf spark.executor.memory=20G \ + --conf spark.memory.offHeap.size=80G \ + --conf spark.task.cpus=1 \ + --conf spark.locality.wait=0s \ + --conf spark.sql.shuffle.partitions=72 \ + --conf spark.executorEnv.ARROW_LIBHDFS3_DIR="$PATH_TO_LIBHDFS3_DIR/" \ + --conf spark.executorEnv.LD_LIBRARY_PATH="$PATH_TO_LIBHDFS3_DEPENDENCIES_DIR" +``` + +For more information about these options, please read the official Spark [documentation](https://spark.apache.org/docs/latest/configuration.html#runtime-environment). + +### Run a query with ArrowDataSource (Scala) + +```scala +val path = "${PATH_TO_YOUR_PARQUET_FILE}" +val df = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .format("arrow") + .load(path) +df.createOrReplaceTempView("my_temp_view") +spark.sql("SELECT * FROM my_temp_view LIMIT 10").show(10) +``` +### To validate if ArrowDataSource works properly + +To validate if ArrowDataSource works, you can go to the DAG to check if ArrowScan has been used from the above example query. + +![Image of ArrowDataSource Validation](./docs/image/arrowdatasource_validation.png) + + +## Work together with ParquetDataSource (experimental) + +We provide a customized replacement of Spark's built-in ParquetFileFormat. By so users don't have +to change existing Parquet-based SQL/code and will be able to read Arrow data from Parquet directly. +More importantly, sometimes the feature could be extremely helpful to make ArrowDataSource work correctly +with some 3rd-party storage tools (e.g. [Delta Lake](https://github.com/delta-io/delta)) that are built on top of ParquetDataSource. + +To replace built-in ParquetDataSource, the only thing has to be done is to place compiled jar `spark-arrow-datasource-parquet-.jar` into +Spark's library folder. + +If you'd like to verify that ParquetDataSource is successfully overwritten by the jar, run following code +before executing SQL job: +``` +ServiceLoaderUtil.ensureParquetFileFormatOverwritten(); +``` + +Note the whole feature is currently **experimental** and only DataSource v1 is supported. V2 support is being planned. diff --git a/arrow-data-source/common/pom.xml b/arrow-data-source/common/pom.xml new file mode 100644 index 000000000000..2a752774a847 --- /dev/null +++ b/arrow-data-source/common/pom.xml @@ -0,0 +1,77 @@ + + + + spark-arrow-datasource + com.intel.oap + 1.1.0 + + 4.0.0 + + spark-arrow-datasource-common + + ${project.basedir}/src/main/scala + ${project.basedir}/src/test/scala + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 1.8 + 1.8 + + + + compile + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-src-1 + generate-sources + + add-source + + + + ${project.basedir}/src/main/java + + + + + + + + + + + org.scala-tools + maven-scala-plugin + + ${scala.version} + + + + + diff --git a/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java new file mode 100644 index 000000000000..45888521e250 --- /dev/null +++ b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryConsumer.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow; + +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.spark.memory.MemoryConsumer; +import org.apache.spark.memory.MemoryMode; +import org.apache.spark.memory.TaskMemoryManager; + +import java.io.IOException; + +public class NativeSQLMemoryConsumer extends MemoryConsumer { + + private final Spiller spiller; + + public NativeSQLMemoryConsumer(TaskMemoryManager taskMemoryManager, Spiller spiller) { + super(taskMemoryManager, taskMemoryManager.pageSizeBytes(), MemoryMode.OFF_HEAP); + this.spiller = spiller; + } + + @Override + public long spill(long size, MemoryConsumer trigger) throws IOException { + return spiller.spill(size, trigger); + } + + + public void acquire(long size) { + if (size == 0) { + return; + } + long granted = acquireMemory(size); + if (granted < size) { + freeMemory(granted); + throw new OutOfMemoryException("Not enough spark off-heap execution memory. " + + "Acquired: " + size + ", granted: " + granted + ". " + + "Try tweaking config option spark.memory.offHeap.size to " + + "get larger space to run this application. "); + } + } + + public void free(long size) { + freeMemory(size); + } +} diff --git a/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java new file mode 100644 index 000000000000..c26054b3c1c9 --- /dev/null +++ b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/NativeSQLMemoryMetrics.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow; + +import java.util.concurrent.atomic.AtomicLong; + +public final class NativeSQLMemoryMetrics { + private final AtomicLong peak = new AtomicLong(0L); + private final AtomicLong total = new AtomicLong(0L); + + public void inc(long bytes) { + final long total = this.total.addAndGet(bytes); + long prev_peak; + do { + prev_peak = this.peak.get(); + if (total <= prev_peak) { + break; + } + } while (!this.peak.compareAndSet(prev_peak, total)); + } + + public long peak() { + return peak.get(); + } + + public long total() { + return total.get(); + } +} diff --git a/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java new file mode 100644 index 000000000000..abdabcc26dc3 --- /dev/null +++ b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedAllocationListener.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow; + +import org.apache.arrow.memory.AllocationListener; + +public class SparkManagedAllocationListener implements AllocationListener { + + private final NativeSQLMemoryConsumer consumer; + private final NativeSQLMemoryMetrics metrics; + + public SparkManagedAllocationListener(NativeSQLMemoryConsumer consumer, NativeSQLMemoryMetrics metrics) { + this.consumer = consumer; + this.metrics = metrics; + } + + @Override + public void onPreAllocation(long size) { + consumer.acquire(size); + metrics.inc(size); + } + + @Override + public void onRelease(long size) { + consumer.free(size); + metrics.inc(-size); + } +} diff --git a/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java new file mode 100644 index 000000000000..47c1a9890770 --- /dev/null +++ b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/SparkManagedReservationListener.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow; + +import org.apache.arrow.memory.ReservationListener; + +/** + * Reserve Spark managed memory. + */ +public class SparkManagedReservationListener implements ReservationListener { + + private final NativeSQLMemoryConsumer consumer; + private final NativeSQLMemoryMetrics metrics; + + public SparkManagedReservationListener(NativeSQLMemoryConsumer consumer, NativeSQLMemoryMetrics metrics) { + this.consumer = consumer; + this.metrics = metrics; + } + + @Override + public void reserve(long size) { + consumer.acquire(size); + metrics.inc(size); + } + + @Override + public void unreserve(long size) { + consumer.free(size); + metrics.inc(-size); + } +} diff --git a/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java new file mode 100644 index 000000000000..7702fbc742c0 --- /dev/null +++ b/arrow-data-source/common/src/main/java/com/intel/oap/spark/sql/execution/datasources/v2/arrow/Spiller.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow; + +import org.apache.spark.memory.MemoryConsumer; + +public interface Spiller { + Spiller NO_OP = new Spiller() { + @Override + public long spill(long size, MemoryConsumer trigger) { + return 0L; + } + }; + + long spill(long size, MemoryConsumer trigger); +} diff --git a/core/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java b/arrow-data-source/common/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java similarity index 99% rename from core/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java rename to arrow-data-source/common/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java index ad0ad8aebe74..b89e74fb6efe 100644 --- a/core/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java +++ b/arrow-data-source/common/src/main/java/com/intel/oap/vectorized/ArrowWritableColumnVector.java @@ -17,7 +17,6 @@ package com.intel.oap.vectorized; -import org.apache.arrow.memory.ArrowBuf; import java.lang.*; import java.math.BigDecimal; import java.nio.ByteBuffer; diff --git a/arrow-data-source/common/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala b/arrow-data-source/common/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala new file mode 100644 index 000000000000..bc11a47e35bd --- /dev/null +++ b/arrow-data-source/common/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkMemoryUtils.scala @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2.arrow + +import java.util +import java.util.UUID + +import scala.collection.JavaConverters._ + +import com.intel.oap.spark.sql.execution.datasources.v2.arrow._ +import org.apache.arrow.dataset.jni.NativeMemoryPool +import org.apache.arrow.memory.BufferAllocator + +import org.apache.spark.TaskContext +import org.apache.spark.memory.TaskMemoryManager +import org.apache.spark.util.TaskCompletionListener + +object SparkMemoryUtils { + + class TaskMemoryResources { + if (!inSparkTask()) { + throw new IllegalStateException("Creating TaskMemoryResources instance out of Spark task") + } + + val sharedMetrics = new NativeSQLMemoryMetrics() + val defaultAllocator: BufferAllocator = { + + val globalAlloc = globalAllocator() + val al = new SparkManagedAllocationListener( + new NativeSQLMemoryConsumer(getTaskMemoryManager(), Spiller.NO_OP), + sharedMetrics) + val parent = globalAlloc + parent.newChildAllocator("Spark Managed Allocator - " + + UUID.randomUUID().toString, al, 0, parent.getLimit) + } + + val defaultMemoryPool: NativeMemoryPool = { + val rl = new SparkManagedReservationListener( + new NativeSQLMemoryConsumer(getTaskMemoryManager(), Spiller.NO_OP), + sharedMetrics) + NativeMemoryPool.createListenable(rl) + } + + private val allocators = new util.ArrayList[BufferAllocator]() + allocators.add(defaultAllocator) + + private val memoryPools = new util.ArrayList[NativeMemoryPool]() + memoryPools.add(defaultMemoryPool) + + def createSpillableMemoryPool(spiller: Spiller): NativeMemoryPool = { + val rl = new SparkManagedReservationListener( + new NativeSQLMemoryConsumer(getTaskMemoryManager(), spiller), + sharedMetrics) + val pool = NativeMemoryPool.createListenable(rl) + memoryPools.add(pool) + pool + } + + def createSpillableAllocator(spiller: Spiller): BufferAllocator = { + val al = new SparkManagedAllocationListener( + new NativeSQLMemoryConsumer(getTaskMemoryManager(), spiller), + sharedMetrics) + val parent = globalAllocator() + val alloc = parent.newChildAllocator("Spark Managed Allocator - " + + UUID.randomUUID().toString, al, 0, parent.getLimit).asInstanceOf[BufferAllocator] + allocators.add(alloc) + alloc + } + + private def close(allocator: BufferAllocator): Unit = { + allocator.getChildAllocators.forEach(close(_)) + allocator.close() + } + + /** + * Close the allocator quietly without having any OOM errors thrown. We rely on Spark's memory + * management system to detect possible memory leaks after the task get successfully down. Any + * leak shown right here is possibly not actual because buffers may be cleaned up after + * this check code is executed. Having said that developers should manage to make sure + * the specific clean up logic of operators is registered at last of the program which means + * it will be executed earlier. + * + * @see org.apache.spark.executor.Executor.TaskRunner#run() + */ + private def softClose(allocator: BufferAllocator): Unit = { + // move to leaked list + leakedAllocators.add(allocator) + } + + private def close(pool: NativeMemoryPool): Unit = { + pool.close() + } + + private def softClose(pool: NativeMemoryPool): Unit = { + // move to leaked list + leakedMemoryPools.add(pool) + } + + def release(): Unit = { + for (allocator <- allocators.asScala) { + val allocated = allocator.getAllocatedMemory + if (allocated == 0L) { + close(allocator) + } else { + softClose(allocator) + } + } + for (pool <- memoryPools.asScala) { + val allocated = pool.getBytesAllocated + if (allocated == 0L) { + close(pool) + } else { + softClose(pool) + } + } + } + } + + private val taskToResourcesMap = new java.util.IdentityHashMap[TaskContext, TaskMemoryResources]() + + private val leakedAllocators = new java.util.Vector[BufferAllocator]() + private val leakedMemoryPools = new java.util.Vector[NativeMemoryPool]() + + private def getLocalTaskContext: TaskContext = TaskContext.get() + + private def getTaskMemoryManager(): TaskMemoryManager = { + getLocalTaskContext.taskMemoryManager() + } + + private def inSparkTask(): Boolean = { + getLocalTaskContext != null + } + + def addLeakSafeTaskCompletionListener[U](f: TaskContext => U): TaskContext = { + if (!inSparkTask()) { + throw new IllegalStateException("Not in a Spark task") + } + getTaskMemoryResources() // initialize cleaners + getLocalTaskContext.addTaskCompletionListener(f) + } + + def getTaskMemoryResources(): TaskMemoryResources = { + if (!inSparkTask()) { + throw new IllegalStateException("Not in a Spark task") + } + val tc = getLocalTaskContext + taskToResourcesMap.synchronized { + + if (!taskToResourcesMap.containsKey(tc)) { + val resources = new TaskMemoryResources + getLocalTaskContext.addTaskCompletionListener( + new TaskCompletionListener { + override def onTaskCompletion(context: TaskContext): Unit = { + taskToResourcesMap.synchronized { + val resources = taskToResourcesMap.remove(context) + resources.release() + context.taskMetrics().incPeakExecutionMemory(resources.sharedMetrics.peak()) + } + } + }) + taskToResourcesMap.put(tc, resources) + } + + return taskToResourcesMap.get(tc) + } + } + + def globalAllocator(): BufferAllocator = { + org.apache.spark.sql.util.ArrowUtils.rootAllocator + } + + def globalMemoryPool(): NativeMemoryPool = { + NativeMemoryPool.getDefault + } + + def createSpillableAllocator(spiller: Spiller): BufferAllocator = { + if (!inSparkTask()) { + throw new IllegalStateException("Spiller must be used in a Spark task") + } + getTaskMemoryResources().createSpillableAllocator(spiller) + } + + def createSpillableMemoryPool(spiller: Spiller): NativeMemoryPool = { + if (!inSparkTask()) { + throw new IllegalStateException("Spiller must be used in a Spark task") + } + getTaskMemoryResources().createSpillableMemoryPool(spiller) + } + + def contextAllocator(): BufferAllocator = { + val globalAlloc = globalAllocator() + if (!inSparkTask()) { + return globalAlloc + } + getTaskMemoryResources().defaultAllocator + } + + def contextMemoryPool(): NativeMemoryPool = { + if (!inSparkTask()) { + return globalMemoryPool() + } + getTaskMemoryResources().defaultMemoryPool + } + + def getLeakedAllocators(): List[BufferAllocator] = { + val list = new util.ArrayList[BufferAllocator](leakedAllocators) + list.asScala.toList + } + + def getLeakedMemoryPools(): List[NativeMemoryPool] = { + val list = new util.ArrayList[NativeMemoryPool](leakedMemoryPools) + list.asScala.toList + } +} diff --git a/arrow-data-source/common/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala b/arrow-data-source/common/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala new file mode 100644 index 000000000000..d403a62f1b40 --- /dev/null +++ b/arrow-data-source/common/src/main/scala/org/apache/spark/sql/execution/datasources/v2/arrow/SparkSchemaUtils.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2.arrow + +import org.apache.arrow.vector.types.pojo.Schema + +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.ArrowUtils + +object SparkSchemaUtils { + + def fromArrowSchema(schema: Schema): StructType = { + ArrowUtils.fromArrowSchema(schema) + } + + def toArrowSchema(schema: StructType, timeZoneId: String): Schema = { + ArrowUtils.toArrowSchema(schema, timeZoneId) + } +} diff --git a/docs/ApacheArrowInstallation.md b/arrow-data-source/docs/ApacheArrowInstallation.md similarity index 100% rename from docs/ApacheArrowInstallation.md rename to arrow-data-source/docs/ApacheArrowInstallation.md diff --git a/docs/Configuration.md b/arrow-data-source/docs/Configuration.md similarity index 100% rename from docs/Configuration.md rename to arrow-data-source/docs/Configuration.md diff --git a/docs/Installation.md b/arrow-data-source/docs/Installation.md similarity index 100% rename from docs/Installation.md rename to arrow-data-source/docs/Installation.md diff --git a/docs/InstallationNotes.md b/arrow-data-source/docs/InstallationNotes.md similarity index 100% rename from docs/InstallationNotes.md rename to arrow-data-source/docs/InstallationNotes.md diff --git a/docs/OAP-Developer-Guide.md b/arrow-data-source/docs/OAP-Developer-Guide.md similarity index 100% rename from docs/OAP-Developer-Guide.md rename to arrow-data-source/docs/OAP-Developer-Guide.md diff --git a/docs/OAP-Installation-Guide.md b/arrow-data-source/docs/OAP-Installation-Guide.md similarity index 100% rename from docs/OAP-Installation-Guide.md rename to arrow-data-source/docs/OAP-Installation-Guide.md diff --git a/docs/Prerequisite.md b/arrow-data-source/docs/Prerequisite.md similarity index 100% rename from docs/Prerequisite.md rename to arrow-data-source/docs/Prerequisite.md diff --git a/docs/SparkInstallation.md b/arrow-data-source/docs/SparkInstallation.md similarity index 100% rename from docs/SparkInstallation.md rename to arrow-data-source/docs/SparkInstallation.md diff --git a/docs/User-Guide.md b/arrow-data-source/docs/User-Guide.md similarity index 100% rename from docs/User-Guide.md rename to arrow-data-source/docs/User-Guide.md diff --git a/arrow-data-source/docs/image/arrowdatasource_validation.png b/arrow-data-source/docs/image/arrowdatasource_validation.png new file mode 100644 index 000000000000..c7583511a44a Binary files /dev/null and b/arrow-data-source/docs/image/arrowdatasource_validation.png differ diff --git a/docs/image/columnar.png b/arrow-data-source/docs/image/columnar.png similarity index 100% rename from docs/image/columnar.png rename to arrow-data-source/docs/image/columnar.png diff --git a/docs/image/core_arch.jpg b/arrow-data-source/docs/image/core_arch.jpg similarity index 100% rename from docs/image/core_arch.jpg rename to arrow-data-source/docs/image/core_arch.jpg diff --git a/docs/image/dataset.png b/arrow-data-source/docs/image/dataset.png similarity index 100% rename from docs/image/dataset.png rename to arrow-data-source/docs/image/dataset.png diff --git a/docs/image/decision_support_bench1_result_by_query.png b/arrow-data-source/docs/image/decision_support_bench1_result_by_query.png similarity index 100% rename from docs/image/decision_support_bench1_result_by_query.png rename to arrow-data-source/docs/image/decision_support_bench1_result_by_query.png diff --git a/docs/image/decision_support_bench1_result_in_total.png b/arrow-data-source/docs/image/decision_support_bench1_result_in_total.png similarity index 100% rename from docs/image/decision_support_bench1_result_in_total.png rename to arrow-data-source/docs/image/decision_support_bench1_result_in_total.png diff --git a/docs/image/decision_support_bench2_result_by_query.png b/arrow-data-source/docs/image/decision_support_bench2_result_by_query.png similarity index 100% rename from docs/image/decision_support_bench2_result_by_query.png rename to arrow-data-source/docs/image/decision_support_bench2_result_by_query.png diff --git a/docs/image/decision_support_bench2_result_in_total.png b/arrow-data-source/docs/image/decision_support_bench2_result_in_total.png similarity index 100% rename from docs/image/decision_support_bench2_result_in_total.png rename to arrow-data-source/docs/image/decision_support_bench2_result_in_total.png diff --git a/docs/image/kernel.png b/arrow-data-source/docs/image/kernel.png similarity index 100% rename from docs/image/kernel.png rename to arrow-data-source/docs/image/kernel.png diff --git a/docs/image/nativesql_arch.png b/arrow-data-source/docs/image/nativesql_arch.png similarity index 100% rename from docs/image/nativesql_arch.png rename to arrow-data-source/docs/image/nativesql_arch.png diff --git a/docs/image/performance.png b/arrow-data-source/docs/image/performance.png similarity index 100% rename from docs/image/performance.png rename to arrow-data-source/docs/image/performance.png diff --git a/docs/image/shuffle.png b/arrow-data-source/docs/image/shuffle.png similarity index 100% rename from docs/image/shuffle.png rename to arrow-data-source/docs/image/shuffle.png diff --git a/docs/index.md b/arrow-data-source/docs/index.md similarity index 100% rename from docs/index.md rename to arrow-data-source/docs/index.md diff --git a/arrow-data-source/mkdocs.yml b/arrow-data-source/mkdocs.yml new file mode 100644 index 000000000000..8be4217d163f --- /dev/null +++ b/arrow-data-source/mkdocs.yml @@ -0,0 +1,23 @@ +site_name: Arrow Data Source + +repo_name: 'Fork on GitHub ' +repo_url: "https://github.com/oap-project/arrow-data-source.git" +edit_uri: "" + + +nav: +- User Guide: User-Guide.md +- OAP Installation Guide: OAP-Installation-Guide.md +- OAP Developer Guide: OAP-Developer-Guide.md +- Version Selector: "../" + + + +theme: readthedocs + + +plugins: + - search + - mkdocs-versioning: + version: master + exclude_from_nav: ["image", "js", "css", "fonts", "img"] diff --git a/arrow-data-source/parquet/pom.xml b/arrow-data-source/parquet/pom.xml new file mode 100644 index 000000000000..eb31a59cb57c --- /dev/null +++ b/arrow-data-source/parquet/pom.xml @@ -0,0 +1,27 @@ + + + + spark-arrow-datasource + com.intel.oap + 1.1.0 + + 4.0.0 + + spark-arrow-datasource-parquet + + + ${project.basedir}/src/main/scala + ${project.basedir}/src/test/scala + + + + + com.intel.oap + spark-arrow-datasource-standard + ${project.version} + + + + diff --git a/arrow-data-source/parquet/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/parquet/ServiceLoaderUtil.scala b/arrow-data-source/parquet/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/parquet/ServiceLoaderUtil.scala new file mode 100644 index 000000000000..001b1c26cf61 --- /dev/null +++ b/arrow-data-source/parquet/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/parquet/ServiceLoaderUtil.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.parquet + +import java.util.Objects + +import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFileFormatIndicator} + +object ServiceLoaderUtil { + def ensureParquetFileFormatOverwritten(): Unit = { + val fmt = new ParquetFileFormat() + if (!Objects.equals(fmt.toString(), ParquetFileFormatIndicator.OVERWRITTEN_INDICATOR)) { + throw new ClassNotFoundException("ParquetFileFormat is not overwritten by Arrow. Consider " + + "reordering jar dependencies to let the overwritten version to be recognized by JVM") + } + } +} diff --git a/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala new file mode 100644 index 000000000000..bdf3166b1858 --- /dev/null +++ b/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +import java.io.IOException +import java.net.URI + +import scala.collection.JavaConverters._ +import scala.util.{Failure, Try} + +import com.intel.oap.spark.sql.execution.datasources.arrow.ArrowFileFormat +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce.{Job, JobID, OutputCommitter, TaskAttemptContext, TaskAttemptID, TaskID, TaskType} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS +import org.apache.parquet.hadoop.{Footer, ParquetFileReader, ParquetInputFormat, ParquetOutputCommitter, ParquetOutputFormat, ParquetRecordReader} +import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel +import org.apache.parquet.hadoop.codec.CodecConfig +import org.apache.parquet.hadoop.util.ContextUtil +import org.apache.parquet.schema.MessageType + +import org.apache.spark.TaskContext +import org.apache.spark.internal.Logging +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.JoinedRow +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection +import org.apache.spark.sql.catalyst.parser.LegacyTypeStringParser +import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.execution.datasources.{DataSourceUtils, OutputWriter, OutputWriterFactory, PartitionedFile, RecordReaderIterator, SchemaMergeUtils} +import org.apache.spark.sql.execution.datasources.parquet.ParquetSQLConf._ +import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.{DataSourceRegister, Filter} +import org.apache.spark.sql.types.{ArrayType, AtomicType, DataType, MapType, StructType, UserDefinedType} +import org.apache.spark.util.{SerializableConfiguration, ThreadUtils} + +/** + * This is expected to overwrite built-in ParquetFileFormat. Read is redirected to ArrowFileFormat's + * read. + */ +class ParquetFileFormat + extends ArrowFileFormat + with DataSourceRegister + with Logging + with Serializable { + + override def shortName(): String = "parquet" + + override def toString: String = ParquetFileFormatIndicator.OVERWRITTEN_INDICATOR + + override def hashCode(): Int = getClass.hashCode() + + override def equals(other: Any): Boolean = other.isInstanceOf[ParquetFileFormat] + + /** + * We copy following codes from Spark's built-in ParquetFileFormat. It's not suggested to + * change any of the logic to make sure we are on the same boat with Spark when it comes + * to Parquet write. + */ + override def prepareWrite(sparkSession: SparkSession, + job: Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) + + val conf = ContextUtil.getConfiguration(job) + + val committerClass = + conf.getClass( + SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key, + classOf[ParquetOutputCommitter], + classOf[OutputCommitter]) + + if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) { + logInfo("Using default output committer for Parquet: " + + classOf[ParquetOutputCommitter].getCanonicalName) + } else { + logInfo("Using user defined output committer for Parquet: " + committerClass.getCanonicalName) + } + + conf.setClass( + SQLConf.OUTPUT_COMMITTER_CLASS.key, + committerClass, + classOf[OutputCommitter]) + + // We're not really using `ParquetOutputFormat[Row]` for writing data here, because we override + // it in `ParquetOutputWriter` to support appending and dynamic partitioning. The reason why + // we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is + // bundled with `ParquetOutputFormat[Row]`. + job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]]) + + ParquetOutputFormat.setWriteSupportClass(job, classOf[ParquetWriteSupport]) + + // This metadata is useful for keeping UDTs like Vector/Matrix. + ParquetWriteSupport.setSchema(dataSchema, conf) + + // Sets flags for `ParquetWriteSupport`, which converts Catalyst schema to Parquet + // schema and writes actual rows to Parquet files. + conf.set( + SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key, + sparkSession.sessionState.conf.writeLegacyParquetFormat.toString) + + conf.set( + SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key, + sparkSession.sessionState.conf.parquetOutputTimestampType.toString) + + // Sets compression scheme + conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName) + + // SPARK-15719: Disables writing Parquet summary files by default. + if (conf.get(ParquetOutputFormat.JOB_SUMMARY_LEVEL) == null + && conf.get(ParquetOutputFormat.ENABLE_JOB_SUMMARY) == null) { + conf.setEnum(ParquetOutputFormat.JOB_SUMMARY_LEVEL, JobSummaryLevel.NONE) + } + + if (ParquetOutputFormat.getJobSummaryLevel(conf) != JobSummaryLevel.NONE + && !classOf[ParquetOutputCommitter].isAssignableFrom(committerClass)) { + // output summary is requested, but the class is not a Parquet Committer + logWarning(s"Committer $committerClass is not a ParquetOutputCommitter and cannot" + + s" create job summaries. " + + s"Set Parquet option ${ParquetOutputFormat.JOB_SUMMARY_LEVEL} to NONE.") + } + + new OutputWriterFactory { + // This OutputWriterFactory instance is deserialized when writing Parquet files on the + // executor side without constructing or deserializing ParquetFileFormat. Therefore, we hold + // another reference to ParquetLogRedirector.INSTANCE here to ensure the latter class is + // initialized. + private val parquetLogRedirector = ParquetLogRedirector.INSTANCE + + override def newInstance( + path: String, + dataSchema: StructType, + context: TaskAttemptContext): OutputWriter = { + new ParquetOutputWriter(path, context) + } + + override def getFileExtension(context: TaskAttemptContext): String = { + CodecConfig.from(context).getCodec.getExtension + ".parquet" + } + } + } + + override def inferSchema(sparkSession: SparkSession, + parameters: Map[String, String], + files: Seq[FileStatus]): Option[StructType] = { + val overwrite = sparkSession.sqlContext.conf.overwriteParquetDataSourceRead + if (overwrite) { + return super.inferSchema(sparkSession, parameters, files) + } + ParquetUtils.inferSchema(sparkSession, parameters, files) + } + + /** + * Returns whether the reader will return the rows as batch or not. + */ + override def supportBatch(sparkSession: SparkSession, schema: StructType): Boolean = { + val conf = sparkSession.sessionState.conf + val overwrite = sparkSession.sqlContext.conf.overwriteParquetDataSourceRead + if (overwrite) { + return super.supportBatch(sparkSession, schema) + } + conf.parquetVectorizedReaderEnabled && conf.wholeStageEnabled && + schema.length <= conf.wholeStageMaxNumFields && + schema.forall(_.dataType.isInstanceOf[AtomicType]) + } + + override def vectorTypes(requiredSchema: StructType, + partitionSchema: StructType, + sqlConf: SQLConf): Option[Seq[String]] = { + val overwrite = sqlConf.overwriteParquetDataSourceRead + if (overwrite) { + return super.vectorTypes(requiredSchema, partitionSchema, sqlConf) + } + Option(Seq.fill(requiredSchema.fields.length + partitionSchema.fields.length)( + if (!sqlConf.offHeapColumnVectorEnabled) { + classOf[OnHeapColumnVector].getName + } else { + classOf[OffHeapColumnVector].getName + } + )) + } + + override def isSplitable(sparkSession: SparkSession, + options: Map[String, String], + path: Path): Boolean = { + val overwrite = sparkSession.sqlContext.conf.overwriteParquetDataSourceRead + if (overwrite) { + return super.isSplitable(sparkSession, options, path) + } + true + } + + override def buildReaderWithPartitionValues(sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration): + (PartitionedFile) => Iterator[InternalRow] = { + val overwrite = sparkSession.sqlContext.conf.overwriteParquetDataSourceRead + if (overwrite) { + return super.buildReaderWithPartitionValues(sparkSession, dataSchema, + partitionSchema, requiredSchema, filters, options, hadoopConf) + } + hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName) + hadoopConf.set( + ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, + requiredSchema.json) + hadoopConf.set( + ParquetWriteSupport.SPARK_ROW_SCHEMA, + requiredSchema.json) + hadoopConf.set( + SQLConf.SESSION_LOCAL_TIMEZONE.key, + sparkSession.sessionState.conf.sessionLocalTimeZone) + hadoopConf.setBoolean( + SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, + sparkSession.sessionState.conf.nestedSchemaPruningEnabled) + hadoopConf.setBoolean( + SQLConf.CASE_SENSITIVE.key, + sparkSession.sessionState.conf.caseSensitiveAnalysis) + + ParquetWriteSupport.setSchema(requiredSchema, hadoopConf) + + // Sets flags for `ParquetToSparkSchemaConverter` + hadoopConf.setBoolean( + SQLConf.PARQUET_BINARY_AS_STRING.key, + sparkSession.sessionState.conf.isParquetBinaryAsString) + hadoopConf.setBoolean( + SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, + sparkSession.sessionState.conf.isParquetINT96AsTimestamp) + + val broadcastedHadoopConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) + + // TODO: if you move this into the closure it reverts to the default values. + // If true, enable using the custom RecordReader for parquet. This only works for + // a subset of the types (no complex types). + val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields) + val sqlConf = sparkSession.sessionState.conf + val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled + val enableVectorizedReader: Boolean = + sqlConf.parquetVectorizedReaderEnabled && + resultSchema.forall(_.dataType.isInstanceOf[AtomicType]) + val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled + val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion + val capacity = sqlConf.parquetVectorizedReaderBatchSize + val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown + // Whole stage codegen (PhysicalRDD) is able to deal with batches directly + val returningBatch = supportBatch(sparkSession, resultSchema) + val pushDownDate = sqlConf.parquetFilterPushDownDate + val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp + val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal + val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith + val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold + val isCaseSensitive = sqlConf.caseSensitiveAnalysis + + (file: PartitionedFile) => { + assert(file.partitionValues.numFields == partitionSchema.size) + + val filePath = new Path(new URI(file.filePath)) + val split = + new org.apache.parquet.hadoop.ParquetInputSplit( + filePath, + file.start, + file.start + file.length, + file.length, + Array.empty, + null) + + val sharedConf = broadcastedHadoopConf.value.value + + lazy val footerFileMetaData = + ParquetFileReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData + // Try to push down filters when filter push-down is enabled. + val pushed = if (enableParquetFilterPushDown) { + val parquetSchema = footerFileMetaData.getSchema + val parquetFilters = new ParquetFilters(parquetSchema, pushDownDate, pushDownTimestamp, + pushDownDecimal, pushDownStringStartWith, pushDownInFilterThreshold, isCaseSensitive) + filters + // Collects all converted Parquet filter predicates. Notice that not all predicates can be + // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap` + // is used here. + .flatMap(parquetFilters.createFilter(_)) + .reduceOption(FilterApi.and) + } else { + None + } + + // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps' + // *only* if the file was created by something other than "parquet-mr", so check the actual + // writer here for this file. We have to do this per-file, as each file in the table may + // have different writers. + // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads. + def isCreatedByParquetMr: Boolean = + footerFileMetaData.getCreatedBy().startsWith("parquet-mr") + + val convertTz = + if (timestampConversion && !isCreatedByParquetMr) { + Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) + } else { + None + } + + val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode( + footerFileMetaData.getKeyValueMetaData.get, + SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ)) + + val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) + val hadoopAttemptContext = + new TaskAttemptContextImpl(broadcastedHadoopConf.value.value, attemptId) + + // Try to push down filters when filter push-down is enabled. + // Notice: This push-down is RowGroups level, not individual records. + if (pushed.isDefined) { + ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get) + } + val taskContext = Option(TaskContext.get()) + if (enableVectorizedReader) { + val vectorizedReader = new VectorizedParquetRecordReader( + convertTz.orNull, + datetimeRebaseMode.toString, + enableOffHeapColumnVector && taskContext.isDefined, + capacity) + val iter = new RecordReaderIterator(vectorizedReader) + // SPARK-23457 Register a task completion listener before `initialization`. + taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close())) + vectorizedReader.initialize(split, hadoopAttemptContext) + logDebug(s"Appending $partitionSchema ${file.partitionValues}") + vectorizedReader.initBatch(partitionSchema, file.partitionValues) + if (returningBatch) { + vectorizedReader.enableReturningBatches() + } + + // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy. + iter.asInstanceOf[Iterator[InternalRow]] + } else { + logDebug(s"Falling back to parquet-mr") + // ParquetRecordReader returns InternalRow + val readSupport = new ParquetReadSupport( + convertTz, enableVectorizedReader = false, datetimeRebaseMode) + val reader = if (pushed.isDefined && enableRecordFilter) { + val parquetFilter = FilterCompat.get(pushed.get, null) + new ParquetRecordReader[InternalRow](readSupport, parquetFilter) + } else { + new ParquetRecordReader[InternalRow](readSupport) + } + val iter = new RecordReaderIterator[InternalRow](reader) + // SPARK-23457 Register a task completion listener before `initialization`. + taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close())) + reader.initialize(split, hadoopAttemptContext) + + val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes + val unsafeProjection = GenerateUnsafeProjection.generate(fullSchema, fullSchema) + + if (partitionSchema.length == 0) { + // There is no partition columns + iter.map(unsafeProjection) + } else { + val joinedRow = new JoinedRow() + iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues))) + } + } + } + } + + override def supportDataType(dataType: DataType): Boolean = dataType match { + case _: AtomicType => true + + case st: StructType => st.forall { f => supportDataType(f.dataType) } + + case ArrayType(elementType, _) => supportDataType(elementType) + + case MapType(keyType, valueType, _) => + supportDataType(keyType) && supportDataType(valueType) + + case udt: UserDefinedType[_] => supportDataType(udt.sqlType) + + case _ => false + } +} + +object ParquetFileFormat extends Logging { + /** + * Reads Spark SQL schema from a Parquet footer. If a valid serialized Spark SQL schema string + * can be found in the file metadata, returns the deserialized [[StructType]], otherwise, returns + * a [[StructType]] converted from the [[MessageType]] stored in this footer. + */ + def readSchemaFromFooter(footer: Footer, converter: ParquetToSparkSchemaConverter): StructType = { + val fileMetaData = footer.getParquetMetadata.getFileMetaData + fileMetaData + .getKeyValueMetaData + .asScala.toMap + .get(ParquetReadSupport.SPARK_METADATA_KEY) + .flatMap(deserializeSchemaString) + .getOrElse(converter.convert(fileMetaData.getSchema)) + } + + private def deserializeSchemaString(schemaString: String): Option[StructType] = { + // Tries to deserialize the schema string as JSON first, then falls back to the case class + // string parser (data generated by older versions of Spark SQL uses this format). + Try(DataType.fromJson(schemaString).asInstanceOf[StructType]).recover { + case _: Throwable => + logInfo( + "Serialized Spark schema in Parquet key-value metadata is not in JSON format, " + + "falling back to the deprecated DataType.fromCaseClassString parser.") + LegacyTypeStringParser.parseString(schemaString).asInstanceOf[StructType] + }.recoverWith { + case cause: Throwable => + logWarning( + "Failed to parse and ignored serialized Spark schema in " + + s"Parquet key-value metadata:\n\t$schemaString", cause) + Failure(cause) + }.toOption + } + + def mergeSchemasInParallel( + filesToTouch: Seq[FileStatus], + sparkSession: SparkSession): Option[StructType] = { + val assumeBinaryIsString = sparkSession.sessionState.conf.isParquetBinaryAsString + val assumeInt96IsTimestamp = sparkSession.sessionState.conf.isParquetINT96AsTimestamp + + val reader = (files: Seq[FileStatus], conf: Configuration, ignoreCorruptFiles: Boolean) => { + // Converter used to convert Parquet `MessageType` to Spark SQL `StructType` + val converter = new ParquetToSparkSchemaConverter( + assumeBinaryIsString = assumeBinaryIsString, + assumeInt96IsTimestamp = assumeInt96IsTimestamp) + + readParquetFootersInParallel(conf, files, ignoreCorruptFiles) + .map(ParquetFileFormat.readSchemaFromFooter(_, converter)) + } + + SchemaMergeUtils.mergeSchemasInParallel(sparkSession, filesToTouch, reader) + } + + private[parquet] def readParquetFootersInParallel( + conf: Configuration, + partFiles: Seq[FileStatus], + ignoreCorruptFiles: Boolean): Seq[Footer] = { + ThreadUtils.parmap(partFiles, "readingParquetFooters", 8) { currentFile => + try { + // Skips row group information since we only need the schema. + // ParquetFileReader.readFooter throws RuntimeException, instead of IOException, + // when it can't read the footer. + Some(new Footer(currentFile.getPath(), + ParquetFileReader.readFooter( + conf, currentFile, SKIP_ROW_GROUPS))) + } catch { case e: RuntimeException => + if (ignoreCorruptFiles) { + logWarning(s"Skipped the footer in the corrupted file: $currentFile", e) + None + } else { + throw new IOException(s"Could not read footer for file: $currentFile", e) + } + } + }.flatten + } +} diff --git a/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatIndicator.scala b/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatIndicator.scala new file mode 100644 index 000000000000..29aa38069b46 --- /dev/null +++ b/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatIndicator.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +object ParquetFileFormatIndicator { + val OVERWRITTEN_INDICATOR = "Parquet-Overwritten-By-Arrow" +} diff --git a/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSQLConf.scala b/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSQLConf.scala new file mode 100644 index 000000000000..9e3096189afa --- /dev/null +++ b/arrow-data-source/parquet/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSQLConf.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.internal.SQLConf + +object ParquetSQLConf { + // We default this option value to TRUE. This is because once the code is executed, the compiled + // arrow-datasource-parquet.jar file is supposed to be placed into Spark's lib folder. Which + // means it's user's intention to use the replaced ParquetDataSource. + val OVERWRITE_PARQUET_DATASOURCE_READ = + SQLConf.buildConf("spark.sql.arrow.overwrite.parquet.read") + .doc("Overwrite Parquet datasource v1 with reader of Arrow datasource.") + .booleanConf + .createWithDefault(true) + + implicit def fromSQLConf(c: SQLConf): ParquetSQLConf = { + new ParquetSQLConf(c) + } +} + +class ParquetSQLConf(c: SQLConf) { + def overwriteParquetDataSourceRead: Boolean = + c.getConf(ParquetSQLConf.OVERWRITE_PARQUET_DATASOURCE_READ) +} diff --git a/arrow-data-source/parquet/src/test/scala/com/intel/oap/spark/sql/execution/datasources/v2/parquet/ParquetFileFormatTest.scala b/arrow-data-source/parquet/src/test/scala/com/intel/oap/spark/sql/execution/datasources/v2/parquet/ParquetFileFormatTest.scala new file mode 100644 index 000000000000..84e0fc410a05 --- /dev/null +++ b/arrow-data-source/parquet/src/test/scala/com/intel/oap/spark/sql/execution/datasources/v2/parquet/ParquetFileFormatTest.scala @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.oap.spark.sql.execution.datasources.v2.parquet + +import java.io.File + +import com.intel.oap.vectorized.ArrowWritableColumnVector + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.parquet.ParquetSQLConf +import org.apache.spark.sql.execution.datasources.v2.arrow.SparkMemoryUtils +import org.apache.spark.sql.test.SharedSparkSession + +class ParquetFileFormatTest extends QueryTest with SharedSparkSession { + + private val parquetFile1 = "parquet-1.parquet" + + override protected def sparkConf: SparkConf = { + val conf = super.sparkConf + conf.set("spark.memory.offHeap.size", String.valueOf(256 * 1024 * 1024)) + conf + } + + def closeAllocators(): Unit = { + SparkMemoryUtils.contextAllocator().close() + } + + test("overwrite write only") { + import testImplicits._ + withSQLConf(ParquetSQLConf.OVERWRITE_PARQUET_DATASOURCE_READ.key -> "false") { + ServiceLoaderUtil.ensureParquetFileFormatOverwritten() + spark.read + .json(Seq("{\"col\": -1}", "{\"col\": 0}", "{\"col\": 1}", "{\"col\": 2}", + "{\"col\": null}") + .toDS()) + .repartition(1) + .write + .mode("overwrite") + .parquet(ParquetFileFormatTest.locateResourcePath(parquetFile1)) + val path = ParquetFileFormatTest.locateResourcePath(parquetFile1) + val frame = spark.read.parquet(path) + val eplan = frame.queryExecution.executedPlan + assert(eplan.toString + .contains("Format: Parquet-Overwritten-By-Arrow")) + val scan = eplan.find(_.isInstanceOf[FileSourceScanExec]).get + val typeAssertions = scan.executeColumnar() + .flatMap(b => (0 until b.numCols()).map(b.column(_))) + .map(!_.isInstanceOf[ArrowWritableColumnVector]) + .collect() + assert(typeAssertions.forall(p => p)) + } + } + + test("overwrite read and write") { + import testImplicits._ + ServiceLoaderUtil.ensureParquetFileFormatOverwritten() + spark.read + .json(Seq("{\"col\": -1}", "{\"col\": 0}", "{\"col\": 1}", "{\"col\": 2}", "{\"col\": null}") + .toDS()) + .repartition(1) + .write + .mode("overwrite") + .parquet(ParquetFileFormatTest.locateResourcePath(parquetFile1)) + val path = ParquetFileFormatTest.locateResourcePath(parquetFile1) + val frame = spark.read.parquet(path) + val eplan = frame.queryExecution.executedPlan + assert(eplan.toString + .contains("Format: Parquet-Overwritten-By-Arrow")) + val scan = eplan.find(_.isInstanceOf[FileSourceScanExec]).get + val typeAssertions = scan.executeColumnar() + .flatMap(b => (0 until b.numCols()).map(b.column(_))) + .map(_.isInstanceOf[ArrowWritableColumnVector]) + .collect() + assert(typeAssertions.forall(p => p)) + } +} + +object ParquetFileFormatTest { + private def locateResourcePath(resource: String): String = { + classOf[ParquetFileFormatTest].getClassLoader.getResource("") + .getPath.concat(File.separator).concat(resource) + } +} diff --git a/arrow-data-source/pom.xml b/arrow-data-source/pom.xml new file mode 100644 index 000000000000..267f3cced97f --- /dev/null +++ b/arrow-data-source/pom.xml @@ -0,0 +1,182 @@ + + 4.0.0 + + + com.intel.oap + spark-arrow-datasource + OAP Project Spark Arrow Datasource + pom + 1.1.0 + 2008 + + common + standard + parquet + + + 2.12.10 + 2.12 + 3.0.0 + 3.0.0 + UTF-8 + UTF-8 + + + + + scala-tools.org + Scala-Tools Maven2 Repository + http://scala-tools.org/repo-releases + + + + + + scala-tools.org + Scala-Tools Maven2 Repository + http://scala-tools.org/repo-releases + + + + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + org.apache.arrow + arrow-dataset + ${arrow.version} + + + io.netty + netty-common + + + io.netty + netty-buffer + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + compile + + + org.scala-lang + scala-library + ${scala.version} + provided + + + org.apache.spark + spark-sql_2.12 + ${spark.version} + + + org.apache.arrow + arrow-format + + + provided + + + junit + junit + 4.4 + test + + + org.apache.spark + spark-core_2.12 + ${spark.version} + test-jar + test + + + org.apache.spark + spark-catalyst_2.12 + ${spark.version} + test-jar + test + + + org.apache.spark + spark-sql_2.12 + ${spark.version} + test-jar + test + + + org.scalatest + scalatest_${scala.binary.version} + 3.0.5 + test + + + + + + + org.scalatest + scalatest-maven-plugin + + + + test + + + + + + org.scala-tools + maven-scala-plugin + + + + compile + testCompile + + process-sources + + + + ${scala.version} + + -target:jvm-1.5 + + + + + org.scalastyle + scalastyle-maven-plugin + 1.0.0 + + false + true + false + false + ${project.basedir}/src/main/scala + ${project.basedir}/src/test/scala + ${user.dir}/scalastyle-config.xml + ${project.basedir}/target/scalastyle-output.xml + ${project.build.sourceEncoding} + ${project.reporting.outputEncoding} + + + + + check + + + + + + + diff --git a/arrow-data-source/resource/arrowdatasource_validation.png b/arrow-data-source/resource/arrowdatasource_validation.png new file mode 100644 index 000000000000..c7583511a44a Binary files /dev/null and b/arrow-data-source/resource/arrowdatasource_validation.png differ diff --git a/arrow-data-source/standard/pom.xml b/arrow-data-source/standard/pom.xml new file mode 100644 index 000000000000..b5b8b29e65a5 --- /dev/null +++ b/arrow-data-source/standard/pom.xml @@ -0,0 +1,105 @@ + + + + spark-arrow-datasource + com.intel.oap + 1.1.0 + + 4.0.0 + + spark-arrow-datasource-standard + + + + + com.intel.oap + spark-arrow-datasource-common + ${project.version} + + + + + ${project.basedir}/src/main/scala + ${project.basedir}/src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 1.8 + 1.8 + + + + compile + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-src-1 + generate-sources + + add-source + + + + ${project.basedir}/src/main/java + + + + + + + maven-assembly-plugin + 3.0.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + + + + + org.scala-tools + maven-scala-plugin + + ${scala.version} + + + + + diff --git a/arrow-data-source/standard/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/arrow-data-source/standard/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 000000000000..dfdfbba20034 --- /dev/null +++ b/arrow-data-source/standard/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1 @@ +com.intel.oap.spark.sql.execution.datasources.v2.arrow.ArrowDataSourceV2 \ No newline at end of file diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/DataFrameReaderImplicits.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/DataFrameReaderImplicits.scala new file mode 100644 index 000000000000..ee3b5f291fc7 --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/DataFrameReaderImplicits.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql + +import org.apache.spark.sql.{DataFrame, DataFrameReader} + +class DataFrameReaderImplicits(r: DataFrameReader) { + + /** + * Loads a file via Arrow Datasets API and returns the result as a `DataFrame`. + * + * @param path input path + * @since 3.0.0-SNAPSHOT + */ + def arrow(path: String): DataFrame = { + // This method ensures that calls that explicit need single argument works, see SPARK-16009 + arrow(Seq(path): _*) + } + + /** + * Loads files via Arrow Datasets API and returns the result as a `DataFrame`. + * + * @param paths input paths + * @since 3.0.0-SNAPSHOT + */ + @scala.annotation.varargs + def arrow(paths: String*): DataFrame = r.format("arrow").load(paths: _*) +} + +object DataFrameReaderImplicits { + implicit def readerConverter(r: DataFrameReader): DataFrameReaderImplicits = { + new DataFrameReaderImplicits(r) + } +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowFileFormat.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowFileFormat.scala new file mode 100644 index 000000000000..61122592de1e --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowFileFormat.scala @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.arrow + +import java.net.URLDecoder + +import scala.collection.JavaConverters._ + +import com.intel.oap.spark.sql.execution.datasources.arrow.ArrowFileFormat.UnsafeItr +import com.intel.oap.spark.sql.execution.datasources.v2.arrow.{ArrowFilters, ArrowOptions, ArrowUtils} +import com.intel.oap.spark.sql.execution.datasources.v2.arrow.ArrowSQLConf._ +import com.intel.oap.vectorized.ArrowWritableColumnVector +import org.apache.arrow.dataset.scanner.ScanOptions +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce.Job + +import org.apache.spark.TaskContext +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriterFactory, PartitionedFile} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.{DataSourceRegister, Filter} +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.sql.vectorized.ColumnarBatch; + +class ArrowFileFormat extends FileFormat with DataSourceRegister with Serializable { + + + override def isSplitable(sparkSession: SparkSession, + options: Map[String, String], path: Path): Boolean = { + ArrowUtils.isOriginalFormatSplitable( + new ArrowOptions(new CaseInsensitiveStringMap(options.asJava).asScala.toMap)) + } + + def convert(files: Seq[FileStatus], options: Map[String, String]): Option[StructType] = { + ArrowUtils.readSchema(files, new CaseInsensitiveStringMap(options.asJava)) + } + + override def inferSchema( + sparkSession: SparkSession, + options: Map[String, String], + files: Seq[FileStatus]): Option[StructType] = { + convert(files, options) + } + + override def prepareWrite( + sparkSession: SparkSession, + job: Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + throw new UnsupportedOperationException("Write is not supported for Arrow source") + } + + override def supportBatch(sparkSession: SparkSession, dataSchema: StructType): Boolean = true + + override def buildReaderWithPartitionValues(sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { + val sqlConf = sparkSession.sessionState.conf; + val batchSize = sqlConf.parquetVectorizedReaderBatchSize + val enableFilterPushDown = sqlConf.arrowFilterPushDown + + (file: PartitionedFile) => { + val factory = ArrowUtils.makeArrowDiscovery( + URLDecoder.decode(file.filePath, "UTF-8"), file.start, file.length, + new ArrowOptions( + new CaseInsensitiveStringMap( + options.asJava).asScala.toMap)) + + // todo predicate validation / pushdown + val dataset = factory.finish(); + + val filter = if (enableFilterPushDown) { + ArrowFilters.translateFilters(filters) + } else { + org.apache.arrow.dataset.filter.Filter.EMPTY + } + + val scanOptions = new ScanOptions(requiredSchema.map(f => f.name).toArray, + filter, batchSize) + val scanner = dataset.newScan(scanOptions) + + val taskList = scanner + .scan() + .iterator() + .asScala + .toList + val itrList = taskList + .map(task => task.scan()) + + Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => { + itrList.foreach(_.close()) + taskList.foreach(_.close()) + scanner.close() + dataset.close() + factory.close() + })) + + val itr = itrList + .toIterator + .flatMap(itr => itr.asScala) + .map(vsr => ArrowUtils.loadVectors(vsr, file.partitionValues, partitionSchema, + requiredSchema)) + new UnsafeItr(itr).asInstanceOf[Iterator[InternalRow]] + } + } + + override def vectorTypes(requiredSchema: StructType, partitionSchema: StructType, + sqlConf: SQLConf): Option[Seq[String]] = { + Option(Seq.fill(requiredSchema.fields.length + partitionSchema.fields.length)( + classOf[ArrowWritableColumnVector].getName + )) + } + + override def shortName(): String = "arrow" +} + +object ArrowFileFormat { + class UnsafeItr[T](delegate: Iterator[ColumnarBatch]) + extends Iterator[ColumnarBatch] { + val holder = new ColumnarBatchRetainer() + + override def hasNext: Boolean = { + holder.release() + val hasNext = delegate.hasNext + hasNext + } + + override def next(): ColumnarBatch = { + val b = delegate.next() + holder.retain(b) + b + } + } + + class ColumnarBatchRetainer { + private var retained: Option[ColumnarBatch] = None + + def retain(batch: ColumnarBatch): Unit = { + if (retained.isDefined) { + throw new IllegalStateException + } + retained = Some(batch) + } + + def release(): Unit = { + retained.foreach(b => b.close()) + retained = None + } + } +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowDataSourceV2.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowDataSourceV2.scala new file mode 100644 index 000000000000..d19446778fe9 --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowDataSourceV2.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import com.intel.oap.spark.sql.execution.datasources.arrow.ArrowFileFormat + +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2 +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +class ArrowDataSourceV2 extends FileDataSourceV2 { + + private val format = classOf[ArrowFileFormat] + + override def fallbackFileFormat: Class[_ <: FileFormat] = { + format + } + + override def getTable(options: CaseInsensitiveStringMap): Table = { + val paths = getPaths(options) + val tableName = getTableName(paths) + ArrowTable(tableName, sparkSession, options, paths, None, fallbackFileFormat) + } + + override def shortName(): String = "arrow" +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowFilters.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowFilters.scala new file mode 100644 index 000000000000..f33c7995a852 --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowFilters.scala @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import org.apache.arrow.dataset.DatasetTypes +import org.apache.arrow.dataset.DatasetTypes.TreeNode +import org.apache.arrow.dataset.filter.FilterImpl + +import org.apache.spark.sql.sources._ +import org.apache.spark.sql.types.StructType + +object ArrowFilters { + def pruneWithSchema(pushedFilters: Array[Filter], schema: StructType): Seq[Filter] = { + pushedFilters.filter(pushedFilter => { + isToBeAccepted(pushedFilter, schema) + }) + } + + private def isToBeAccepted(pushedFilter: Filter, schema: StructType): Boolean = { + pushedFilter match { + case EqualTo(attribute, value) => existsIn(attribute, schema) + case GreaterThan(attribute, value) => existsIn(attribute, schema) + case GreaterThanOrEqual(attribute, value) => existsIn(attribute, schema) + case LessThan(attribute, value) => existsIn(attribute, schema) + case LessThanOrEqual(attribute, value) => existsIn(attribute, schema) + case Not(child) => isToBeAccepted(child, schema) + case And(left, right) => isToBeAccepted(left, schema) && isToBeAccepted(right, schema) + case Or(left, right) => isToBeAccepted(left, schema) && isToBeAccepted(right, schema) + case IsNotNull(attribute) => existsIn(attribute, schema) + case IsNull(attribute) => existsIn(attribute, schema) + case _ => false // fixme complete this + } + } + + private def existsIn(attr: String, schema: StructType): Boolean = { + schema.foreach(f => { + if (f.name == attr) { + return true; + } + }) + false + } + + def translateFilters(pushedFilters: Seq[Filter]): org.apache.arrow.dataset.filter.Filter = { + val node = pushedFilters + .flatMap(translateFilter) + .reduceOption((t1: TreeNode, t2: TreeNode) => { + DatasetTypes.TreeNode.newBuilder.setAndNode( + DatasetTypes.AndNode.newBuilder() + .setLeftArg(t1) + .setRightArg(t2) + .build()).build() + }) + if (node.isDefined) { + new FilterImpl(DatasetTypes.Condition.newBuilder() + .setRoot(node.get).build) + } else { + org.apache.arrow.dataset.filter.Filter.EMPTY + } + } + + private def translateValue(value: Any): Option[TreeNode] = { + value match { + case v: Integer => Some( + DatasetTypes.TreeNode.newBuilder.setIntNode( + DatasetTypes.IntNode.newBuilder.setValue(v).build) + .build) + case v: Long => Some( + DatasetTypes.TreeNode.newBuilder.setLongNode( + DatasetTypes.LongNode.newBuilder.setValue(v).build) + .build) + case v: Float => Some( + DatasetTypes.TreeNode.newBuilder.setFloatNode( + DatasetTypes.FloatNode.newBuilder.setValue(v).build) + .build) + case v: Double => Some( + DatasetTypes.TreeNode.newBuilder.setDoubleNode( + DatasetTypes.DoubleNode.newBuilder.setValue(v).build) + .build) + case v: Boolean => Some( + DatasetTypes.TreeNode.newBuilder.setBooleanNode( + DatasetTypes.BooleanNode.newBuilder.setValue(v).build) + .build) + case _ => None // fixme complete this + } + } + + private def translateFilter(pushedFilter: Filter): Option[TreeNode] = { + pushedFilter match { + case EqualTo(attribute, value) => + createComparisonNode("equal", attribute, value) + case GreaterThan(attribute, value) => + createComparisonNode("greater", attribute, value) + case GreaterThanOrEqual(attribute, value) => + createComparisonNode("greater_equal", attribute, value) + case LessThan(attribute, value) => + createComparisonNode("less", attribute, value) + case LessThanOrEqual(attribute, value) => + createComparisonNode("less_equal", attribute, value) + case Not(child) => + createNotNode(child) + case And(left, right) => + createAndNode(left, right) + case Or(left, right) => + createOrNode(left, right) + case IsNotNull(attribute) => + createIsNotNullNode(attribute) + case IsNull(attribute) => + createIsNullNode(attribute) + case _ => None // fixme complete this + } + } + + private def createComparisonNode(opName: String, + attribute: String, value: Any): Option[TreeNode] = { + val translatedValue = translateValue(value) + translatedValue match { + case Some(v) => Some( + DatasetTypes.TreeNode.newBuilder.setCpNode( + DatasetTypes.ComparisonNode.newBuilder + .setOpName(opName) // todo make op names enumerable + .setLeftArg( + DatasetTypes.TreeNode.newBuilder.setFieldNode( + DatasetTypes.FieldNode.newBuilder.setName(attribute).build) + .build) + .setRightArg(v) + .build) + .build) + case None => None + } + } + + def createNotNode(child: Filter): Option[TreeNode] = { + val translatedChild = translateFilter(child) + if (translatedChild.isEmpty) { + return None + } + Some(DatasetTypes.TreeNode.newBuilder + .setNotNode(DatasetTypes.NotNode.newBuilder.setArgs(translatedChild.get).build()).build()) + } + + def createIsNotNullNode(attribute: String): Option[TreeNode] = { + Some(DatasetTypes.TreeNode.newBuilder + .setIsValidNode( + DatasetTypes.IsValidNode.newBuilder.setArgs( + DatasetTypes.TreeNode.newBuilder.setFieldNode( + DatasetTypes.FieldNode.newBuilder.setName(attribute).build) + .build).build()).build()) + } + + def createIsNullNode(attribute: String): Option[TreeNode] = { + Some(DatasetTypes.TreeNode.newBuilder + .setNotNode( + DatasetTypes.NotNode.newBuilder.setArgs( + DatasetTypes.TreeNode.newBuilder + .setIsValidNode( + DatasetTypes.IsValidNode.newBuilder.setArgs( + DatasetTypes.TreeNode.newBuilder.setFieldNode( + DatasetTypes.FieldNode.newBuilder.setName(attribute).build) + .build) + .build()).build()).build()).build()) + } + + def createAndNode(left: Filter, right: Filter): Option[TreeNode] = { + val translatedLeft = translateFilter(left) + val translatedRight = translateFilter(right) + if (translatedLeft.isEmpty || translatedRight.isEmpty) { + return None + } + Some(DatasetTypes.TreeNode.newBuilder + .setAndNode(DatasetTypes.AndNode.newBuilder + .setLeftArg(translatedLeft.get) + .setRightArg(translatedRight.get) + .build()) + .build()) + } + + def createOrNode(left: Filter, right: Filter): Option[TreeNode] = { + val translatedLeft = translateFilter(left) + val translatedRight = translateFilter(right) + if (translatedLeft.isEmpty || translatedRight.isEmpty) { + return None + } + Some(DatasetTypes.TreeNode.newBuilder + .setOrNode(DatasetTypes.OrNode.newBuilder + .setLeftArg(translatedLeft.get) + .setRightArg(translatedRight.get) + .build()) + .build()) + } +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowOptions.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowOptions.scala new file mode 100644 index 000000000000..5be9f6a4a707 --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowOptions.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap + +class ArrowOptions(@transient private val parameters: CaseInsensitiveMap[String]) + extends Serializable { + + def this(parameters: Map[String, String]) = this(CaseInsensitiveMap(parameters)) + + val originalFormat = parameters + .get(ArrowOptions.KEY_ORIGINAL_FORMAT) + .getOrElse(ArrowOptions.DEFAULT_ORIGINAL_FORMAT) + @deprecated + val filesystem = parameters + .get(ArrowOptions.KEY_FILESYSTEM) + .getOrElse(ArrowOptions.DEFAULT_FILESYSTEM) +} + +object ArrowOptions { + val KEY_ORIGINAL_FORMAT = "originalFormat" + val DEFAULT_ORIGINAL_FORMAT = "parquet" + @deprecated + val KEY_FILESYSTEM = "filesystem" + val DEFAULT_FILESYSTEM = "hdfs" +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowPartitionReaderFactory.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowPartitionReaderFactory.scala new file mode 100644 index 000000000000..1f9145fb6ccd --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowPartitionReaderFactory.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import java.net.URLDecoder + +import scala.collection.JavaConverters._ + +import com.intel.oap.spark.sql.execution.datasources.v2.arrow.ArrowPartitionReaderFactory.ColumnarBatchRetainer +import com.intel.oap.spark.sql.execution.datasources.v2.arrow.ArrowSQLConf._ +import org.apache.arrow.dataset.scanner.ScanOptions + +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader} +import org.apache.spark.sql.execution.datasources.PartitionedFile +import org.apache.spark.sql.execution.datasources.v2.FilePartitionReaderFactory +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.SerializableConfiguration + +case class ArrowPartitionReaderFactory( + sqlConf: SQLConf, + broadcastedConf: Broadcast[SerializableConfiguration], + readDataSchema: StructType, + readPartitionSchema: StructType, + pushedFilters: Array[Filter], + options: ArrowOptions) + extends FilePartitionReaderFactory { + + private val batchSize = sqlConf.parquetVectorizedReaderBatchSize + private val enableFilterPushDown: Boolean = sqlConf.arrowFilterPushDown + + override def supportColumnarReads(partition: InputPartition): Boolean = true + + override def buildReader(partitionedFile: PartitionedFile): PartitionReader[InternalRow] = { + // disable row based read + throw new UnsupportedOperationException + } + + override def buildColumnarReader( + partitionedFile: PartitionedFile): PartitionReader[ColumnarBatch] = { + val path = partitionedFile.filePath + val factory = ArrowUtils.makeArrowDiscovery(URLDecoder.decode(path, "UTF-8"), + partitionedFile.start, partitionedFile.length, options) + val dataset = factory.finish() + val filter = if (enableFilterPushDown) { + ArrowFilters.translateFilters(ArrowFilters.pruneWithSchema(pushedFilters, readDataSchema)) + } else { + org.apache.arrow.dataset.filter.Filter.EMPTY + } + val scanOptions = new ScanOptions(readDataSchema.map(f => f.name).toArray, + filter, batchSize) + val scanner = dataset.newScan(scanOptions) + + val taskList = scanner + .scan() + .iterator() + .asScala + .toList + + val vsrItrList = taskList + .map(task => task.scan()) + + val batchItr = vsrItrList + .toIterator + .flatMap(itr => itr.asScala) + .map(bundledVectors => ArrowUtils.loadVectors(bundledVectors, partitionedFile.partitionValues, + readPartitionSchema, readDataSchema)) + + new PartitionReader[ColumnarBatch] { + val holder = new ColumnarBatchRetainer() + + override def next(): Boolean = { + holder.release() + batchItr.hasNext + } + + override def get(): ColumnarBatch = { + val batch = batchItr.next() + holder.retain(batch) + batch + } + + override def close(): Unit = { + holder.release() + vsrItrList.foreach(itr => itr.close()) + taskList.foreach(task => task.close()) + scanner.close() + dataset.close() + factory.close() + } + } + } +} + +object ArrowPartitionReaderFactory { + private class ColumnarBatchRetainer { + private var retained: Option[ColumnarBatch] = None + + def retain(batch: ColumnarBatch): Unit = { + if (retained.isDefined) { + throw new IllegalStateException + } + retained = Some(batch) + } + + def release(): Unit = { + retained.foreach(b => b.close()) + retained = None + } + } +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowSQLConf.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowSQLConf.scala new file mode 100644 index 000000000000..3b0cb69325dc --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowSQLConf.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import org.apache.spark.sql.internal.SQLConf + +object ArrowSQLConf { + val ARROW_FILTER_PUSHDOWN_ENABLED = SQLConf.buildConf("spark.sql.arrow.filterPushdown") + .doc("Enables Arrow filter push-down optimization when set to true.") + .booleanConf + .createWithDefault(true) + + implicit def fromSQLConf(c: SQLConf): ArrowSQLConf = { + new ArrowSQLConf(c) + } +} + +class ArrowSQLConf(c: SQLConf) { + def arrowFilterPushDown: Boolean = c.getConf(ArrowSQLConf.ARROW_FILTER_PUSHDOWN_ENABLED) +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowScan.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowScan.scala new file mode 100644 index 000000000000..20e069e01108 --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowScan.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import scala.collection.JavaConverters._ + +import org.apache.hadoop.fs.Path + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.connector.read.PartitionReaderFactory +import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex +import org.apache.spark.sql.execution.datasources.v2.FileScan +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.util.SerializableConfiguration + +case class ArrowScan( + sparkSession: SparkSession, + fileIndex: PartitioningAwareFileIndex, + readDataSchema: StructType, + readPartitionSchema: StructType, + pushedFilters: Array[Filter], + options: CaseInsensitiveStringMap, + partitionFilters: Seq[Expression] = Seq.empty, + dataFilters: Seq[Expression] = Seq.empty) + extends FileScan { + + override def isSplitable(path: Path): Boolean = { + ArrowUtils.isOriginalFormatSplitable( + new ArrowOptions(new CaseInsensitiveStringMap(options).asScala.toMap)) + } + + override def createReaderFactory(): PartitionReaderFactory = { + val caseSensitiveMap = options.asCaseSensitiveMap().asScala.toMap + val hconf = sparkSession.sessionState.newHadoopConfWithOptions(caseSensitiveMap) + val broadcastedConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hconf)) + ArrowPartitionReaderFactory( + sparkSession.sessionState.conf, + broadcastedConf, + readDataSchema, + readPartitionSchema, + pushedFilters, + new ArrowOptions(options.asScala.toMap)) + } + + override def withFilters(partitionFilters: Seq[Expression], + dataFilters: Seq[Expression]): FileScan = + this.copy(partitionFilters = partitionFilters, dataFilters = dataFilters) +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowScanBuilder.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowScanBuilder.scala new file mode 100644 index 000000000000..6583455e030f --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowScanBuilder.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters} +import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex +import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +case class ArrowScanBuilder( + sparkSession: SparkSession, + fileIndex: PartitioningAwareFileIndex, + schema: StructType, + dataSchema: StructType, + options: CaseInsensitiveStringMap) + extends FileScanBuilder(sparkSession, fileIndex, dataSchema) + with SupportsPushDownFilters { + + private var filters: Array[Filter] = Array.empty + private lazy val pushedArrowFilters: Array[Filter] = { + filters // todo filter validation & pushdown + } + + override def pushFilters(filters: Array[Filter]): Array[Filter] = { + this.filters = filters + this.filters + } + + override def pushedFilters: Array[Filter] = pushedArrowFilters + + override def build(): Scan = { + ArrowScan( + sparkSession, + fileIndex, + readDataSchema(), + readPartitionSchema(), + pushedFilters, + options) + } +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowTable.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowTable.scala new file mode 100644 index 000000000000..c25864a538c0 --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowTable.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import org.apache.arrow.memory.AllocationListener +import org.apache.hadoop.fs.FileStatus + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.FileTable +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +case class ArrowTable( + name: String, + sparkSession: SparkSession, + options: CaseInsensitiveStringMap, + paths: Seq[String], + userSpecifiedSchema: Option[StructType], + fallbackFileFormat: Class[_ <: FileFormat]) + extends FileTable(sparkSession, options, paths, userSpecifiedSchema) { + + override def inferSchema(files: Seq[FileStatus]): Option[StructType] = { + ArrowUtils.readSchema(files, options) + } + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { + ArrowScanBuilder(sparkSession, fileIndex, schema, dataSchema, options) + } + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + throw new UnsupportedOperationException // fixme implement later + } + + override def formatName: String = "ARROW" +} diff --git a/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala new file mode 100644 index 000000000000..bbffab14da7b --- /dev/null +++ b/arrow-data-source/standard/src/main/scala/com/intel/oap/spark/sql/execution/datasources/v2/arrow/ArrowUtils.scala @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.v2.arrow + +import java.net.URI +import java.util.TimeZone + +import scala.collection.JavaConverters._ + +import com.intel.oap.vectorized.ArrowWritableColumnVector +import org.apache.arrow.dataset.file.SingleFileDatasetFactory +import org.apache.arrow.dataset.scanner.ScanTask +import org.apache.arrow.vector.FieldVector +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID +import org.apache.arrow.vector.types.pojo.Schema +import org.apache.hadoop.fs.FileStatus + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.datasources.v2.arrow.{SparkMemoryUtils, SparkSchemaUtils} +import org.apache.spark.sql.execution.vectorized.ColumnVectorUtils +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.sql.vectorized.ColumnarBatch + +object ArrowUtils { + + def readSchema(file: FileStatus, options: CaseInsensitiveStringMap): Option[StructType] = { + val factory: SingleFileDatasetFactory = + makeArrowDiscovery(file.getPath.toString, -1L, -1L, + new ArrowOptions(options.asScala.toMap)) + val schema = factory.inspect() + try { + Option(SparkSchemaUtils.fromArrowSchema(schema)) + } finally { + factory.close() + } + } + + def readSchema(files: Seq[FileStatus], options: CaseInsensitiveStringMap): Option[StructType] = + readSchema(files.toList.head, options) // todo merge schema + + def isOriginalFormatSplitable(options: ArrowOptions): Boolean = { + val format = getFormat(options).getOrElse(throw new IllegalStateException) + format match { + case org.apache.arrow.dataset.file.FileFormat.PARQUET => + true + case _ => + false + } + } + + def makeArrowDiscovery(file: String, startOffset: Long, length: Long, + options: ArrowOptions): SingleFileDatasetFactory = { + + val format = getFormat(options).getOrElse(throw new IllegalStateException) + val allocator = SparkMemoryUtils.contextAllocator() + val factory = new SingleFileDatasetFactory(allocator, + SparkMemoryUtils.contextMemoryPool(), + format, + rewriteUri(file), + startOffset, + length) + factory + } + + def toArrowSchema(t: StructType): Schema = { + // fixme this might be platform dependent + SparkSchemaUtils.toArrowSchema(t, TimeZone.getDefault.getID) + } + + def loadVectors(bundledVectors: ScanTask.ArrowBundledVectors, partitionValues: InternalRow, + partitionSchema: StructType, dataSchema: StructType): ColumnarBatch = { + val rowCount: Int = getRowCount(bundledVectors) + val dataVectors = getDataVectors(bundledVectors, dataSchema) + val dictionaryVectors = getDictionaryVectors(bundledVectors, dataSchema) + + val vectors = ArrowWritableColumnVector.loadColumns(rowCount, dataVectors.asJava, + dictionaryVectors.asJava) + val partitionColumns = ArrowWritableColumnVector.allocateColumns(rowCount, partitionSchema) + (0 until partitionColumns.length).foreach(i => { + ColumnVectorUtils.populate(partitionColumns(i), partitionValues, i) + partitionColumns(i).setIsConstant() + }) + + val batch = new ColumnarBatch(vectors ++ partitionColumns, rowCount) + batch + } + + private def getRowCount(bundledVectors: ScanTask.ArrowBundledVectors) = { + val valueVectors = bundledVectors.valueVectors + val rowCount = valueVectors.getRowCount + rowCount + } + + private def getDataVectors(bundledVectors: ScanTask.ArrowBundledVectors, + dataSchema: StructType): List[FieldVector] = { + // TODO Deprecate following (bad performance maybe brought). + // TODO Assert vsr strictly matches dataSchema instead. + val valueVectors = bundledVectors.valueVectors + dataSchema.map(f => { + val vector = valueVectors.getVector(f.name) + if (vector == null) { + throw new IllegalStateException("Error: no vector named " + f.name + " in record bach") + } + vector + }).toList + } + + private def getDictionaryVectors(bundledVectors: ScanTask.ArrowBundledVectors, + dataSchema: StructType): List[FieldVector] = { + val valueVectors = bundledVectors.valueVectors + val dictionaryVectorMap = bundledVectors.dictionaryVectors + + val fieldNameToDictionaryEncoding = valueVectors.getSchema.getFields.asScala.map(f => { + f.getName -> f.getDictionary + }).toMap + + val dictionaryVectorsWithNulls = dataSchema.map(f => { + val de = fieldNameToDictionaryEncoding(f.name) + + Option(de) match { + case None => null + case _ => + if (de.getIndexType.getTypeID != ArrowTypeID.Int) { + throw new IllegalArgumentException("Wrong index type: " + de.getIndexType) + } + dictionaryVectorMap.get(de.getId).getVector + } + }).toList + dictionaryVectorsWithNulls + } + + private def getFormat( + options: ArrowOptions): Option[org.apache.arrow.dataset.file.FileFormat] = { + Option(options.originalFormat match { + case "parquet" => org.apache.arrow.dataset.file.FileFormat.PARQUET + case _ => throw new IllegalArgumentException("Unrecognizable format") + }) + } + + private def rewriteUri(uriStr: String): String = { + val uri = URI.create(uriStr) + val sch = uri.getScheme match { + case "hdfs" => "hdfs" + case "file" => "file" + } + val ssp = uri.getScheme match { + case "hdfs" => uri.getRawSchemeSpecificPart + case "file" => "//" + uri.getRawSchemeSpecificPart + } + val rewritten = new URI(sch, ssp, uri.getFragment) + rewritten.toString + } +} diff --git a/core/src/test/resources/test-data/cars.csv b/arrow-data-source/standard/src/test/resources/cars.csv similarity index 100% rename from core/src/test/resources/test-data/cars.csv rename to arrow-data-source/standard/src/test/resources/cars.csv diff --git a/arrow-data-source/standard/src/test/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowDataSourceTPCHBasedTest.scala b/arrow-data-source/standard/src/test/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowDataSourceTPCHBasedTest.scala new file mode 100644 index 000000000000..b33d4a1d32cb --- /dev/null +++ b/arrow-data-source/standard/src/test/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowDataSourceTPCHBasedTest.scala @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.oap.spark.sql.execution.datasources.arrow + +import java.util.concurrent.{Executors, TimeUnit} + +import com.intel.oap.spark.sql.DataFrameReaderImplicits._ +import com.intel.oap.spark.sql.execution.datasources.v2.arrow.{ArrowOptions, ArrowUtils} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.execution.datasources.v2.arrow.SparkMemoryUtils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession + +class ArrowDataSourceTPCHBasedTest extends QueryTest with SharedSparkSession { + + // tpc-h query cases: generated tpc-h dataset required + private val prefix = "/root/Downloads/" + private val tpchFolder = "date_tpch_10" + private val lineitem = prefix + tpchFolder + "/lineitem" + private val part = prefix + tpchFolder + "/part" + private val partSupp = prefix + tpchFolder + "/partsupp" + private val supplier = prefix + tpchFolder + "/supplier" + private val orders = prefix + tpchFolder + "/orders" + private val nation = prefix + tpchFolder + "/nation" + + + override protected def sparkConf: SparkConf = { + val conf = super.sparkConf + conf.set("spark.memory.offHeap.size", String.valueOf(128 * 1024 * 1024)) + conf + } + + ignore("tpch lineitem - desc") { + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(lineitem) + frame.createOrReplaceTempView("lineitem") + + spark.sql("describe lineitem").show() + } + + ignore("tpch part - special characters in path") { + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(part) + frame.createOrReplaceTempView("part") + + spark.sql("select * from part limit 100").show() + } + + ignore("tpch lineitem - read partition values") { + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(orders) + frame.createOrReplaceTempView("orders") + + spark.sql("select o_orderdate from orders limit 100").show() + } + + ignore("tpch lineitem - asterisk select") { + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(lineitem) + frame.createOrReplaceTempView("lineitem") + + spark.sql("select * from lineitem limit 10").show() + } + + ignore("tpch query 6") { + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(lineitem) + frame.createOrReplaceTempView("lineitem") + + spark.sql("select\n\tsum(l_extendedprice * l_discount) as revenue\n" + + "from\n\tlineitem\n" + + "where\n\tl_shipdate >= date '1994-01-01'\n\t" + + "and l_shipdate < date '1994-01-01' + interval '1' year\n\t" + + "and l_discount between .06 - 0.01 and .06 + 0.01\n\t" + + "and l_quantity < 24").show() + } + + ignore("tpch query 6 - performance comparision") { + val iterations = 10 + withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "false") { + val frame1 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(lineitem) + frame1.createOrReplaceTempView("lineitem_arrow") + + val frame2 = spark.read + .parquet(lineitem) + frame2.createOrReplaceTempView("lineitem_parquet") + + val pPrev = System.currentTimeMillis() + (0 until iterations).foreach(_ => + spark.sql("select\n\tsum(l_extendedprice * l_discount) as revenue\n" + + "from\n\tlineitem_parquet\n" + + "where\n\tl_shipdate >= date '1994-01-01'\n\t" + + "and l_shipdate < date '1994-01-01' + interval '1' year\n\t" + + "and l_discount between .06 - 0.01 and .06 + 0.01\n\t" + + "and l_quantity < 24").show() + ) + val parquetExecTime = System.currentTimeMillis() - pPrev + + val aPrev = System.currentTimeMillis() + (0 until iterations).foreach(_ => { + // scalastyle:off println + println(SparkMemoryUtils.contextAllocator().getAllocatedMemory) + // scalastyle:on println + spark.sql("select\n\tsum(l_extendedprice * l_discount) as revenue\n" + + "from\n\tlineitem_arrow\n" + + "where\n\tl_shipdate >= date '1994-01-01'\n\t" + + "and l_shipdate < date '1994-01-01' + interval '1' year\n\t" + + "and l_discount between .06 - 0.01 and .06 + 0.01\n\t" + + "and l_quantity < 24").show() + } + ) + val arrowExecTime = System.currentTimeMillis() - aPrev + + // unstable assert + assert(arrowExecTime < parquetExecTime) + } + } + + ignore("tpch query 16 - performance comparision") { + val iterations = 1 + withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "false") { + val frame1 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(partSupp) + frame1.createOrReplaceTempView("partsupp_arrow") + + val frame2 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(part) + frame2.createOrReplaceTempView("part_arrow") + + val frame3 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(supplier) + frame3.createOrReplaceTempView("supplier_arrow") + + val frame4 = spark.read + .parquet(partSupp) + frame4.createOrReplaceTempView("partsupp_parquet") + + val frame5 = spark.read + .parquet(part) + frame5.createOrReplaceTempView("part_parquet") + + val frame6 = spark.read + .parquet(supplier) + frame6.createOrReplaceTempView("supplier_parquet") + + val pPrev = System.currentTimeMillis() + (0 until iterations).foreach(_ => + spark.sql("select\n\tp_brand,\n\tp_type,\n\tp_size," + + "\n\tcount(distinct ps_suppkey) as supplier_cnt\n" + + "from\n\tpartsupp_parquet,\n\tpart_parquet\nwhere\n\tp_partkey" + + " = ps_partkey\n\tand p_brand <> 'Brand#45'\n\t" + + "and p_type not like 'MEDIUM POLISHED%'\n\tand p_size in " + + "(49, 14, 23, 45, 19, 3, 36, 9)\n\t" + + "and ps_suppkey not in (\n\t\tselect\n\t\t\ts_suppkey\n\t\t" + + "from\n\t\t\tsupplier_parquet\n\t\twhere\n\t\t\t" + + "s_comment like '%Customer%Complaints%'\n\t)\ngroup by\n\t" + + "p_brand,\n\tp_type,\n\tp_size\norder by\n\t" + + "supplier_cnt desc,\n\tp_brand,\n\tp_type,\n\tp_size").show() + ) + val parquetExecTime = System.currentTimeMillis() - pPrev + + val aPrev = System.currentTimeMillis() + (0 until iterations).foreach(_ => + spark.sql("select\n\tp_brand,\n\tp_type,\n\tp_size," + + "\n\tcount(distinct ps_suppkey) as supplier_cnt\n" + + "from\n\tpartsupp_arrow,\n\tpart_arrow\nwhere\n\tp_partkey" + + " = ps_partkey\n\tand p_brand <> 'Brand#45'\n\t" + + "and p_type not like 'MEDIUM POLISHED%'\n\tand p_size in " + + "(49, 14, 23, 45, 19, 3, 36, 9)\n\t" + + "and ps_suppkey not in (\n\t\tselect\n\t\t\ts_suppkey\n\t\t" + + "from\n\t\t\tsupplier_arrow\n\t\twhere\n\t\t\t" + + "s_comment like '%Customer%Complaints%'\n\t)\ngroup by\n\t" + + "p_brand,\n\tp_type,\n\tp_size\norder by\n\t" + + "supplier_cnt desc,\n\tp_brand,\n\tp_type,\n\tp_size").show() + ) + val arrowExecTime = System.currentTimeMillis() - aPrev + + // scalastyle:off println + println(arrowExecTime) + println(parquetExecTime) + // scalastyle:on println + // unstable assert + assert(arrowExecTime < parquetExecTime) + } + } + + ignore("tpch query 1") { + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(lineitem) + frame.createOrReplaceTempView("lineitem") + + spark.sql("select\n\tl_returnflag,\n\tl_linestatus," + + "\n\tsum(l_quantity) as sum_qty,\n\t" + + "sum(l_extendedprice) as sum_base_price," + + "\n\tsum(l_extendedprice * (1 - l_discount)) as sum_disc_price,\n\t" + + "sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge," + + "\n\tavg(l_quantity) as avg_qty,\n\t" + + "avg(l_extendedprice) as avg_price,\n\tavg(l_discount) as avg_disc," + + "\n\tcount(*) as count_order\nfrom\n\t" + + "lineitem\nwhere\n\tl_shipdate <= date '1998-12-01' - interval '90' day" + + "\ngroup by\n\tl_returnflag,\n\t" + + "l_linestatus\norder by\n\tl_returnflag,\n\tl_linestatus").explain(true) + } + + ignore("tpch query 21 - memory leak") { + val frame1 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(supplier) + frame1.createOrReplaceTempView("supplier") + val frame2 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(lineitem) + frame2.createOrReplaceTempView("lineitem") + val frame3 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(orders) + frame3.createOrReplaceTempView("orders") + val frame4 = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(nation) + frame4.createOrReplaceTempView("nation") + + Executors.newSingleThreadExecutor().execute(() => { + spark.sql("select\n\ts_name,\n\tcount(*) as numwait\nfrom\n\tsupplier,\n\t" + + "lineitem l1,\n\torders,\n\tnation\nwhere\n\ts_suppkey = l1.l_suppkey\n\t" + + "and o_orderkey = l1.l_orderkey\n\tand o_orderstatus = 'F'\n\tand " + + "l1.l_receiptdate > l1.l_commitdate\n\tand exists (\n\t\tselect\n\t\t\t*\n\t\tfrom\n\t\t\t" + + "lineitem l2\n\t\twhere\n\t\t\tl2.l_orderkey = l1.l_orderkey\n\t\t\tand " + + "l2.l_suppkey <> l1.l_suppkey\n\t)\n\tand not exists (\n\t\tselect\n\t\t\t*\n\t\t" + + "from\n\t\t\tlineitem l3\n\t\twhere\n\t\t\tl3.l_orderkey = l1.l_orderkey\n\t\t\t" + + "and l3.l_suppkey <> l1.l_suppkey\n\t\t\tand l3.l_receiptdate > " + + "l3.l_commitdate\n\t)\n\tand s_nationkey = n_nationkey\n\tand n_name = 'SAUDI ARABIA'\n" + + "group by\n\ts_name\norder by\n\tnumwait desc,\n\t" + + "s_name\nlimit 100").show() + }) + Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() => { + println("[org.apache.spark.sql.util.ArrowUtils.rootAllocator] " + + "Allocated memory amount: " + SparkMemoryUtils.contextAllocator()) + println("[com.intel.oap.vectorized.ArrowWritableColumnVector.allocator] " + + "Allocated memory amount: " + SparkMemoryUtils.contextAllocator().getAllocatedMemory) + }, 0L, 100L, TimeUnit.MILLISECONDS) + Thread.sleep(60 * 60 * 1000L) + } + +} diff --git a/arrow-data-source/standard/src/test/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowDataSourceTest.scala b/arrow-data-source/standard/src/test/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowDataSourceTest.scala new file mode 100644 index 000000000000..f88e085fa26f --- /dev/null +++ b/arrow-data-source/standard/src/test/scala/com/intel/oap/spark/sql/execution/datasources/arrow/ArrowDataSourceTest.scala @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.spark.sql.execution.datasources.arrow + +import java.io.File +import java.lang.management.ManagementFactory + +import com.intel.oap.spark.sql.DataFrameReaderImplicits._ +import com.intel.oap.spark.sql.execution.datasources.v2.arrow.ArrowOptions +import com.sun.management.UnixOperatingSystemMXBean +import org.apache.commons.io.FileUtils + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, QueryTest, Row} +import org.apache.spark.sql.execution.datasources.v2.arrow.SparkMemoryUtils +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType} + +class ArrowDataSourceTest extends QueryTest with SharedSparkSession { + private val parquetFile1 = "parquet-1.parquet" + private val parquetFile2 = "parquet-2.parquet" + private val parquetFile3 = "parquet-3.parquet" + private val parquetFile4 = "parquet-4.parquet" + private val parquetFile5 = "parquet-5.parquet" + + override protected def sparkConf: SparkConf = { + val conf = super.sparkConf + conf.set("spark.memory.offHeap.size", String.valueOf(10 * 1024 * 1024)) + conf + } + + override def beforeAll(): Unit = { + super.beforeAll() + import testImplicits._ + spark.read + .json(Seq("{\"col\": -1}", "{\"col\": 0}", "{\"col\": 1}", "{\"col\": 2}", "{\"col\": null}") + .toDS()) + .repartition(1) + .write + .mode("overwrite") + .parquet(ArrowDataSourceTest.locateResourcePath(parquetFile1)) + + spark.read + .json(Seq("{\"col\": \"a\"}", "{\"col\": \"b\"}") + .toDS()) + .repartition(1) + .write + .mode("overwrite") + .parquet(ArrowDataSourceTest.locateResourcePath(parquetFile2)) + + spark.read + .json(Seq("{\"col1\": \"apple\", \"col2\": 100}", "{\"col1\": \"pear\", \"col2\": 200}", + "{\"col1\": \"apple\", \"col2\": 300}") + .toDS()) + .repartition(1) + .write + .mode("overwrite") + .parquet(ArrowDataSourceTest.locateResourcePath(parquetFile3)) + + spark.range(1000) + .select(col("id"), col("id").as("k")) + .write + .partitionBy("k") + .format("parquet") + .mode("overwrite") + .parquet(ArrowDataSourceTest.locateResourcePath(parquetFile4)) + + spark.range(100) + .select(col("id"), col("id").as("k")) + .write + .partitionBy("k") + .format("parquet") + .mode("overwrite") + .parquet(ArrowDataSourceTest.locateResourcePath(parquetFile5)) + + } + + override def afterAll(): Unit = { + delete(ArrowDataSourceTest.locateResourcePath(parquetFile1)) + delete(ArrowDataSourceTest.locateResourcePath(parquetFile2)) + delete(ArrowDataSourceTest.locateResourcePath(parquetFile3)) + delete(ArrowDataSourceTest.locateResourcePath(parquetFile4)) + delete(ArrowDataSourceTest.locateResourcePath(parquetFile5)) + super.afterAll() + } + + test("reading parquet file") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile1) + verifyParquet( + spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(path)) + } + + test("create catalog table") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile1) + spark.catalog.createTable("ptab", path, "arrow") + val sql = "select * from ptab" + spark.sql(sql).explain() + verifyParquet(spark.sql(sql)) + } + + test("create table statement") { + spark.sql("drop table if exists ptab") + spark.sql("create table ptab (col1 varchar(14), col2 bigint, col3 bigint) " + + "using arrow " + + "partitioned by (col1)") + spark.sql("select * from ptab") + } + + test("simple SQL query on parquet file - 1") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile1) + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(path) + frame.createOrReplaceTempView("ptab") + verifyParquet(spark.sql("select * from ptab")) + verifyParquet(spark.sql("select col from ptab")) + verifyParquet(spark.sql("select col from ptab where col is not null or col is null")) + } + + test("simple SQL query on parquet file - 2") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile3) + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(path) + frame.createOrReplaceTempView("ptab") + val sqlFrame = spark.sql("select * from ptab") + assert( + sqlFrame.schema === + StructType(Seq(StructField("col1", StringType), StructField("col2", LongType)))) + val rows = sqlFrame.collect() + assert(rows(0).get(0) == "apple") + assert(rows(0).get(1) == 100) + assert(rows(1).get(0) == "pear") + assert(rows(1).get(1) == 200) + assert(rows(2).get(0) == "apple") + assert(rows(2).get(1) == 300) + assert(rows.length === 3) + } + + test("simple SQL query on parquet file with pushed filters") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile1) + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(path) + frame.createOrReplaceTempView("ptab") + spark.sql("select col from ptab where col = 1").explain(true) + val result = spark.sql("select col from ptab where col = 1") // fixme rowcount == 2? + assert( + result.schema === + StructType(Seq(StructField("col", LongType)))) + assert(result.collect().length === 1) + } + + test("ignore unrecognizable types when pushing down filters") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile2) + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(path) + frame.createOrReplaceTempView("ptab") + val rows = spark.sql("select * from ptab where col = 'b'").collect() + assert(rows.length === 1) + } + + ignore("dynamic partition pruning") { + withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", + SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", + SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false", + SQLConf.USE_V1_SOURCE_LIST.key -> "arrow", + SQLConf.CBO_ENABLED.key -> "true") { + + var path: String = null + path = ArrowDataSourceTest.locateResourcePath(parquetFile4) + spark.catalog.createTable("df1", path, "arrow") + path = ArrowDataSourceTest.locateResourcePath(parquetFile5) + spark.catalog.createTable("df2", path, "arrow") + + sql("ALTER TABLE df1 RECOVER PARTITIONS") + sql("ALTER TABLE df2 RECOVER PARTITIONS") + + sql("ANALYZE TABLE df1 COMPUTE STATISTICS FOR COLUMNS id") + sql("ANALYZE TABLE df2 COMPUTE STATISTICS FOR COLUMNS id") + + val df = sql("SELECT df1.id, df2.k FROM df1 " + + "JOIN df2 ON df1.k = df2.k AND df2.id < 2") + assert(df.queryExecution.executedPlan.toString().contains("dynamicpruningexpression")) + checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil) + } + } + + test("file descriptor leak") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile1) + val frame = spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "parquet") + .option(ArrowOptions.KEY_FILESYSTEM, "hdfs") + .arrow(path) + frame.createOrReplaceTempView("ptab") + + def getFdCount: Long = { + ManagementFactory.getOperatingSystemMXBean + .asInstanceOf[UnixOperatingSystemMXBean] + .getOpenFileDescriptorCount + } + + val initialFdCount = getFdCount + for (_ <- 0 until 100) { + verifyParquet(spark.sql("select * from ptab")) + } + val fdGrowth = getFdCount - initialFdCount + assert(fdGrowth < 100) + } + + test("file descriptor leak - v1") { + val path = ArrowDataSourceTest.locateResourcePath(parquetFile1) + spark.catalog.createTable("ptab2", path, "arrow") + + def getFdCount: Long = { + ManagementFactory.getOperatingSystemMXBean + .asInstanceOf[UnixOperatingSystemMXBean] + .getOpenFileDescriptorCount + } + + val initialFdCount = getFdCount + for (_ <- 0 until 100) { + verifyParquet(spark.sql("select * from ptab2")) + } + val fdGrowth = getFdCount - initialFdCount + assert(fdGrowth < 100) + } + + // csv cases: not implemented + private val csvFile = "cars.csv" + + ignore("reading csv file without specifying original format") { + verifyCsv(spark.read.format("arrow").load(csvFile)) + } + + ignore("reading csv file") { + val path = ArrowDataSourceTest.locateResourcePath(csvFile) + verifyCsv( + spark.read + .format("arrow") + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "csv") + .load(path)) + } + + ignore("read csv file - programmatic API ") { + val path = ArrowDataSourceTest.locateResourcePath(csvFile) + verifyCsv( + spark.read + .option(ArrowOptions.KEY_ORIGINAL_FORMAT, "csv") + .arrow(path)) + } + + def verifyCsv(frame: DataFrame): Unit = { + // todo assert something + } + + def verifyParquet(frame: DataFrame): Unit = { + assert( + frame.schema === + StructType(Seq(StructField("col", LongType)))) + assert(frame.collect().length === 5) + } + + def delete(path: String): Unit = { + FileUtils.forceDelete(new File(path)) + } + + def closeAllocators(): Unit = { + SparkMemoryUtils.contextAllocator().close() + } +} + +object ArrowDataSourceTest { + private def locateResourcePath(resource: String): String = { + classOf[ArrowDataSourceTest].getClassLoader.getResource("") + .getPath.concat(File.separator).concat(resource) + } +} diff --git a/cpp/.gitignore b/cpp/.gitignore deleted file mode 100644 index 22da1f13b17f..000000000000 --- a/cpp/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -thirdparty/*.tar* -CMakeFiles/ -CMakeCache.txt -CTestTestfile.cmake -Makefile -cmake_install.cmake -build/ -*-build/ -Testing/ -cmake-build-debug/ -cmake-build-release/ - -######################################### -# Editor temporary/working/backup files # -.#* -*\#*\# -[#]*# -*~ -*$ -*.bak -*flymake* -*.kdev4 -*.log -*.swp diff --git a/dep/arrow-data-source b/dep/arrow-data-source deleted file mode 160000 index 0ae27bcf9073..000000000000 --- a/dep/arrow-data-source +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0ae27bcf90731360de8135c8ba4002d943ce6d64 diff --git a/core/pom.xml b/native-sql-engine/core/pom.xml similarity index 98% rename from core/pom.xml rename to native-sql-engine/core/pom.xml index f5452b34b81c..6fd02b5ffa71 100644 --- a/core/pom.xml +++ b/native-sql-engine/core/pom.xml @@ -141,12 +141,12 @@ spark-arrow-datasource-common com.intel.oap - 1.1.0 + ${project.version} spark-arrow-datasource-standard com.intel.oap - 1.1.0 + ${project.version} test @@ -168,7 +168,7 @@ org.scalatest scalatest_${scala.binary.version} - 3.0.6 + 3.0.8 test @@ -368,7 +368,7 @@ false ${project.basedir}/src/main/scala ${project.basedir}/src/test/scala - ${project.basedir}/lib/scalastyle_config.xml + ${user.dir}/scalastyle-config.xml ${project.basedir}/scalastyle-output.xml UTF-8 @@ -383,7 +383,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.0.0 + 3.1.1 false true diff --git a/core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java similarity index 100% rename from core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java rename to native-sql-engine/core/src/main/java/com/intel/oap/datasource/VectorizedParquetArrowReader.java diff --git a/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java similarity index 100% rename from core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java rename to native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReader.java diff --git a/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java similarity index 100% rename from core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java rename to native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetReaderJniWrapper.java diff --git a/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java similarity index 100% rename from core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java rename to native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriter.java diff --git a/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java similarity index 100% rename from core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java rename to native-sql-engine/core/src/main/java/com/intel/oap/datasource/parquet/ParquetWriterJniWrapper.java diff --git a/core/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java b/native-sql-engine/core/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java similarity index 100% rename from core/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java rename to native-sql-engine/core/src/main/java/com/intel/oap/expression/ArrowColumnarBatch.java diff --git a/core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java b/native-sql-engine/core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java similarity index 100% rename from core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java rename to native-sql-engine/core/src/main/java/com/intel/oap/expression/ColumnarArithmeticWithGandiva.java diff --git a/core/src/main/java/com/intel/oap/vectorized/AdaptorReferenceManager.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/AdaptorReferenceManager.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/AdaptorReferenceManager.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/AdaptorReferenceManager.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowBufBuilder.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowCompressedStreamReader.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowFieldNodeBuilder.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilder.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilder.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilder.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilder.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilderImpl.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilderImpl.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilderImpl.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ArrowRecordBatchBuilderImpl.java diff --git a/core/src/main/java/com/intel/oap/vectorized/BatchIterator.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/BatchIterator.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/BatchIterator.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/BatchIterator.java diff --git a/core/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/CompressedVectorLoader.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluator.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ExpressionEvaluatorJniWrapper.java diff --git a/core/src/main/java/com/intel/oap/vectorized/JniUtils.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/JniUtils.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java diff --git a/core/src/main/java/com/intel/oap/vectorized/MetricsObject.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/MetricsObject.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/MetricsObject.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/MetricsObject.java diff --git a/core/src/main/java/com/intel/oap/vectorized/NativePartitioning.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativePartitioning.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/NativePartitioning.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativePartitioning.java diff --git a/core/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/NativeSerializableObject.java diff --git a/core/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/PartitionFileInfo.java diff --git a/core/src/main/java/com/intel/oap/vectorized/SerializableObject.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SerializableObject.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/SerializableObject.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SerializableObject.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleDecompressionJniWrapper.java diff --git a/core/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/ShuffleSplitterJniWrapper.java diff --git a/core/src/main/java/com/intel/oap/vectorized/SplitResult.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SplitResult.java similarity index 100% rename from core/src/main/java/com/intel/oap/vectorized/SplitResult.java rename to native-sql-engine/core/src/main/java/com/intel/oap/vectorized/SplitResult.java diff --git a/core/src/main/java/test b/native-sql-engine/core/src/main/java/test similarity index 100% rename from core/src/main/java/test rename to native-sql-engine/core/src/main/java/test diff --git a/core/src/main/scala/com/intel/oap/ColumnarGuardRule.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarGuardRule.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/ColumnarGuardRule.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarGuardRule.scala diff --git a/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/ColumnarPlugin.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPlugin.scala diff --git a/core/src/main/scala/com/intel/oap/ColumnarPluginConfig.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPluginConfig.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/ColumnarPluginConfig.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/ColumnarPluginConfig.scala diff --git a/core/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/BroadcastColumnarRDD.scala diff --git a/core/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/CoalesceBatchesExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarBasicPhysicalOperators.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarBasicPhysicalOperators.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarBasicPhysicalOperators.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarBasicPhysicalOperators.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarBatchScanExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarBroadcastHashJoinExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarBroadcastHashJoinExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarBroadcastHashJoinExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarBroadcastHashJoinExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarDataSourceRDD.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarDataSourceRDD.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarDataSourceRDD.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarDataSourceRDD.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarExpandExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarExpandExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarExpandExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarExpandExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarHashAggregateExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarHashAggregateExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarHashAggregateExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarHashAggregateExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarShuffledHashJoinExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarShuffledHashJoinExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarShuffledHashJoinExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarShuffledHashJoinExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarSortExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarSortExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarSortExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarSortExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarSortMergeJoinExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarSortMergeJoinExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarSortMergeJoinExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarSortMergeJoinExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarWholeStageCodegenExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWholeStageCodegenExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarWholeStageCodegenExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWholeStageCodegenExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/DataToArrowColumnarExec.scala diff --git a/core/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/execution/RowToArrowColumnarExec.scala diff --git a/core/src/main/scala/com/intel/oap/expression/CodeGeneration.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/CodeGeneration.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/CodeGeneration.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/CodeGeneration.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarBoundAttribute.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBoundAttribute.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarBoundAttribute.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBoundAttribute.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarConcatOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConcatOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarConcatOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConcatOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarConditionProjector.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionProjector.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarConditionProjector.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionProjector.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarFilter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarFilter.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarFilter.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarFilter.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarIfOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarIfOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarIfOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarIfOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarInOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarInOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarInOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarInOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarInSetOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarInSetOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarInSetOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarInSetOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarProjection.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarProjection.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarProjection.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarProjection.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarSortMergeJoin.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarSortMergeJoin.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarSortMergeJoin.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarSortMergeJoin.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarSorter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarSorter.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarSorter.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarSorter.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarSubquery.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarSubquery.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarSubquery.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarSubquery.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala diff --git a/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala diff --git a/core/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializer.scala diff --git a/core/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala similarity index 100% rename from core/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/vectorized/CloseableColumnBatchIterator.scala diff --git a/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala diff --git a/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseCodegenStages.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarExchange.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarHashedRelation.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ColumnarCustomShuffleReaderExec.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala diff --git a/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/VectorizedFilePartitionReaderHandler.scala diff --git a/core/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala diff --git a/core/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/OASPackageBridge.scala diff --git a/core/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/sql/util/StructTypeFWD.scala diff --git a/core/src/main/scala/org/apache/spark/util/ExecutorManager.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/util/ExecutorManager.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/util/ExecutorManager.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/util/ExecutorManager.scala diff --git a/core/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala b/native-sql-engine/core/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala rename to native-sql-engine/core/src/main/scala/org/apache/spark/util/UserAddedJarUtils.scala diff --git a/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadTest.java b/native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadTest.java similarity index 100% rename from core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadTest.java rename to native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadTest.java diff --git a/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadWriteTest.java b/native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadWriteTest.java similarity index 100% rename from core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadWriteTest.java rename to native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadWriteTest.java diff --git a/core/src/test/java/com/intel/oap/datasource/parquet/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet b/native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet similarity index 100% rename from core/src/test/java/com/intel/oap/datasource/parquet/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet rename to native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet diff --git a/core/src/test/java/com/intel/oap/tpc/MallocUtils.java b/native-sql-engine/core/src/test/java/com/intel/oap/tpc/MallocUtils.java similarity index 100% rename from core/src/test/java/com/intel/oap/tpc/MallocUtils.java rename to native-sql-engine/core/src/test/java/com/intel/oap/tpc/MallocUtils.java diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/metadata diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/offsets/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/0/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/1/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/2/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/3/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/4/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/5/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/6/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/7/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/8/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.1.0/state/0/9/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/commits/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/metadata diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/offsets/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/0/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/1/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/2/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/3/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/state/0/4/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/commits/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/metadata diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-for-multi-watermark-policy/offsets/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/commits/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/metadata diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/offsets/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/0/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/1/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/2/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/3/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-streaming-aggregate-state-format-1/state/0/4/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/commits/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/metadata diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/1 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/1 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/1 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/offsets/1 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/0/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/1/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/2/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/2.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/2.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/2.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.3.1-without-commit-log-metadata/state/0/4/2.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/commits/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/commits/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/commits/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/commits/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/metadata diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/offsets/0 diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/left-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/0/right-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/left-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/1/right-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/left-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/2/right-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/left-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/3/right-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/left-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyToNumValues/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyToNumValues/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyToNumValues/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyToNumValues/1.delta diff --git a/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyWithIndexToValue/1.delta b/native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyWithIndexToValue/1.delta similarity index 100% rename from core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyWithIndexToValue/1.delta rename to native-sql-engine/core/src/test/resources/structured-streaming/checkpoint-version-2.4.0-streaming-join/state/0/4/right-keyWithIndexToValue/1.delta diff --git a/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/commits/0 b/native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/commits/0 similarity index 100% rename from core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/commits/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/commits/0 diff --git a/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/metadata b/native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/metadata similarity index 100% rename from core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/metadata rename to native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/metadata diff --git a/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/offsets/0 b/native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/offsets/0 similarity index 100% rename from core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/offsets/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/chk%252520%252525@%252523chk/offsets/0 diff --git a/core/src/test/resources/structured-streaming/escaped-path-2.4.0/output %@#output/part-00000-97f675a2-bb82-4201-8245-05f3dae4c372-c000.snappy.parquet b/native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/output %@#output/part-00000-97f675a2-bb82-4201-8245-05f3dae4c372-c000.snappy.parquet similarity index 100% rename from core/src/test/resources/structured-streaming/escaped-path-2.4.0/output %@#output/part-00000-97f675a2-bb82-4201-8245-05f3dae4c372-c000.snappy.parquet rename to native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/output %@#output/part-00000-97f675a2-bb82-4201-8245-05f3dae4c372-c000.snappy.parquet diff --git a/core/src/test/resources/structured-streaming/escaped-path-2.4.0/output%20%25@%23output/_spark_metadata/0 b/native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/output%20%25@%23output/_spark_metadata/0 similarity index 100% rename from core/src/test/resources/structured-streaming/escaped-path-2.4.0/output%20%25@%23output/_spark_metadata/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/escaped-path-2.4.0/output%20%25@%23output/_spark_metadata/0 diff --git a/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/7.compact b/native-sql-engine/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/7.compact similarity index 100% rename from core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/7.compact rename to native-sql-engine/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/7.compact diff --git a/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8 b/native-sql-engine/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8 similarity index 100% rename from core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8 rename to native-sql-engine/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/8 diff --git a/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/9 b/native-sql-engine/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/9 similarity index 100% rename from core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/9 rename to native-sql-engine/core/src/test/resources/structured-streaming/file-sink-log-version-2.1.0/9 diff --git a/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/2.compact b/native-sql-engine/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/2.compact similarity index 100% rename from core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/2.compact rename to native-sql-engine/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/2.compact diff --git a/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/3 b/native-sql-engine/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/3 similarity index 100% rename from core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/3 rename to native-sql-engine/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/3 diff --git a/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/4 b/native-sql-engine/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/4 similarity index 100% rename from core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/4 rename to native-sql-engine/core/src/test/resources/structured-streaming/file-source-log-version-2.1.0/4 diff --git a/core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-json.txt b/native-sql-engine/core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-json.txt similarity index 100% rename from core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-json.txt rename to native-sql-engine/core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-json.txt diff --git a/core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-long.txt b/native-sql-engine/core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-long.txt similarity index 100% rename from core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-long.txt rename to native-sql-engine/core/src/test/resources/structured-streaming/file-source-offset-version-2.1.0-long.txt diff --git a/core/src/test/resources/structured-streaming/offset-log-version-2.1.0/0 b/native-sql-engine/core/src/test/resources/structured-streaming/offset-log-version-2.1.0/0 similarity index 100% rename from core/src/test/resources/structured-streaming/offset-log-version-2.1.0/0 rename to native-sql-engine/core/src/test/resources/structured-streaming/offset-log-version-2.1.0/0 diff --git a/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.0.txt b/native-sql-engine/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.0.txt similarity index 100% rename from core/src/test/resources/structured-streaming/query-event-logs-version-2.0.0.txt rename to native-sql-engine/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.0.txt diff --git a/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.1.txt b/native-sql-engine/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.1.txt similarity index 100% rename from core/src/test/resources/structured-streaming/query-event-logs-version-2.0.1.txt rename to native-sql-engine/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.1.txt diff --git a/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.2.txt b/native-sql-engine/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.2.txt similarity index 100% rename from core/src/test/resources/structured-streaming/query-event-logs-version-2.0.2.txt rename to native-sql-engine/core/src/test/resources/structured-streaming/query-event-logs-version-2.0.2.txt diff --git a/core/src/test/resources/structured-streaming/query-metadata-logs-version-2.1.0.txt b/native-sql-engine/core/src/test/resources/structured-streaming/query-metadata-logs-version-2.1.0.txt similarity index 100% rename from core/src/test/resources/structured-streaming/query-metadata-logs-version-2.1.0.txt rename to native-sql-engine/core/src/test/resources/structured-streaming/query-metadata-logs-version-2.1.0.txt diff --git a/core/src/test/resources/test-data/bad_after_good.csv b/native-sql-engine/core/src/test/resources/test-data/bad_after_good.csv similarity index 100% rename from core/src/test/resources/test-data/bad_after_good.csv rename to native-sql-engine/core/src/test/resources/test-data/bad_after_good.csv diff --git a/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc b/native-sql-engine/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc similarity index 100% rename from core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc rename to native-sql-engine/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc diff --git a/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet b/native-sql-engine/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet similarity index 100% rename from core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet rename to native-sql-engine/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet diff --git a/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet b/native-sql-engine/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet similarity index 100% rename from core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet rename to native-sql-engine/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet diff --git a/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet b/native-sql-engine/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet similarity index 100% rename from core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet rename to native-sql-engine/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet diff --git a/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet b/native-sql-engine/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet similarity index 100% rename from core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet rename to native-sql-engine/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet diff --git a/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc b/native-sql-engine/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc similarity index 100% rename from core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc rename to native-sql-engine/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc diff --git a/core/src/test/resources/test-data/bool.csv b/native-sql-engine/core/src/test/resources/test-data/bool.csv similarity index 100% rename from core/src/test/resources/test-data/bool.csv rename to native-sql-engine/core/src/test/resources/test-data/bool.csv diff --git a/core/src/test/resources/test-data/cars-alternative.csv b/native-sql-engine/core/src/test/resources/test-data/cars-alternative.csv similarity index 100% rename from core/src/test/resources/test-data/cars-alternative.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-alternative.csv diff --git a/core/src/test/resources/test-data/cars-blank-column-name.csv b/native-sql-engine/core/src/test/resources/test-data/cars-blank-column-name.csv similarity index 100% rename from core/src/test/resources/test-data/cars-blank-column-name.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-blank-column-name.csv diff --git a/core/src/test/resources/test-data/cars-crlf.csv b/native-sql-engine/core/src/test/resources/test-data/cars-crlf.csv similarity index 100% rename from core/src/test/resources/test-data/cars-crlf.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-crlf.csv diff --git a/core/src/test/resources/test-data/cars-empty-value.csv b/native-sql-engine/core/src/test/resources/test-data/cars-empty-value.csv similarity index 100% rename from core/src/test/resources/test-data/cars-empty-value.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-empty-value.csv diff --git a/core/src/test/resources/test-data/cars-malformed.csv b/native-sql-engine/core/src/test/resources/test-data/cars-malformed.csv similarity index 100% rename from core/src/test/resources/test-data/cars-malformed.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-malformed.csv diff --git a/core/src/test/resources/test-data/cars-multichar-delim-crazy.csv b/native-sql-engine/core/src/test/resources/test-data/cars-multichar-delim-crazy.csv similarity index 100% rename from core/src/test/resources/test-data/cars-multichar-delim-crazy.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-multichar-delim-crazy.csv diff --git a/core/src/test/resources/test-data/cars-multichar-delim.csv b/native-sql-engine/core/src/test/resources/test-data/cars-multichar-delim.csv similarity index 100% rename from core/src/test/resources/test-data/cars-multichar-delim.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-multichar-delim.csv diff --git a/core/src/test/resources/test-data/cars-null.csv b/native-sql-engine/core/src/test/resources/test-data/cars-null.csv similarity index 100% rename from core/src/test/resources/test-data/cars-null.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-null.csv diff --git a/core/src/test/resources/test-data/cars-unbalanced-quotes.csv b/native-sql-engine/core/src/test/resources/test-data/cars-unbalanced-quotes.csv similarity index 100% rename from core/src/test/resources/test-data/cars-unbalanced-quotes.csv rename to native-sql-engine/core/src/test/resources/test-data/cars-unbalanced-quotes.csv diff --git a/native-sql-engine/core/src/test/resources/test-data/cars.csv b/native-sql-engine/core/src/test/resources/test-data/cars.csv new file mode 100644 index 000000000000..40ded573ade5 --- /dev/null +++ b/native-sql-engine/core/src/test/resources/test-data/cars.csv @@ -0,0 +1,7 @@ + +year,make,model,comment,blank +"2012","Tesla","S","No comment", + +1997,Ford,E350,"Go get one now they are going fast", +2015,Chevy,Volt + diff --git a/core/src/test/resources/test-data/cars.tsv b/native-sql-engine/core/src/test/resources/test-data/cars.tsv similarity index 100% rename from core/src/test/resources/test-data/cars.tsv rename to native-sql-engine/core/src/test/resources/test-data/cars.tsv diff --git a/core/src/test/resources/test-data/cars_iso-8859-1.csv b/native-sql-engine/core/src/test/resources/test-data/cars_iso-8859-1.csv similarity index 100% rename from core/src/test/resources/test-data/cars_iso-8859-1.csv rename to native-sql-engine/core/src/test/resources/test-data/cars_iso-8859-1.csv diff --git a/core/src/test/resources/test-data/comments-whitespaces.csv b/native-sql-engine/core/src/test/resources/test-data/comments-whitespaces.csv similarity index 100% rename from core/src/test/resources/test-data/comments-whitespaces.csv rename to native-sql-engine/core/src/test/resources/test-data/comments-whitespaces.csv diff --git a/core/src/test/resources/test-data/comments.csv b/native-sql-engine/core/src/test/resources/test-data/comments.csv similarity index 100% rename from core/src/test/resources/test-data/comments.csv rename to native-sql-engine/core/src/test/resources/test-data/comments.csv diff --git a/core/src/test/resources/test-data/dates.csv b/native-sql-engine/core/src/test/resources/test-data/dates.csv similarity index 100% rename from core/src/test/resources/test-data/dates.csv rename to native-sql-engine/core/src/test/resources/test-data/dates.csv diff --git a/core/src/test/resources/test-data/dec-in-fixed-len.parquet b/native-sql-engine/core/src/test/resources/test-data/dec-in-fixed-len.parquet similarity index 100% rename from core/src/test/resources/test-data/dec-in-fixed-len.parquet rename to native-sql-engine/core/src/test/resources/test-data/dec-in-fixed-len.parquet diff --git a/core/src/test/resources/test-data/dec-in-i32.parquet b/native-sql-engine/core/src/test/resources/test-data/dec-in-i32.parquet similarity index 100% rename from core/src/test/resources/test-data/dec-in-i32.parquet rename to native-sql-engine/core/src/test/resources/test-data/dec-in-i32.parquet diff --git a/core/src/test/resources/test-data/dec-in-i64.parquet b/native-sql-engine/core/src/test/resources/test-data/dec-in-i64.parquet similarity index 100% rename from core/src/test/resources/test-data/dec-in-i64.parquet rename to native-sql-engine/core/src/test/resources/test-data/dec-in-i64.parquet diff --git a/core/src/test/resources/test-data/decimal.csv b/native-sql-engine/core/src/test/resources/test-data/decimal.csv similarity index 100% rename from core/src/test/resources/test-data/decimal.csv rename to native-sql-engine/core/src/test/resources/test-data/decimal.csv diff --git a/core/src/test/resources/test-data/disable_comments.csv b/native-sql-engine/core/src/test/resources/test-data/disable_comments.csv similarity index 100% rename from core/src/test/resources/test-data/disable_comments.csv rename to native-sql-engine/core/src/test/resources/test-data/disable_comments.csv diff --git a/core/src/test/resources/test-data/empty.csv b/native-sql-engine/core/src/test/resources/test-data/empty.csv similarity index 100% rename from core/src/test/resources/test-data/empty.csv rename to native-sql-engine/core/src/test/resources/test-data/empty.csv diff --git a/core/src/test/resources/test-data/impala_timestamp.parq b/native-sql-engine/core/src/test/resources/test-data/impala_timestamp.parq similarity index 100% rename from core/src/test/resources/test-data/impala_timestamp.parq rename to native-sql-engine/core/src/test/resources/test-data/impala_timestamp.parq diff --git a/core/src/test/resources/test-data/malformedRow.csv b/native-sql-engine/core/src/test/resources/test-data/malformedRow.csv similarity index 100% rename from core/src/test/resources/test-data/malformedRow.csv rename to native-sql-engine/core/src/test/resources/test-data/malformedRow.csv diff --git a/core/src/test/resources/test-data/native-splitter-output-all-null b/native-sql-engine/core/src/test/resources/test-data/native-splitter-output-all-null similarity index 100% rename from core/src/test/resources/test-data/native-splitter-output-all-null rename to native-sql-engine/core/src/test/resources/test-data/native-splitter-output-all-null diff --git a/core/src/test/resources/test-data/native-splitter-output-nullable-string b/native-sql-engine/core/src/test/resources/test-data/native-splitter-output-nullable-string similarity index 100% rename from core/src/test/resources/test-data/native-splitter-output-nullable-string rename to native-sql-engine/core/src/test/resources/test-data/native-splitter-output-nullable-string diff --git a/core/src/test/resources/test-data/nested-array-struct.parquet b/native-sql-engine/core/src/test/resources/test-data/nested-array-struct.parquet similarity index 100% rename from core/src/test/resources/test-data/nested-array-struct.parquet rename to native-sql-engine/core/src/test/resources/test-data/nested-array-struct.parquet diff --git a/core/src/test/resources/test-data/numbers.csv b/native-sql-engine/core/src/test/resources/test-data/numbers.csv similarity index 100% rename from core/src/test/resources/test-data/numbers.csv rename to native-sql-engine/core/src/test/resources/test-data/numbers.csv diff --git a/core/src/test/resources/test-data/old-repeated-int.parquet b/native-sql-engine/core/src/test/resources/test-data/old-repeated-int.parquet similarity index 100% rename from core/src/test/resources/test-data/old-repeated-int.parquet rename to native-sql-engine/core/src/test/resources/test-data/old-repeated-int.parquet diff --git a/core/src/test/resources/test-data/old-repeated-message.parquet b/native-sql-engine/core/src/test/resources/test-data/old-repeated-message.parquet similarity index 100% rename from core/src/test/resources/test-data/old-repeated-message.parquet rename to native-sql-engine/core/src/test/resources/test-data/old-repeated-message.parquet diff --git a/core/src/test/resources/test-data/parquet-1217.parquet b/native-sql-engine/core/src/test/resources/test-data/parquet-1217.parquet similarity index 100% rename from core/src/test/resources/test-data/parquet-1217.parquet rename to native-sql-engine/core/src/test/resources/test-data/parquet-1217.parquet diff --git a/core/src/test/resources/test-data/parquet-thrift-compat.snappy.parquet b/native-sql-engine/core/src/test/resources/test-data/parquet-thrift-compat.snappy.parquet similarity index 100% rename from core/src/test/resources/test-data/parquet-thrift-compat.snappy.parquet rename to native-sql-engine/core/src/test/resources/test-data/parquet-thrift-compat.snappy.parquet diff --git a/core/src/test/resources/test-data/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet b/native-sql-engine/core/src/test/resources/test-data/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet similarity index 100% rename from core/src/test/resources/test-data/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet rename to native-sql-engine/core/src/test/resources/test-data/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet diff --git a/core/src/test/resources/test-data/postgresql/agg.data b/native-sql-engine/core/src/test/resources/test-data/postgresql/agg.data similarity index 100% rename from core/src/test/resources/test-data/postgresql/agg.data rename to native-sql-engine/core/src/test/resources/test-data/postgresql/agg.data diff --git a/core/src/test/resources/test-data/postgresql/onek.data b/native-sql-engine/core/src/test/resources/test-data/postgresql/onek.data similarity index 100% rename from core/src/test/resources/test-data/postgresql/onek.data rename to native-sql-engine/core/src/test/resources/test-data/postgresql/onek.data diff --git a/core/src/test/resources/test-data/postgresql/tenk.data b/native-sql-engine/core/src/test/resources/test-data/postgresql/tenk.data similarity index 100% rename from core/src/test/resources/test-data/postgresql/tenk.data rename to native-sql-engine/core/src/test/resources/test-data/postgresql/tenk.data diff --git a/core/src/test/resources/test-data/proto-repeated-string.parquet b/native-sql-engine/core/src/test/resources/test-data/proto-repeated-string.parquet similarity index 100% rename from core/src/test/resources/test-data/proto-repeated-string.parquet rename to native-sql-engine/core/src/test/resources/test-data/proto-repeated-string.parquet diff --git a/core/src/test/resources/test-data/proto-repeated-struct.parquet b/native-sql-engine/core/src/test/resources/test-data/proto-repeated-struct.parquet similarity index 100% rename from core/src/test/resources/test-data/proto-repeated-struct.parquet rename to native-sql-engine/core/src/test/resources/test-data/proto-repeated-struct.parquet diff --git a/core/src/test/resources/test-data/proto-struct-with-array-many.parquet b/native-sql-engine/core/src/test/resources/test-data/proto-struct-with-array-many.parquet similarity index 100% rename from core/src/test/resources/test-data/proto-struct-with-array-many.parquet rename to native-sql-engine/core/src/test/resources/test-data/proto-struct-with-array-many.parquet diff --git a/core/src/test/resources/test-data/proto-struct-with-array.parquet b/native-sql-engine/core/src/test/resources/test-data/proto-struct-with-array.parquet similarity index 100% rename from core/src/test/resources/test-data/proto-struct-with-array.parquet rename to native-sql-engine/core/src/test/resources/test-data/proto-struct-with-array.parquet diff --git a/core/src/test/resources/test-data/simple_sparse.csv b/native-sql-engine/core/src/test/resources/test-data/simple_sparse.csv similarity index 100% rename from core/src/test/resources/test-data/simple_sparse.csv rename to native-sql-engine/core/src/test/resources/test-data/simple_sparse.csv diff --git a/core/src/test/resources/test-data/text-partitioned/year=2014/data.txt b/native-sql-engine/core/src/test/resources/test-data/text-partitioned/year=2014/data.txt similarity index 100% rename from core/src/test/resources/test-data/text-partitioned/year=2014/data.txt rename to native-sql-engine/core/src/test/resources/test-data/text-partitioned/year=2014/data.txt diff --git a/core/src/test/resources/test-data/text-partitioned/year=2015/data.txt b/native-sql-engine/core/src/test/resources/test-data/text-partitioned/year=2015/data.txt similarity index 100% rename from core/src/test/resources/test-data/text-partitioned/year=2015/data.txt rename to native-sql-engine/core/src/test/resources/test-data/text-partitioned/year=2015/data.txt diff --git a/core/src/test/resources/test-data/text-suite.txt b/native-sql-engine/core/src/test/resources/test-data/text-suite.txt similarity index 100% rename from core/src/test/resources/test-data/text-suite.txt rename to native-sql-engine/core/src/test/resources/test-data/text-suite.txt diff --git a/core/src/test/resources/test-data/text-suite2.txt b/native-sql-engine/core/src/test/resources/test-data/text-suite2.txt similarity index 100% rename from core/src/test/resources/test-data/text-suite2.txt rename to native-sql-engine/core/src/test/resources/test-data/text-suite2.txt diff --git a/core/src/test/resources/test-data/timemillis-in-i64.parquet b/native-sql-engine/core/src/test/resources/test-data/timemillis-in-i64.parquet similarity index 100% rename from core/src/test/resources/test-data/timemillis-in-i64.parquet rename to native-sql-engine/core/src/test/resources/test-data/timemillis-in-i64.parquet diff --git a/core/src/test/resources/test-data/unescaped-quotes.csv b/native-sql-engine/core/src/test/resources/test-data/unescaped-quotes.csv similarity index 100% rename from core/src/test/resources/test-data/unescaped-quotes.csv rename to native-sql-engine/core/src/test/resources/test-data/unescaped-quotes.csv diff --git a/core/src/test/resources/test-data/utf16LE.json b/native-sql-engine/core/src/test/resources/test-data/utf16LE.json similarity index 100% rename from core/src/test/resources/test-data/utf16LE.json rename to native-sql-engine/core/src/test/resources/test-data/utf16LE.json diff --git a/core/src/test/resources/test-data/utf16WithBOM.json b/native-sql-engine/core/src/test/resources/test-data/utf16WithBOM.json similarity index 100% rename from core/src/test/resources/test-data/utf16WithBOM.json rename to native-sql-engine/core/src/test/resources/test-data/utf16WithBOM.json diff --git a/core/src/test/resources/test-data/utf32BEWithBOM.json b/native-sql-engine/core/src/test/resources/test-data/utf32BEWithBOM.json similarity index 100% rename from core/src/test/resources/test-data/utf32BEWithBOM.json rename to native-sql-engine/core/src/test/resources/test-data/utf32BEWithBOM.json diff --git a/core/src/test/resources/test-data/value-malformed.csv b/native-sql-engine/core/src/test/resources/test-data/value-malformed.csv similarity index 100% rename from core/src/test/resources/test-data/value-malformed.csv rename to native-sql-engine/core/src/test/resources/test-data/value-malformed.csv diff --git a/core/src/test/resources/test-data/with-array-fields.json b/native-sql-engine/core/src/test/resources/test-data/with-array-fields.json similarity index 100% rename from core/src/test/resources/test-data/with-array-fields.json rename to native-sql-engine/core/src/test/resources/test-data/with-array-fields.json diff --git a/core/src/test/resources/test-data/with-map-fields.json b/native-sql-engine/core/src/test/resources/test-data/with-map-fields.json similarity index 100% rename from core/src/test/resources/test-data/with-map-fields.json rename to native-sql-engine/core/src/test/resources/test-data/with-map-fields.json diff --git a/core/src/test/resources/tpcds-queries/q1.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q1.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q1.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q1.sql diff --git a/core/src/test/resources/tpcds-queries/q10.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q10.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q10.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q10.sql diff --git a/core/src/test/resources/tpcds-queries/q11.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q11.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q11.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q11.sql diff --git a/core/src/test/resources/tpcds-queries/q12.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q12.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q12.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q12.sql diff --git a/core/src/test/resources/tpcds-queries/q13.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q13.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q13.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q13.sql diff --git a/core/src/test/resources/tpcds-queries/q14a.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q14a.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q14a.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q14a.sql diff --git a/core/src/test/resources/tpcds-queries/q14b.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q14b.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q14b.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q14b.sql diff --git a/core/src/test/resources/tpcds-queries/q15.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q15.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q15.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q15.sql diff --git a/core/src/test/resources/tpcds-queries/q16.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q16.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q16.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q16.sql diff --git a/core/src/test/resources/tpcds-queries/q17.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q17.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q17.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q17.sql diff --git a/core/src/test/resources/tpcds-queries/q18.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q18.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q18.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q18.sql diff --git a/core/src/test/resources/tpcds-queries/q19.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q19.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q19.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q19.sql diff --git a/core/src/test/resources/tpcds-queries/q2.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q2.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q2.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q2.sql diff --git a/core/src/test/resources/tpcds-queries/q20.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q20.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q20.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q20.sql diff --git a/core/src/test/resources/tpcds-queries/q21.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q21.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q21.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q21.sql diff --git a/core/src/test/resources/tpcds-queries/q22.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q22.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q22.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q22.sql diff --git a/core/src/test/resources/tpcds-queries/q23a.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q23a.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q23a.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q23a.sql diff --git a/core/src/test/resources/tpcds-queries/q23b.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q23b.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q23b.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q23b.sql diff --git a/core/src/test/resources/tpcds-queries/q24a.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q24a.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q24a.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q24a.sql diff --git a/core/src/test/resources/tpcds-queries/q24b.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q24b.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q24b.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q24b.sql diff --git a/core/src/test/resources/tpcds-queries/q25.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q25.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q25.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q25.sql diff --git a/core/src/test/resources/tpcds-queries/q26.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q26.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q26.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q26.sql diff --git a/core/src/test/resources/tpcds-queries/q27.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q27.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q27.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q27.sql diff --git a/core/src/test/resources/tpcds-queries/q28.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q28.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q28.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q28.sql diff --git a/core/src/test/resources/tpcds-queries/q29.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q29.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q29.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q29.sql diff --git a/core/src/test/resources/tpcds-queries/q3.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q3.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q3.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q3.sql diff --git a/core/src/test/resources/tpcds-queries/q30.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q30.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q30.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q30.sql diff --git a/core/src/test/resources/tpcds-queries/q31.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q31.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q31.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q31.sql diff --git a/core/src/test/resources/tpcds-queries/q32.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q32.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q32.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q32.sql diff --git a/core/src/test/resources/tpcds-queries/q33.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q33.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q33.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q33.sql diff --git a/core/src/test/resources/tpcds-queries/q34.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q34.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q34.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q34.sql diff --git a/core/src/test/resources/tpcds-queries/q35.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q35.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q35.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q35.sql diff --git a/core/src/test/resources/tpcds-queries/q36.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q36.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q36.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q36.sql diff --git a/core/src/test/resources/tpcds-queries/q37.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q37.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q37.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q37.sql diff --git a/core/src/test/resources/tpcds-queries/q38.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q38.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q38.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q38.sql diff --git a/core/src/test/resources/tpcds-queries/q39a.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q39a.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q39a.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q39a.sql diff --git a/core/src/test/resources/tpcds-queries/q39b.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q39b.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q39b.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q39b.sql diff --git a/core/src/test/resources/tpcds-queries/q4.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q4.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q4.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q4.sql diff --git a/core/src/test/resources/tpcds-queries/q40.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q40.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q40.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q40.sql diff --git a/core/src/test/resources/tpcds-queries/q41.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q41.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q41.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q41.sql diff --git a/core/src/test/resources/tpcds-queries/q42.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q42.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q42.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q42.sql diff --git a/core/src/test/resources/tpcds-queries/q43.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q43.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q43.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q43.sql diff --git a/core/src/test/resources/tpcds-queries/q44.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q44.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q44.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q44.sql diff --git a/core/src/test/resources/tpcds-queries/q45.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q45.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q45.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q45.sql diff --git a/core/src/test/resources/tpcds-queries/q46.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q46.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q46.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q46.sql diff --git a/core/src/test/resources/tpcds-queries/q47.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q47.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q47.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q47.sql diff --git a/core/src/test/resources/tpcds-queries/q48.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q48.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q48.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q48.sql diff --git a/core/src/test/resources/tpcds-queries/q49.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q49.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q49.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q49.sql diff --git a/core/src/test/resources/tpcds-queries/q5.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q5.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q5.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q5.sql diff --git a/core/src/test/resources/tpcds-queries/q50.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q50.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q50.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q50.sql diff --git a/core/src/test/resources/tpcds-queries/q51.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q51.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q51.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q51.sql diff --git a/core/src/test/resources/tpcds-queries/q52.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q52.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q52.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q52.sql diff --git a/core/src/test/resources/tpcds-queries/q53.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q53.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q53.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q53.sql diff --git a/core/src/test/resources/tpcds-queries/q54.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q54.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q54.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q54.sql diff --git a/core/src/test/resources/tpcds-queries/q55.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q55.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q55.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q55.sql diff --git a/core/src/test/resources/tpcds-queries/q56.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q56.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q56.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q56.sql diff --git a/core/src/test/resources/tpcds-queries/q57.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q57.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q57.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q57.sql diff --git a/core/src/test/resources/tpcds-queries/q58.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q58.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q58.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q58.sql diff --git a/core/src/test/resources/tpcds-queries/q59.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q59.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q59.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q59.sql diff --git a/core/src/test/resources/tpcds-queries/q6.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q6.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q6.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q6.sql diff --git a/core/src/test/resources/tpcds-queries/q60.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q60.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q60.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q60.sql diff --git a/core/src/test/resources/tpcds-queries/q61.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q61.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q61.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q61.sql diff --git a/core/src/test/resources/tpcds-queries/q62.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q62.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q62.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q62.sql diff --git a/core/src/test/resources/tpcds-queries/q63.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q63.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q63.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q63.sql diff --git a/core/src/test/resources/tpcds-queries/q64.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q64.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q64.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q64.sql diff --git a/core/src/test/resources/tpcds-queries/q65.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q65.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q65.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q65.sql diff --git a/core/src/test/resources/tpcds-queries/q66.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q66.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q66.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q66.sql diff --git a/core/src/test/resources/tpcds-queries/q67.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q67.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q67.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q67.sql diff --git a/core/src/test/resources/tpcds-queries/q68.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q68.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q68.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q68.sql diff --git a/core/src/test/resources/tpcds-queries/q69.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q69.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q69.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q69.sql diff --git a/core/src/test/resources/tpcds-queries/q7.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q7.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q7.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q7.sql diff --git a/core/src/test/resources/tpcds-queries/q70.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q70.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q70.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q70.sql diff --git a/core/src/test/resources/tpcds-queries/q71.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q71.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q71.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q71.sql diff --git a/core/src/test/resources/tpcds-queries/q72.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q72.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q72.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q72.sql diff --git a/core/src/test/resources/tpcds-queries/q73.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q73.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q73.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q73.sql diff --git a/core/src/test/resources/tpcds-queries/q74.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q74.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q74.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q74.sql diff --git a/core/src/test/resources/tpcds-queries/q75.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q75.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q75.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q75.sql diff --git a/core/src/test/resources/tpcds-queries/q76.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q76.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q76.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q76.sql diff --git a/core/src/test/resources/tpcds-queries/q77.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q77.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q77.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q77.sql diff --git a/core/src/test/resources/tpcds-queries/q78.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q78.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q78.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q78.sql diff --git a/core/src/test/resources/tpcds-queries/q79.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q79.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q79.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q79.sql diff --git a/core/src/test/resources/tpcds-queries/q8.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q8.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q8.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q8.sql diff --git a/core/src/test/resources/tpcds-queries/q80.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q80.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q80.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q80.sql diff --git a/core/src/test/resources/tpcds-queries/q81.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q81.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q81.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q81.sql diff --git a/core/src/test/resources/tpcds-queries/q82.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q82.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q82.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q82.sql diff --git a/core/src/test/resources/tpcds-queries/q83.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q83.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q83.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q83.sql diff --git a/core/src/test/resources/tpcds-queries/q84.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q84.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q84.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q84.sql diff --git a/core/src/test/resources/tpcds-queries/q85.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q85.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q85.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q85.sql diff --git a/core/src/test/resources/tpcds-queries/q86.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q86.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q86.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q86.sql diff --git a/core/src/test/resources/tpcds-queries/q87.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q87.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q87.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q87.sql diff --git a/core/src/test/resources/tpcds-queries/q88.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q88.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q88.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q88.sql diff --git a/core/src/test/resources/tpcds-queries/q89.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q89.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q89.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q89.sql diff --git a/core/src/test/resources/tpcds-queries/q9.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q9.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q9.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q9.sql diff --git a/core/src/test/resources/tpcds-queries/q90.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q90.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q90.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q90.sql diff --git a/core/src/test/resources/tpcds-queries/q91.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q91.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q91.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q91.sql diff --git a/core/src/test/resources/tpcds-queries/q92.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q92.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q92.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q92.sql diff --git a/core/src/test/resources/tpcds-queries/q93.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q93.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q93.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q93.sql diff --git a/core/src/test/resources/tpcds-queries/q94.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q94.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q94.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q94.sql diff --git a/core/src/test/resources/tpcds-queries/q95.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q95.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q95.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q95.sql diff --git a/core/src/test/resources/tpcds-queries/q96.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q96.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q96.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q96.sql diff --git a/core/src/test/resources/tpcds-queries/q97.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q97.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q97.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q97.sql diff --git a/core/src/test/resources/tpcds-queries/q98.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q98.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q98.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q98.sql diff --git a/core/src/test/resources/tpcds-queries/q99.sql b/native-sql-engine/core/src/test/resources/tpcds-queries/q99.sql similarity index 100% rename from core/src/test/resources/tpcds-queries/q99.sql rename to native-sql-engine/core/src/test/resources/tpcds-queries/q99.sql diff --git a/core/src/test/resources/tpch-queries/q1.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q1.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q1.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q1.sql diff --git a/core/src/test/resources/tpch-queries/q10.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q10.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q10.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q10.sql diff --git a/core/src/test/resources/tpch-queries/q11.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q11.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q11.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q11.sql diff --git a/core/src/test/resources/tpch-queries/q12.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q12.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q12.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q12.sql diff --git a/core/src/test/resources/tpch-queries/q13.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q13.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q13.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q13.sql diff --git a/core/src/test/resources/tpch-queries/q14.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q14.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q14.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q14.sql diff --git a/core/src/test/resources/tpch-queries/q15.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q15.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q15.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q15.sql diff --git a/core/src/test/resources/tpch-queries/q16.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q16.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q16.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q16.sql diff --git a/core/src/test/resources/tpch-queries/q17.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q17.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q17.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q17.sql diff --git a/core/src/test/resources/tpch-queries/q18.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q18.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q18.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q18.sql diff --git a/core/src/test/resources/tpch-queries/q19.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q19.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q19.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q19.sql diff --git a/core/src/test/resources/tpch-queries/q2.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q2.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q2.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q2.sql diff --git a/core/src/test/resources/tpch-queries/q20.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q20.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q20.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q20.sql diff --git a/core/src/test/resources/tpch-queries/q21.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q21.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q21.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q21.sql diff --git a/core/src/test/resources/tpch-queries/q22.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q22.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q22.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q22.sql diff --git a/core/src/test/resources/tpch-queries/q3.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q3.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q3.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q3.sql diff --git a/core/src/test/resources/tpch-queries/q4.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q4.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q4.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q4.sql diff --git a/core/src/test/resources/tpch-queries/q5.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q5.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q5.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q5.sql diff --git a/core/src/test/resources/tpch-queries/q6.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q6.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q6.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q6.sql diff --git a/core/src/test/resources/tpch-queries/q7.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q7.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q7.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q7.sql diff --git a/core/src/test/resources/tpch-queries/q8.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q8.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q8.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q8.sql diff --git a/core/src/test/resources/tpch-queries/q9.sql b/native-sql-engine/core/src/test/resources/tpch-queries/q9.sql similarity index 100% rename from core/src/test/resources/tpch-queries/q9.sql rename to native-sql-engine/core/src/test/resources/tpch-queries/q9.sql diff --git a/core/src/test/scala/com/intel/oap/tags/BroadcastHashJoinMode.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tags/BroadcastHashJoinMode.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tags/BroadcastHashJoinMode.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tags/BroadcastHashJoinMode.scala diff --git a/core/src/test/scala/com/intel/oap/tags/CommentOnContextPR.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tags/CommentOnContextPR.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tags/CommentOnContextPR.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tags/CommentOnContextPR.scala diff --git a/core/src/test/scala/com/intel/oap/tags/SortMergeJoinMode.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tags/SortMergeJoinMode.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tags/SortMergeJoinMode.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tags/SortMergeJoinMode.scala diff --git a/core/src/test/scala/com/intel/oap/tags/TestAndWriteLogs.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tags/TestAndWriteLogs.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tags/TestAndWriteLogs.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tags/TestAndWriteLogs.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/TableGen.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/TableGen.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/TableGen.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/TableGen.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSTableGen.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSTableGen.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/ds/TPCDSTableGen.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSTableGen.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/h/TPCHTableGen.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHTableGen.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/h/TPCHTableGen.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHTableGen.scala diff --git a/core/src/test/scala/com/intel/oap/tpc/util/TPCRunner.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/util/TPCRunner.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/tpc/util/TPCRunner.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/tpc/util/TPCRunner.scala diff --git a/core/src/test/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializerSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializerSuite.scala similarity index 100% rename from core/src/test/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializerSuite.scala rename to native-sql-engine/core/src/test/scala/com/intel/oap/vectorized/ArrowColumnarBatchSerializerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/shuffle/ColumnarShuffleWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/shuffle/ColumnarShuffleWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/shuffle/ColumnarShuffleWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/shuffle/ColumnarShuffleWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproxCountDistinctForIntervalsQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/CountMinSketchAggQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameComplexTypeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameHintSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameImplicitsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameRangeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTimeWindowingSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameTungstenSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFramesSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DeprecatedDatasetAggregatorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ExtraStrategiesSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala diff --git a/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/JoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala diff --git a/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/QueryTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/QueryTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/QueryTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/RepartitionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RepartitionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/RepartitionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/RepartitionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ReplaceNullWithFalseInPredicateEndToEndSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/RowSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RowSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/RowSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/RowSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SSBQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SSBQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SSBQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SSBQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala diff --git a/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala diff --git a/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/TPCHQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala diff --git a/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/UDFSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/UDTRegistrationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/XPathFunctionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SimpleWritableDataSource.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TableCapabilityCheckSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1ReadFallbackSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/AggregatingAccumulatorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/BroadcastExchangeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/CoGroupedIteratorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/CoGroupedIteratorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/CoGroupedIteratorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/CoGroupedIteratorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExchangeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SameResultSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkPlannerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/TakeOrderedAndProjectSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMapSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/UnsafeRowSerializerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeRowSerializerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/UnsafeRowSerializerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/UnsafeRowSerializerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/adaptive/ColumnarAdaptiveQueryExecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/ColumnarAdaptiveQueryExecSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/adaptive/ColumnarAdaptiveQueryExecSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/adaptive/ColumnarAdaptiveQueryExecSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationStoreSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationStoreSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationStoreSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/aggregate/SortBasedAggregationStoreSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnStatsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnTypeSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnarTestUtils.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnarTestUtils.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnarTestUtils.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/ColumnarTestUtils.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnAccessorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnAccessorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnAccessorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnAccessorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnBuilderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnBuilderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnBuilderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/NullableColumnBuilderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/DictionaryEncodingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/DictionaryEncodingSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/DictionaryEncodingSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/DictionaryEncodingSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/IntegralDeltaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/IntegralDeltaSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/IntegralDeltaSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/IntegralDeltaSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/RunLengthEncodingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/RunLengthEncodingSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/RunLengthEncodingSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/RunLengthEncodingSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/TestCompressibleColumnBuilder.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/TestCompressibleColumnBuilder.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/TestCompressibleColumnBuilder.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/TestCompressibleColumnBuilder.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/command/CommandUtilsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/CommandUtilsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/command/CommandUtilsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/CommandUtilsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteTaskStatsTrackerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/BucketingUtilsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/BucketingUtilsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/BucketingUtilsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/BucketingUtilsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileBasedDataSourceTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadNestedSchemaTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadNestedSchemaTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadNestedSchemaTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadNestedSchemaTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/ReadSchemaTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/RowDataSourceStrategySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SaveIntoDataSourceCommandSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/TestCsvData.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/TestCsvData.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/TestCsvData.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/TestCsvData.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtilsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtilsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtilsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtilsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/TestJsonData.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/TestJsonData.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/json/TestJsonData.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/TestJsonData.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopStreamSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/noop/NoopSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcPartitionDiscoverySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1FilterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV1SchemaPruningSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcV2SchemaPruningSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCommitterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompressionCodecPrecedenceSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormatSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetProtobufCompatibilitySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetThriftCompatibilitySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/history/SQLEventFilterBuilderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/history/SQLEventFilterBuilderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/history/SQLEventFilterBuilderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/history/SQLEventFilterBuilderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/history/SQLLiveEntitiesEventFilterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/history/SQLLiveEntitiesEventFilterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/history/SQLLiveEntitiesEventFilterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/history/SQLLiveEntitiesEventFilterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/BatchEvalPythonExecSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/python/PythonForeachWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonForeachWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/python/PythonForeachWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonForeachWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/python/RowQueueSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManagerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MemorySinkSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamMetadataSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/continuous/shuffle/ContinuousShuffleSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupportSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSinkSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProviderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/sources/TextSocketStreamSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/FlatMapGroupsWithStateExecHelperSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/MemoryStateStore.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/MemoryStateStore.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/MemoryStateStore.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/MemoryStateStore.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreCoordinatorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDDSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManagerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManagerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StreamingAggregationStateManagerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManagerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManagerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManagerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/internal/VariableSubstitutionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DDLSourceLoadSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ExternalCommandRunnerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PathOptionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/ResolvedDataSourceSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/sources/fakeExternalSources.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/fakeExternalSources.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/sources/fakeExternalSources.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/sources/fakeExternalSources.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/DeprecatedStreamingAggregationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamStressSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/OffsetSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/OffsetSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/OffsetSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/OffsetSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StateStoreMetricsTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenersConfSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryManagerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSymmetricHashJoinHelperSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousAggregationSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueryStatusAndProgressSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueryStatusAndProgressSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueryStatusAndProgressSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueryStatusAndProgressSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousQueuedDataReaderSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/continuous/EpochCoordinatorSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/EpochCoordinatorSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/continuous/EpochCoordinatorSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/continuous/EpochCoordinatorSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/sources/StreamingDataSourceV2Suite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/ui/UIUtilsSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/util/BlockOnStopSource.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/BlockOnStopSource.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/util/BlockOnStopSource.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/BlockOnStopSource.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/util/BlockingSource.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/BlockingSource.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/util/BlockingSource.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/BlockingSource.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/util/MockSourceProvider.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/MockSourceProvider.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/util/MockSourceProvider.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/MockSourceProvider.scala diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/util/StreamManualClock.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/StreamManualClock.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/streaming/util/StreamManualClock.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/streaming/util/StreamManualClock.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFlatSpecSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericFunSpecSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/GenericWordSpecSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/ProcessTestUtils.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/ProcessTestUtils.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/ProcessTestUtils.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/ProcessTestUtils.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/TestSQLContext.scala diff --git a/core/src/test/scala/org/apache/spark/sql/test/TestSparkSessionSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/TestSparkSessionSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/test/TestSparkSessionSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/test/TestSparkSessionSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameAggregateSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/travis/TravisDataFrameJoinSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala diff --git a/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala similarity index 100% rename from core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala rename to native-sql-engine/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala diff --git a/cpp/CMakeLists.txt b/native-sql-engine/cpp/CMakeLists.txt similarity index 100% rename from cpp/CMakeLists.txt rename to native-sql-engine/cpp/CMakeLists.txt diff --git a/cpp/compile.sh b/native-sql-engine/cpp/compile.sh similarity index 100% rename from cpp/compile.sh rename to native-sql-engine/cpp/compile.sh diff --git a/cpp/src/CMakeLists.txt b/native-sql-engine/cpp/src/CMakeLists.txt similarity index 100% rename from cpp/src/CMakeLists.txt rename to native-sql-engine/cpp/src/CMakeLists.txt diff --git a/cpp/src/benchmarks/CMakeLists.txt b/native-sql-engine/cpp/src/benchmarks/CMakeLists.txt similarity index 100% rename from cpp/src/benchmarks/CMakeLists.txt rename to native-sql-engine/cpp/src/benchmarks/CMakeLists.txt diff --git a/cpp/src/benchmarks/arrow_compute_benchmark_hash_aggregate.cc b/native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_hash_aggregate.cc similarity index 100% rename from cpp/src/benchmarks/arrow_compute_benchmark_hash_aggregate.cc rename to native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_hash_aggregate.cc diff --git a/cpp/src/benchmarks/arrow_compute_benchmark_join.cc b/native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_join.cc similarity index 100% rename from cpp/src/benchmarks/arrow_compute_benchmark_join.cc rename to native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_join.cc diff --git a/cpp/src/benchmarks/arrow_compute_benchmark_sort.cc b/native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_sort.cc similarity index 100% rename from cpp/src/benchmarks/arrow_compute_benchmark_sort.cc rename to native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_sort.cc diff --git a/cpp/src/benchmarks/arrow_compute_benchmark_wscg.cc b/native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_wscg.cc similarity index 100% rename from cpp/src/benchmarks/arrow_compute_benchmark_wscg.cc rename to native-sql-engine/cpp/src/benchmarks/arrow_compute_benchmark_wscg.cc diff --git a/cpp/src/benchmarks/shuffle_split_benchmark.cc b/native-sql-engine/cpp/src/benchmarks/shuffle_split_benchmark.cc similarity index 100% rename from cpp/src/benchmarks/shuffle_split_benchmark.cc rename to native-sql-engine/cpp/src/benchmarks/shuffle_split_benchmark.cc diff --git a/cpp/src/codegen/arrow_compute/code_generator.h b/native-sql-engine/cpp/src/codegen/arrow_compute/code_generator.h similarity index 100% rename from cpp/src/codegen/arrow_compute/code_generator.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/code_generator.h diff --git a/cpp/src/codegen/arrow_compute/expr_visitor.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/expr_visitor.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/expr_visitor.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/expr_visitor.cc diff --git a/cpp/src/codegen/arrow_compute/expr_visitor.h b/native-sql-engine/cpp/src/codegen/arrow_compute/expr_visitor.h similarity index 100% rename from cpp/src/codegen/arrow_compute/expr_visitor.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/expr_visitor.h diff --git a/cpp/src/codegen/arrow_compute/expr_visitor_impl.h b/native-sql-engine/cpp/src/codegen/arrow_compute/expr_visitor_impl.h similarity index 100% rename from cpp/src/codegen/arrow_compute/expr_visitor_impl.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/expr_visitor_impl.h diff --git a/cpp/src/codegen/arrow_compute/ext/action_codegen.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/action_codegen.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/action_codegen.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/action_codegen.h diff --git a/cpp/src/codegen/arrow_compute/ext/actions_impl.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/actions_impl.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/actions_impl.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/actions_impl.cc diff --git a/cpp/src/codegen/arrow_compute/ext/actions_impl.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/actions_impl.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/actions_impl.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/actions_impl.h diff --git a/cpp/src/codegen/arrow_compute/ext/array_appender.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/array_appender.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/array_appender.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/array_appender.h diff --git a/cpp/src/codegen/arrow_compute/ext/array_item_index.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/array_item_index.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/array_item_index.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/array_item_index.h diff --git a/cpp/src/codegen/arrow_compute/ext/basic_physical_kernels.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/basic_physical_kernels.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/basic_physical_kernels.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/basic_physical_kernels.cc diff --git a/cpp/src/codegen/arrow_compute/ext/cmp_function.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/cmp_function.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/cmp_function.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/cmp_function.h diff --git a/cpp/src/codegen/arrow_compute/ext/code_generator_base.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/code_generator_base.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/code_generator_base.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/code_generator_base.h diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_common.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_common.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_common.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_common.cc diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_common.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_common.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_common.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_common.h diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_context.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_context.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_context.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_context.h diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.cc diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_node_visitor.h diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_register.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_register.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_register.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_register.cc diff --git a/cpp/src/codegen/arrow_compute/ext/codegen_register.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_register.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/codegen_register.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/codegen_register.h diff --git a/cpp/src/codegen/arrow_compute/ext/conditioned_merge_join_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/conditioned_merge_join_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/conditioned_merge_join_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/conditioned_merge_join_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/conditioned_probe_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/conditioned_probe_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/conditioned_probe_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/conditioned_probe_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc diff --git a/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.h diff --git a/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/hash_relation_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_relation_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/hash_relation_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_relation_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/kernels_ext.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/kernels_ext.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/kernels_ext.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/kernels_ext.cc diff --git a/cpp/src/codegen/arrow_compute/ext/kernels_ext.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/kernels_ext.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/kernels_ext.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/kernels_ext.h diff --git a/cpp/src/codegen/arrow_compute/ext/merge_join_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/merge_join_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/merge_join_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/merge_join_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/probe_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/probe_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/probe_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/probe_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/sort_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/sort_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/sort_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/sort_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/typed_action_codegen_impl.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/typed_action_codegen_impl.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/typed_action_codegen_impl.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/typed_action_codegen_impl.h diff --git a/cpp/src/codegen/arrow_compute/ext/typed_node_visitor.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/typed_node_visitor.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/typed_node_visitor.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/typed_node_visitor.cc diff --git a/cpp/src/codegen/arrow_compute/ext/typed_node_visitor.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/typed_node_visitor.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/typed_node_visitor.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/typed_node_visitor.h diff --git a/cpp/src/codegen/arrow_compute/ext/whole_stage_codegen_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/whole_stage_codegen_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/whole_stage_codegen_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/whole_stage_codegen_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/window_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/window_kernel.cc similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/window_kernel.cc rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/window_kernel.cc diff --git a/cpp/src/codegen/arrow_compute/ext/window_sort_kernel.h b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/window_sort_kernel.h similarity index 100% rename from cpp/src/codegen/arrow_compute/ext/window_sort_kernel.h rename to native-sql-engine/cpp/src/codegen/arrow_compute/ext/window_sort_kernel.h diff --git a/cpp/src/codegen/code_generator.h b/native-sql-engine/cpp/src/codegen/code_generator.h similarity index 100% rename from cpp/src/codegen/code_generator.h rename to native-sql-engine/cpp/src/codegen/code_generator.h diff --git a/cpp/src/codegen/code_generator_factory.h b/native-sql-engine/cpp/src/codegen/code_generator_factory.h similarity index 100% rename from cpp/src/codegen/code_generator_factory.h rename to native-sql-engine/cpp/src/codegen/code_generator_factory.h diff --git a/cpp/src/codegen/common/hash_relation.h b/native-sql-engine/cpp/src/codegen/common/hash_relation.h similarity index 100% rename from cpp/src/codegen/common/hash_relation.h rename to native-sql-engine/cpp/src/codegen/common/hash_relation.h diff --git a/cpp/src/codegen/common/hash_relation_number.h b/native-sql-engine/cpp/src/codegen/common/hash_relation_number.h similarity index 100% rename from cpp/src/codegen/common/hash_relation_number.h rename to native-sql-engine/cpp/src/codegen/common/hash_relation_number.h diff --git a/cpp/src/codegen/common/hash_relation_string.h b/native-sql-engine/cpp/src/codegen/common/hash_relation_string.h similarity index 100% rename from cpp/src/codegen/common/hash_relation_string.h rename to native-sql-engine/cpp/src/codegen/common/hash_relation_string.h diff --git a/cpp/src/codegen/common/relation.cc b/native-sql-engine/cpp/src/codegen/common/relation.cc similarity index 100% rename from cpp/src/codegen/common/relation.cc rename to native-sql-engine/cpp/src/codegen/common/relation.cc diff --git a/cpp/src/codegen/common/relation_column.h b/native-sql-engine/cpp/src/codegen/common/relation_column.h similarity index 100% rename from cpp/src/codegen/common/relation_column.h rename to native-sql-engine/cpp/src/codegen/common/relation_column.h diff --git a/cpp/src/codegen/common/result_iterator.h b/native-sql-engine/cpp/src/codegen/common/result_iterator.h similarity index 100% rename from cpp/src/codegen/common/result_iterator.h rename to native-sql-engine/cpp/src/codegen/common/result_iterator.h diff --git a/cpp/src/codegen/common/sort_relation.h b/native-sql-engine/cpp/src/codegen/common/sort_relation.h similarity index 100% rename from cpp/src/codegen/common/sort_relation.h rename to native-sql-engine/cpp/src/codegen/common/sort_relation.h diff --git a/cpp/src/codegen/common/visitor_base.h b/native-sql-engine/cpp/src/codegen/common/visitor_base.h similarity index 100% rename from cpp/src/codegen/common/visitor_base.h rename to native-sql-engine/cpp/src/codegen/common/visitor_base.h diff --git a/cpp/src/codegen/compute_ext/code_generator.h b/native-sql-engine/cpp/src/codegen/compute_ext/code_generator.h similarity index 100% rename from cpp/src/codegen/compute_ext/code_generator.h rename to native-sql-engine/cpp/src/codegen/compute_ext/code_generator.h diff --git a/cpp/src/codegen/expr_visitor.cc b/native-sql-engine/cpp/src/codegen/expr_visitor.cc similarity index 100% rename from cpp/src/codegen/expr_visitor.cc rename to native-sql-engine/cpp/src/codegen/expr_visitor.cc diff --git a/cpp/src/codegen/expr_visitor.h b/native-sql-engine/cpp/src/codegen/expr_visitor.h similarity index 100% rename from cpp/src/codegen/expr_visitor.h rename to native-sql-engine/cpp/src/codegen/expr_visitor.h diff --git a/cpp/src/codegen/gandiva/code_generator.h b/native-sql-engine/cpp/src/codegen/gandiva/code_generator.h similarity index 100% rename from cpp/src/codegen/gandiva/code_generator.h rename to native-sql-engine/cpp/src/codegen/gandiva/code_generator.h diff --git a/cpp/src/jni/concurrent_map.h b/native-sql-engine/cpp/src/jni/concurrent_map.h similarity index 100% rename from cpp/src/jni/concurrent_map.h rename to native-sql-engine/cpp/src/jni/concurrent_map.h diff --git a/cpp/src/jni/jni_common.h b/native-sql-engine/cpp/src/jni/jni_common.h similarity index 100% rename from cpp/src/jni/jni_common.h rename to native-sql-engine/cpp/src/jni/jni_common.h diff --git a/cpp/src/jni/jni_wrapper.cc b/native-sql-engine/cpp/src/jni/jni_wrapper.cc similarity index 100% rename from cpp/src/jni/jni_wrapper.cc rename to native-sql-engine/cpp/src/jni/jni_wrapper.cc diff --git a/cpp/src/precompile/array.cc b/native-sql-engine/cpp/src/precompile/array.cc similarity index 100% rename from cpp/src/precompile/array.cc rename to native-sql-engine/cpp/src/precompile/array.cc diff --git a/cpp/src/precompile/array.h b/native-sql-engine/cpp/src/precompile/array.h similarity index 100% rename from cpp/src/precompile/array.h rename to native-sql-engine/cpp/src/precompile/array.h diff --git a/cpp/src/precompile/builder.cc b/native-sql-engine/cpp/src/precompile/builder.cc similarity index 100% rename from cpp/src/precompile/builder.cc rename to native-sql-engine/cpp/src/precompile/builder.cc diff --git a/cpp/src/precompile/builder.h b/native-sql-engine/cpp/src/precompile/builder.h similarity index 100% rename from cpp/src/precompile/builder.h rename to native-sql-engine/cpp/src/precompile/builder.h diff --git a/cpp/src/precompile/gandiva.h b/native-sql-engine/cpp/src/precompile/gandiva.h similarity index 100% rename from cpp/src/precompile/gandiva.h rename to native-sql-engine/cpp/src/precompile/gandiva.h diff --git a/cpp/src/precompile/gandiva_projector.cc b/native-sql-engine/cpp/src/precompile/gandiva_projector.cc similarity index 100% rename from cpp/src/precompile/gandiva_projector.cc rename to native-sql-engine/cpp/src/precompile/gandiva_projector.cc diff --git a/cpp/src/precompile/gandiva_projector.h b/native-sql-engine/cpp/src/precompile/gandiva_projector.h similarity index 100% rename from cpp/src/precompile/gandiva_projector.h rename to native-sql-engine/cpp/src/precompile/gandiva_projector.h diff --git a/cpp/src/precompile/hash_arrays_kernel.cc b/native-sql-engine/cpp/src/precompile/hash_arrays_kernel.cc similarity index 100% rename from cpp/src/precompile/hash_arrays_kernel.cc rename to native-sql-engine/cpp/src/precompile/hash_arrays_kernel.cc diff --git a/cpp/src/precompile/hash_arrays_kernel.h b/native-sql-engine/cpp/src/precompile/hash_arrays_kernel.h similarity index 100% rename from cpp/src/precompile/hash_arrays_kernel.h rename to native-sql-engine/cpp/src/precompile/hash_arrays_kernel.h diff --git a/cpp/src/precompile/hash_map.cc b/native-sql-engine/cpp/src/precompile/hash_map.cc similarity index 100% rename from cpp/src/precompile/hash_map.cc rename to native-sql-engine/cpp/src/precompile/hash_map.cc diff --git a/cpp/src/precompile/hash_map.h b/native-sql-engine/cpp/src/precompile/hash_map.h similarity index 100% rename from cpp/src/precompile/hash_map.h rename to native-sql-engine/cpp/src/precompile/hash_map.h diff --git a/cpp/src/precompile/sort.cc b/native-sql-engine/cpp/src/precompile/sort.cc similarity index 100% rename from cpp/src/precompile/sort.cc rename to native-sql-engine/cpp/src/precompile/sort.cc diff --git a/cpp/src/precompile/sort.h b/native-sql-engine/cpp/src/precompile/sort.h similarity index 100% rename from cpp/src/precompile/sort.h rename to native-sql-engine/cpp/src/precompile/sort.h diff --git a/cpp/src/precompile/sparse_hash_map.cc b/native-sql-engine/cpp/src/precompile/sparse_hash_map.cc similarity index 100% rename from cpp/src/precompile/sparse_hash_map.cc rename to native-sql-engine/cpp/src/precompile/sparse_hash_map.cc diff --git a/cpp/src/precompile/sparse_hash_map.h b/native-sql-engine/cpp/src/precompile/sparse_hash_map.h similarity index 100% rename from cpp/src/precompile/sparse_hash_map.h rename to native-sql-engine/cpp/src/precompile/sparse_hash_map.h diff --git a/cpp/src/precompile/type.cc b/native-sql-engine/cpp/src/precompile/type.cc similarity index 100% rename from cpp/src/precompile/type.cc rename to native-sql-engine/cpp/src/precompile/type.cc diff --git a/cpp/src/precompile/type.h b/native-sql-engine/cpp/src/precompile/type.h similarity index 100% rename from cpp/src/precompile/type.h rename to native-sql-engine/cpp/src/precompile/type.h diff --git a/cpp/src/precompile/type_traits.h b/native-sql-engine/cpp/src/precompile/type_traits.h similarity index 100% rename from cpp/src/precompile/type_traits.h rename to native-sql-engine/cpp/src/precompile/type_traits.h diff --git a/cpp/src/precompile/unsafe_array.cc b/native-sql-engine/cpp/src/precompile/unsafe_array.cc similarity index 100% rename from cpp/src/precompile/unsafe_array.cc rename to native-sql-engine/cpp/src/precompile/unsafe_array.cc diff --git a/cpp/src/precompile/unsafe_array.h b/native-sql-engine/cpp/src/precompile/unsafe_array.h similarity index 100% rename from cpp/src/precompile/unsafe_array.h rename to native-sql-engine/cpp/src/precompile/unsafe_array.h diff --git a/cpp/src/precompile/vector.cc b/native-sql-engine/cpp/src/precompile/vector.cc similarity index 100% rename from cpp/src/precompile/vector.cc rename to native-sql-engine/cpp/src/precompile/vector.cc diff --git a/cpp/src/precompile/vector.h b/native-sql-engine/cpp/src/precompile/vector.h similarity index 100% rename from cpp/src/precompile/vector.h rename to native-sql-engine/cpp/src/precompile/vector.h diff --git a/cpp/src/proto/Exprs.proto b/native-sql-engine/cpp/src/proto/Exprs.proto similarity index 100% rename from cpp/src/proto/Exprs.proto rename to native-sql-engine/cpp/src/proto/Exprs.proto diff --git a/cpp/src/proto/protobuf_utils.cc b/native-sql-engine/cpp/src/proto/protobuf_utils.cc similarity index 100% rename from cpp/src/proto/protobuf_utils.cc rename to native-sql-engine/cpp/src/proto/protobuf_utils.cc diff --git a/cpp/src/proto/protobuf_utils.h b/native-sql-engine/cpp/src/proto/protobuf_utils.h similarity index 100% rename from cpp/src/proto/protobuf_utils.h rename to native-sql-engine/cpp/src/proto/protobuf_utils.h diff --git a/cpp/src/resources/libhdfs.so b/native-sql-engine/cpp/src/resources/libhdfs.so similarity index 100% rename from cpp/src/resources/libhdfs.so rename to native-sql-engine/cpp/src/resources/libhdfs.so diff --git a/cpp/src/resources/libhdfs3.so b/native-sql-engine/cpp/src/resources/libhdfs3.so similarity index 100% rename from cpp/src/resources/libhdfs3.so rename to native-sql-engine/cpp/src/resources/libhdfs3.so diff --git a/cpp/src/resources/libprotobuf.so.13 b/native-sql-engine/cpp/src/resources/libprotobuf.so.13 similarity index 100% rename from cpp/src/resources/libprotobuf.so.13 rename to native-sql-engine/cpp/src/resources/libprotobuf.so.13 diff --git a/cpp/src/shuffle/splitter.cc b/native-sql-engine/cpp/src/shuffle/splitter.cc similarity index 100% rename from cpp/src/shuffle/splitter.cc rename to native-sql-engine/cpp/src/shuffle/splitter.cc diff --git a/cpp/src/shuffle/splitter.h b/native-sql-engine/cpp/src/shuffle/splitter.h similarity index 100% rename from cpp/src/shuffle/splitter.h rename to native-sql-engine/cpp/src/shuffle/splitter.h diff --git a/cpp/src/shuffle/type.h b/native-sql-engine/cpp/src/shuffle/type.h similarity index 100% rename from cpp/src/shuffle/type.h rename to native-sql-engine/cpp/src/shuffle/type.h diff --git a/cpp/src/shuffle/utils.h b/native-sql-engine/cpp/src/shuffle/utils.h similarity index 100% rename from cpp/src/shuffle/utils.h rename to native-sql-engine/cpp/src/shuffle/utils.h diff --git a/cpp/src/tests/CMakeLists.txt b/native-sql-engine/cpp/src/tests/CMakeLists.txt similarity index 100% rename from cpp/src/tests/CMakeLists.txt rename to native-sql-engine/cpp/src/tests/CMakeLists.txt diff --git a/cpp/src/tests/arrow_compute_test_aggregate.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_aggregate.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_aggregate.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_aggregate.cc diff --git a/cpp/src/tests/arrow_compute_test_aggregate_decimal.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_aggregate_decimal.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_aggregate_decimal.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_aggregate_decimal.cc diff --git a/cpp/src/tests/arrow_compute_test_check_condition.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_check_condition.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_check_condition.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_check_condition.cc diff --git a/cpp/src/tests/arrow_compute_test_join_smj.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_join_smj.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_join_smj.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_join_smj.cc diff --git a/cpp/src/tests/arrow_compute_test_join_wocg.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_join_wocg.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_join_wocg.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_join_wocg.cc diff --git a/cpp/src/tests/arrow_compute_test_precompile.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_precompile.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_precompile.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_precompile.cc diff --git a/cpp/src/tests/arrow_compute_test_sort.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_sort.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_sort.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_sort.cc diff --git a/cpp/src/tests/arrow_compute_test_window.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_window.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_window.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_window.cc diff --git a/cpp/src/tests/arrow_compute_test_wscg.cc b/native-sql-engine/cpp/src/tests/arrow_compute_test_wscg.cc similarity index 100% rename from cpp/src/tests/arrow_compute_test_wscg.cc rename to native-sql-engine/cpp/src/tests/arrow_compute_test_wscg.cc diff --git a/cpp/src/tests/shuffle_split_test.cc b/native-sql-engine/cpp/src/tests/shuffle_split_test.cc similarity index 100% rename from cpp/src/tests/shuffle_split_test.cc rename to native-sql-engine/cpp/src/tests/shuffle_split_test.cc diff --git a/cpp/src/tests/test_utils.h b/native-sql-engine/cpp/src/tests/test_utils.h similarity index 100% rename from cpp/src/tests/test_utils.h rename to native-sql-engine/cpp/src/tests/test_utils.h diff --git a/cpp/src/third_party/arrow/utils/hashing.h b/native-sql-engine/cpp/src/third_party/arrow/utils/hashing.h similarity index 100% rename from cpp/src/third_party/arrow/utils/hashing.h rename to native-sql-engine/cpp/src/third_party/arrow/utils/hashing.h diff --git a/cpp/src/third_party/arrow/vendored/xxhash/README.md b/native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/README.md similarity index 100% rename from cpp/src/third_party/arrow/vendored/xxhash/README.md rename to native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/README.md diff --git a/cpp/src/third_party/arrow/vendored/xxhash/xxh3.h b/native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/xxh3.h similarity index 100% rename from cpp/src/third_party/arrow/vendored/xxhash/xxh3.h rename to native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/xxh3.h diff --git a/cpp/src/third_party/arrow/vendored/xxhash/xxhash.c b/native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/xxhash.c similarity index 100% rename from cpp/src/third_party/arrow/vendored/xxhash/xxhash.c rename to native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/xxhash.c diff --git a/cpp/src/third_party/arrow/vendored/xxhash/xxhash.h b/native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/xxhash.h similarity index 100% rename from cpp/src/third_party/arrow/vendored/xxhash/xxhash.h rename to native-sql-engine/cpp/src/third_party/arrow/vendored/xxhash/xxhash.h diff --git a/cpp/src/third_party/datetime/README.md b/native-sql-engine/cpp/src/third_party/datetime/README.md similarity index 100% rename from cpp/src/third_party/datetime/README.md rename to native-sql-engine/cpp/src/third_party/datetime/README.md diff --git a/cpp/src/third_party/datetime/date.h b/native-sql-engine/cpp/src/third_party/datetime/date.h similarity index 100% rename from cpp/src/third_party/datetime/date.h rename to native-sql-engine/cpp/src/third_party/datetime/date.h diff --git a/cpp/src/third_party/gandiva/CMakeLists.txt b/native-sql-engine/cpp/src/third_party/gandiva/CMakeLists.txt similarity index 100% rename from cpp/src/third_party/gandiva/CMakeLists.txt rename to native-sql-engine/cpp/src/third_party/gandiva/CMakeLists.txt diff --git a/cpp/src/third_party/gandiva/decimal_ops.cc b/native-sql-engine/cpp/src/third_party/gandiva/decimal_ops.cc similarity index 100% rename from cpp/src/third_party/gandiva/decimal_ops.cc rename to native-sql-engine/cpp/src/third_party/gandiva/decimal_ops.cc diff --git a/cpp/src/third_party/gandiva/decimal_ops.h b/native-sql-engine/cpp/src/third_party/gandiva/decimal_ops.h similarity index 100% rename from cpp/src/third_party/gandiva/decimal_ops.h rename to native-sql-engine/cpp/src/third_party/gandiva/decimal_ops.h diff --git a/cpp/src/third_party/gandiva/epoch_time_point.h b/native-sql-engine/cpp/src/third_party/gandiva/epoch_time_point.h similarity index 100% rename from cpp/src/third_party/gandiva/epoch_time_point.h rename to native-sql-engine/cpp/src/third_party/gandiva/epoch_time_point.h diff --git a/cpp/src/third_party/gandiva/time.cc b/native-sql-engine/cpp/src/third_party/gandiva/time.cc similarity index 100% rename from cpp/src/third_party/gandiva/time.cc rename to native-sql-engine/cpp/src/third_party/gandiva/time.cc diff --git a/cpp/src/third_party/gandiva/time_constants.h b/native-sql-engine/cpp/src/third_party/gandiva/time_constants.h similarity index 100% rename from cpp/src/third_party/gandiva/time_constants.h rename to native-sql-engine/cpp/src/third_party/gandiva/time_constants.h diff --git a/cpp/src/third_party/gandiva/time_fields.h b/native-sql-engine/cpp/src/third_party/gandiva/time_fields.h similarity index 100% rename from cpp/src/third_party/gandiva/time_fields.h rename to native-sql-engine/cpp/src/third_party/gandiva/time_fields.h diff --git a/cpp/src/third_party/gandiva/types.h b/native-sql-engine/cpp/src/third_party/gandiva/types.h similarity index 100% rename from cpp/src/third_party/gandiva/types.h rename to native-sql-engine/cpp/src/third_party/gandiva/types.h diff --git a/cpp/src/third_party/murmurhash/murmurhash32.h b/native-sql-engine/cpp/src/third_party/murmurhash/murmurhash32.h similarity index 100% rename from cpp/src/third_party/murmurhash/murmurhash32.h rename to native-sql-engine/cpp/src/third_party/murmurhash/murmurhash32.h diff --git a/cpp/src/third_party/murmurhash/murmurhash64.h b/native-sql-engine/cpp/src/third_party/murmurhash/murmurhash64.h similarity index 100% rename from cpp/src/third_party/murmurhash/murmurhash64.h rename to native-sql-engine/cpp/src/third_party/murmurhash/murmurhash64.h diff --git a/cpp/src/third_party/row_wise_memory/hashMap.h b/native-sql-engine/cpp/src/third_party/row_wise_memory/hashMap.h similarity index 100% rename from cpp/src/third_party/row_wise_memory/hashMap.h rename to native-sql-engine/cpp/src/third_party/row_wise_memory/hashMap.h diff --git a/cpp/src/third_party/row_wise_memory/native_memory.h b/native-sql-engine/cpp/src/third_party/row_wise_memory/native_memory.h similarity index 100% rename from cpp/src/third_party/row_wise_memory/native_memory.h rename to native-sql-engine/cpp/src/third_party/row_wise_memory/native_memory.h diff --git a/cpp/src/third_party/row_wise_memory/unsafe_row.h b/native-sql-engine/cpp/src/third_party/row_wise_memory/unsafe_row.h similarity index 100% rename from cpp/src/third_party/row_wise_memory/unsafe_row.h rename to native-sql-engine/cpp/src/third_party/row_wise_memory/unsafe_row.h diff --git a/cpp/src/third_party/ska_sort.hpp b/native-sql-engine/cpp/src/third_party/ska_sort.hpp similarity index 100% rename from cpp/src/third_party/ska_sort.hpp rename to native-sql-engine/cpp/src/third_party/ska_sort.hpp diff --git a/cpp/src/third_party/sparsehash/dense_hash_map b/native-sql-engine/cpp/src/third_party/sparsehash/dense_hash_map similarity index 100% rename from cpp/src/third_party/sparsehash/dense_hash_map rename to native-sql-engine/cpp/src/third_party/sparsehash/dense_hash_map diff --git a/cpp/src/third_party/sparsehash/dense_hash_set b/native-sql-engine/cpp/src/third_party/sparsehash/dense_hash_set similarity index 100% rename from cpp/src/third_party/sparsehash/dense_hash_set rename to native-sql-engine/cpp/src/third_party/sparsehash/dense_hash_set diff --git a/cpp/src/third_party/sparsehash/internal/densehashtable.h b/native-sql-engine/cpp/src/third_party/sparsehash/internal/densehashtable.h similarity index 100% rename from cpp/src/third_party/sparsehash/internal/densehashtable.h rename to native-sql-engine/cpp/src/third_party/sparsehash/internal/densehashtable.h diff --git a/cpp/src/third_party/sparsehash/internal/hashtable-common.h b/native-sql-engine/cpp/src/third_party/sparsehash/internal/hashtable-common.h similarity index 100% rename from cpp/src/third_party/sparsehash/internal/hashtable-common.h rename to native-sql-engine/cpp/src/third_party/sparsehash/internal/hashtable-common.h diff --git a/cpp/src/third_party/sparsehash/internal/libc_allocator_with_realloc.h b/native-sql-engine/cpp/src/third_party/sparsehash/internal/libc_allocator_with_realloc.h similarity index 100% rename from cpp/src/third_party/sparsehash/internal/libc_allocator_with_realloc.h rename to native-sql-engine/cpp/src/third_party/sparsehash/internal/libc_allocator_with_realloc.h diff --git a/cpp/src/third_party/sparsehash/internal/sparsehashtable.h b/native-sql-engine/cpp/src/third_party/sparsehash/internal/sparsehashtable.h similarity index 100% rename from cpp/src/third_party/sparsehash/internal/sparsehashtable.h rename to native-sql-engine/cpp/src/third_party/sparsehash/internal/sparsehashtable.h diff --git a/cpp/src/third_party/sparsehash/sparse_hash_map b/native-sql-engine/cpp/src/third_party/sparsehash/sparse_hash_map similarity index 100% rename from cpp/src/third_party/sparsehash/sparse_hash_map rename to native-sql-engine/cpp/src/third_party/sparsehash/sparse_hash_map diff --git a/cpp/src/third_party/sparsehash/sparse_hash_map.h b/native-sql-engine/cpp/src/third_party/sparsehash/sparse_hash_map.h similarity index 100% rename from cpp/src/third_party/sparsehash/sparse_hash_map.h rename to native-sql-engine/cpp/src/third_party/sparsehash/sparse_hash_map.h diff --git a/cpp/src/third_party/sparsehash/sparse_hash_set b/native-sql-engine/cpp/src/third_party/sparsehash/sparse_hash_set similarity index 100% rename from cpp/src/third_party/sparsehash/sparse_hash_set rename to native-sql-engine/cpp/src/third_party/sparsehash/sparse_hash_set diff --git a/cpp/src/third_party/sparsehash/sparsetable b/native-sql-engine/cpp/src/third_party/sparsehash/sparsetable similarity index 100% rename from cpp/src/third_party/sparsehash/sparsetable rename to native-sql-engine/cpp/src/third_party/sparsehash/sparsetable diff --git a/cpp/src/third_party/sparsehash/traits b/native-sql-engine/cpp/src/third_party/sparsehash/traits similarity index 100% rename from cpp/src/third_party/sparsehash/traits rename to native-sql-engine/cpp/src/third_party/sparsehash/traits diff --git a/cpp/src/third_party/timsort.hpp b/native-sql-engine/cpp/src/third_party/timsort.hpp similarity index 100% rename from cpp/src/third_party/timsort.hpp rename to native-sql-engine/cpp/src/third_party/timsort.hpp diff --git a/cpp/src/utils/macros.h b/native-sql-engine/cpp/src/utils/macros.h similarity index 100% rename from cpp/src/utils/macros.h rename to native-sql-engine/cpp/src/utils/macros.h diff --git a/tools/gitdiff_AdaptiveSparkPlanExec.patch b/native-sql-engine/tools/gitdiff_AdaptiveSparkPlanExec.patch similarity index 100% rename from tools/gitdiff_AdaptiveSparkPlanExec.patch rename to native-sql-engine/tools/gitdiff_AdaptiveSparkPlanExec.patch diff --git a/tools/process.py b/native-sql-engine/tools/process.py similarity index 100% rename from tools/process.py rename to native-sql-engine/tools/process.py diff --git a/tools/run_ut.sh b/native-sql-engine/tools/run_ut.sh similarity index 100% rename from tools/run_ut.sh rename to native-sql-engine/tools/run_ut.sh diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000000..ecba8cd39028 --- /dev/null +++ b/pom.xml @@ -0,0 +1,40 @@ + + + + 4.0.0 + com.intel.oap + native-sql-engine-parent + 1.1.0 + pom + + Native SQL Engine Parent Pom + https://github.com/oap-project/native-sql-engine.git + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + + + arrow-data-source + native-sql-engine/core + + + diff --git a/scalastyle-config.xml b/scalastyle-config.xml new file mode 100644 index 000000000000..4892819ae997 --- /dev/null +++ b/scalastyle-config.xml @@ -0,0 +1,386 @@ + + + + + Scalastyle standard configuration + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + + + + ^FunSuite[A-Za-z]*$ + Tests must extend org.apache.spark.SparkFunSuite instead. + + + + + ^println$ + + + + + spark(.sqlContext)?.sparkContext.hadoopConfiguration + + + + + @VisibleForTesting + + + + + Runtime\.getRuntime\.addShutdownHook + + + + + mutable\.SynchronizedBuffer + + + + + Class\.forName + + + + + Await\.result + + + + + Await\.ready + + + + + (\.toUpperCase|\.toLowerCase)(?!(\(|\(Locale.ROOT\))) + + + + + throw new \w+Error\( + + + + + + JavaConversions + Instead of importing implicits in scala.collection.JavaConversions._, import + scala.collection.JavaConverters._ and use .asScala / .asJava methods + + + + org\.apache\.commons\.lang\. + Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead + of Commons Lang 2 (package org.apache.commons.lang.*) + + + + extractOpt + Use jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter + is slower. + + + + + java,scala,3rdParty,spark + javax?\..* + scala\..* + (?!org\.apache\.spark\.).* + org\.apache\.spark\..* + + + + + + COMMA + + + + + + \)\{ + + + + + (?m)^(\s*)/[*][*].*$(\r|)\n^\1 [*] + Use Javadoc style indentation for multiline comments + + + + case[^\n>]*=>\s*\{ + Omit braces in case clauses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 800> + + + + + 30 + + + + + 10 + + + + + 50 + + + + + + + + + + + -1,0,1,2,3 + + +