Merge pull request prestodb#1 from prestodb/master

rebase to prestodb/presto

.circleci/config.yml

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+version: 2.1
+
+setup: true
+
+orbs:
+  path-filtering: circleci/[email protected]
+
+workflows:
+  always-run:
+    jobs:
+      - path-filtering/filter:
+          name: check-updated-files
+          # 3-column, whitespace-delimited mapping. One mapping per line
+          # <regex path-to-test> <parameter-to-set> <value-of-pipeline-parameter>
+          mapping: |
+            presto-native-execution/.* run_native_specific_jobs true
+          base-revision: master
+          config-path: .circleci/continue_config.yml

.circleci/continue_config.yml

@@ -0,0 +1,275 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+version: 2.1
+
+parameters:
+  run_native_specific_jobs:
+    type: boolean
+    default: false
+
+workflows:
+  version: 2
+  dist-compile:
+    jobs:
+      - linux-build-and-unit-test
+      - linux-presto-e2e-tests:
+          requires:
+            - linux-build-and-unit-test
+      - linux-spark-e2e-tests:
+          requires:
+            - linux-build-and-unit-test
+
+  conditionals:
+    when: << pipeline.parameters.run_native_specific_jobs >>
+    jobs:
+      - format-check
+      - header-check
+      - linux-build-all
+
+commands:
+  maven_install:
+    description: "Maven Install"
+    parameters:
+      maven_install_opts:
+        type: string
+      maven_fast_install:
+        type: string
+    steps:
+      - run:
+          name: "Maven Install"
+          command: |
+            export MAVEN_OPTS=<< parameters.maven_install_opts>>
+            # Sometimes mvn build fails because of network failures. Add logic to rerun the build to avoid non PR-relelated CI failures.
+            for i in $(seq 1 3); do ./mvnw clean install << parameters.maven_fast_install>> -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
+
+executors:
+  build:
+    docker:
+      - image: prestocpp/prestocpp-avx-centos:root-20230613
+    resource_class: 2xlarge
+    environment:
+      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
+  check:
+    docker:
+      - image: prestocpp/velox-check:mikesh-20210609
+  macos-intel:
+    macos:
+      xcode: "14.3.0"
+    resource_class: macos.x86.medium.gen2
+
+jobs:
+  linux-build-and-unit-test:
+    executor: build
+    steps:
+      - checkout
+      - run:
+          name: "Update velox"
+          command: |
+            cd presto-native-execution
+            make velox-submodule
+      - run:
+          name: "Calculate merge-base date for CCache"
+          command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
+      - restore_cache:
+          name: "Restore CCache cache"
+          keys:
+            - native-exe-linux-ccache-{{ arch }}-{{ checksum "merge-base-date" }}
+      - run:
+          name: Build
+          command: |
+            mkdir -p .ccache
+            export CCACHE_DIR=$(realpath .ccache)
+            ccache -sz -M 8Gi
+            source /opt/rh/gcc-toolset-9/enable
+            cd presto-native-execution
+            cmake \
+              -B _build/debug \
+              -GNinja \
+              -DTREAT_WARNINGS_AS_ERRORS=1 \
+              -DENABLE_ALL_WARNINGS=1 \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DPRESTO_ENABLE_PARQUET=ON \
+              -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
+              -DCMAKE_PREFIX_PATH=/usr/local \
+              -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+            ninja -C _build/debug -j 8
+            ccache -s
+      - save_cache:
+          name: "Save CCache cache"
+          key: native-exe-linux-ccache-{{ arch }}-{{ checksum "merge-base-date" }}
+          paths:
+            - .ccache/
+      - run:
+          name: 'Run Unit Tests'
+          command: |
+            cd presto-native-execution/_build/debug
+            ctest -j 8 -VV --output-on-failure --exclude-regex velox.*
+      - persist_to_workspace:
+          root: presto-native-execution
+          paths:
+            - _build/debug/presto_cpp/main/presto_server
+
+  linux-presto-e2e-tests:
+    executor: build
+    parallelism: 5
+    steps:
+      - checkout
+      - attach_workspace:
+          at: presto-native-execution
+      - maven_install:
+          maven_install_opts: ${MAVEN_INSTALL_OPTS}
+          maven_fast_install: ${MAVEN_FAST_INSTALL}
+
+      - run:
+          name: 'Run presto-native e2e tests'
+          command: |
+            export PRESTO_SERVER_PATH="${HOME}/project/presto-native-execution/_build/debug/presto_cpp/main/presto_server"
+            export TEMP_PATH="/tmp"
+            TESTFILES=$(circleci tests glob "presto-native-execution/src/test/**/TestPrestoNative*.java" | circleci tests split --split-by=timings)
+            # Convert file paths to comma separated class names
+            export TESTCLASSES=
+            for test_file in $TESTFILES
+            do
+              tmp=${test_file##*/}
+              test_class=${tmp%%\.*}
+              export TESTCLASSES="${TESTCLASSES},$test_class"
+            done
+            export TESTCLASSES=${TESTCLASSES#,}
+            if [ ! -z $TESTCLASSES ]; then
+              mvn test ${MAVEN_TEST} -pl 'presto-native-execution' -Dtest="${TESTCLASSES}" -DPRESTO_SERVER=${PRESTO_SERVER_PATH} -DDATA_DIR=${TEMP_PATH} -Duser.timezone=America/Bahia_Banderas -T1C
+            fi
+
+  linux-spark-e2e-tests:
+    executor: build
+    parallelism: 5
+    steps:
+      - checkout
+      - attach_workspace:
+          at: presto-native-execution
+      - maven_install:
+          maven_install_opts: ${MAVEN_INSTALL_OPTS}
+          maven_fast_install: ${MAVEN_FAST_INSTALL}
+      - run:
+          name: 'Run spark e2e tests'
+          command: |
+            export PRESTO_SERVER_PATH="${HOME}/project/presto-native-execution/_build/debug/presto_cpp/main/presto_server"
+            export TEMP_PATH="/tmp"
+            TESTFILES=$(circleci tests glob "presto-native-execution/src/test/**/TestPrestoSpark*.java" | circleci tests split --split-by=timings)
+            # Convert file paths to comma separated class names
+            export TESTCLASSES=
+            for test_file in $TESTFILES
+            do
+              tmp=${test_file##*/}
+              test_class=${tmp%%\.*}
+              export TESTCLASSES="${TESTCLASSES},$test_class"
+            done
+            export TESTCLASSES=${TESTCLASSES#,}
+            if [ ! -z $TESTCLASSES ]; then
+              mvn test ${MAVEN_TEST} -pl 'presto-native-execution' -Dtest="${TESTCLASSES}" -DPRESTO_SERVER=${PRESTO_SERVER_PATH} -DDATA_DIR=${TEMP_PATH} -Duser.timezone=America/Bahia_Banderas -T1C
+            fi
+
+  linux-build-all:
+    executor: build
+    steps:
+      - checkout
+      - run:
+          name: "Update velox submodule"
+          command: |
+            cd presto-native-execution
+            make velox-submodule
+      - run:
+          name: "Install S3 adapter dependencies"
+          command: |
+            mkdir -p ${HOME}/adapter-deps/install
+            source /opt/rh/gcc-toolset-9/enable
+            set -xu
+            cd presto-native-execution
+            DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
+      - run:
+          name: "Build All"
+          command: |
+            source /opt/rh/gcc-toolset-9/enable
+            cd presto-native-execution
+            cmake \
+              -B _build/release \
+              -GNinja \
+              -DAWSSDK_ROOT_DIR=${HOME}/adapter-deps/install \
+              -DTREAT_WARNINGS_AS_ERRORS=1 \
+              -DENABLE_ALL_WARNINGS=1 \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DPRESTO_ENABLE_PARQUET=ON \
+              -DPRESTO_ENABLE_S3=ON \
+              -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
+              -DCMAKE_PREFIX_PATH=/usr/local \
+              -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+            ninja -C _build/release -j 8
+
+  format-check:
+    executor: check
+    steps:
+      - checkout
+      - run:
+          name: "Update velox"
+          command: |
+            cd presto-native-execution
+            make velox-submodule
+      - run:
+          name: Check formatting
+          command: |
+            cd presto-native-execution
+            make format-check
+
+  header-check:
+    executor: check
+    steps:
+      - checkout
+      - run:
+          name: "Update velox"
+          command: |
+            cd presto-native-execution
+            make velox-submodule
+      - run:
+          name: "Check license headers"
+          command: |
+            cd presto-native-execution
+            make header-check
+
+  native-specific-macos:
+    executor: macos-intel
+    steps:
+      - checkout
+      - run:
+          name: "Update submodules"
+          command: |
+            cd presto-native-execution
+            make submodules
+      - run:
+          name: "Setup MacOS"
+          command: |
+            set -xu
+            mkdir ~/deps ~/deps-src
+            git clone --depth 1 https://github.com/Homebrew/brew ~/deps
+            PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
+            # Calculate the prefix path before we delete brew's repos and taps.
+            echo "$(pwd)/deps;$(brew --prefix [email protected]);$(brew --prefix icu4c)" > ~/deps/PREFIX_PATH
+            rm -rf ~/deps/.git ~/deps/Library/Taps/  # Reduce cache size by 70%.
+            rm -rf ~/deps-src
+      - run:
+          name: "Build presto_cpp on MacOS"
+          command: |
+            export PATH=~/deps/bin:${PATH}
+            cd presto-native-execution
+            cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=$(cat ~/deps/PREFIX_PATH) -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+            ninja -C _build/debug

.clang-format

@@ -0,0 +1,87 @@
+---
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ FOR_EACH, FOR_EACH_ENUMERATE, FOR_EACH_KV, FOR_EACH_R, FOR_EACH_RANGE, ]
+IncludeCategories:
+  - Regex:           '^<.*\.h(pp)?>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...