test runner wont be merged

.github/workflows/build_and_test.yml

@@ -0,0 +1,55 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: build and test
+
+on:
+  push:
+    branches: 
+      - developer
+  pull_request:
+    branches: 
+      - developer
+
+jobs:
+  build:
+    runs-on: velox_ci
+    steps:
+      - name: Clean up CI directory
+        working-directory: ${{github.workspace}}
+        run: |
+          rm -rf * .[!.]*
+
+      - uses: actions/checkout@v3
+
+      - name: Update submodules
+        run: |
+          git submodule sync --recursive
+          git submodule update --init --recursive
+
+      - name: Build
+        run: |
+          make release NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
+
+      - name: Run Unit Tests
+        run: |
+          cd _build/release && ctest -j 16
+
+      # cd _build/release && ctest -j 16 -VV --output-on-failure
+      - name: Run Fuzzer Tests
+        run: |
+          _build/release/velox/expression/tests/velox_expression_fuzzer_test --seed 123456 --duration_sec 60 --logtostderr=1 --minloglevel=0
+
+      - name: Run Example Binaries
+        run: |
+          find _build/release/velox/examples/ -maxdepth 1 -type f -executable -exec "{}" \;

.github/workflows/check.yml

@@ -0,0 +1,58 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: check
+
+on:
+  push:
+    branches: 
+      - developer
+  pull_request:
+    branches: 
+      - developer
+
+
+jobs:
+  build:
+    runs-on: velox_ci
+    steps:
+      - name: Clean up CI directory
+        working-directory: ${{github.workspace}}
+        run: |
+          rm -rf * .[!.]*
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: format-check
+        run: |
+          if ! make format-check; then
+            make format-fix
+            echo -e "\n==== Apply using:"
+            echo "patch -p1 \<<EOF"
+            git --no-pager diff
+            echo "EOF"
+            false
+          fi
+
+      - name: header-check
+        run: |
+          if ! make header-check; then
+            make header-fix
+            echo -e "\n==== Apply using:"
+            echo "patch -p1 \<<EOF"
+            git --no-pager diff
+            echo "EOF"
+            false
+          fi

.gitignore

@@ -311,3 +311,7 @@ src/amalgamation/
 .cproject
 .settings
 ~
+
+# build deps
+fmt
+folly

CMakeLists.txt

@@ -41,7 +41,7 @@ option(VELOX_ENABLE_PARSE "Build parser used for unit tests." ON)
 option(VELOX_ENABLE_EXAMPLES
        "Build examples. This will enable VELOX_ENABLE_EXPRESSION automatically."
        OFF)
-option(VELOX_ENABLE_SUBSTRAIT "Buid Substrait-to-Velox converter." OFF)
+option(VELOX_ENABLE_SUBSTRAIT "Buid Substrait-to-Velox converter." ON)
 option(VELOX_ENABLE_BENCHMARKS "Build velox top level benchmarks." OFF)
 option(VELOX_ENABLE_BENCHMARKS_BASIC "Build velox basic benchmarks." OFF)
 option(VELOX_ENABLE_S3 "Build S3 Connector" OFF)

Makefile

@@ -116,17 +116,17 @@ fuzzertest: debug
 		--minloglevel=0
 
 format-fix: 			#: Fix formatting issues in the main branch
-	scripts/check.py format main --fix
+	scripts/check.py format developer --fix
 
 format-check: 			#: Check for formatting issues on the main branch
 	clang-format --version
-	scripts/check.py format main
+	scripts/check.py format developer
 
 header-fix:				#: Fix license header issues in the current branch
-	scripts/check.py header main --fix
+	scripts/check.py header developer --fix
 
 header-check:			#: Check for license header issues on the main branch
-	scripts/check.py header main
+	scripts/check.py header developer
 
 circleci-container:			#: Build the linux container for CircleCi
 	$(MAKE) linux-container CONTAINER_NAME=circleci

scripts/check.py

@@ -173,7 +173,7 @@ def get_commit(files):
     if files == "commit":
         return "HEAD^"
 
-    if files == "main" or files == "master":
+    if files == "main" or files == "master" or files == "developer":
         return util.run(f"git merge-base origin/{files} HEAD")[1]
 
     return ""
@@ -222,6 +222,7 @@ def add_options(parser):
 
     branch_parser = add_check_options(files, "main")
     branch_parser = add_check_options(files, "master")
+    branch_parser = add_check_options(files, "developer")
     commit_parser = add_check_options(files, "commit")
 
 

scripts/docker/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:20.04 AS base
+
+USER root
+WORKDIR /
+
+SHELL ["/bin/bash", "-c"]
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY ./ci-setup.sh ./scripts/ci-setup.sh
+COPY ./ci-setup-helper-functions.sh ./scripts/ci-setup-helper-functions.sh
+
+RUN bash ./scripts/ci-setup.sh

scripts/docker/ci-setup-helper-functions.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# github_checkout $REPO $VERSION $GIT_CLONE_PARAMS clones or re-uses an existing clone of the
+# specified repo, checking out the requested version.
+function github_checkout {
+  local REPO=$1
+  shift
+  local VERSION=$1
+  shift
+  local GIT_CLONE_PARAMS=$@
+  local DIRNAME=$(basename $REPO)
+  cd "${DEPENDENCY_DIR}"
+  if [ -z "${DIRNAME}" ]; then
+    echo "Failed to get repo name from ${REPO}"
+    exit 1
+  fi
+  if [ -d "${DIRNAME}" ] && prompt "${DIRNAME} already exists. Delete?"; then
+    rm -rf "${DIRNAME}"
+  fi
+  if [ ! -d "${DIRNAME}" ]; then
+    git clone -q -b $VERSION $GIT_CLONE_PARAMS "https://github.com/${REPO}.git"
+  fi
+  cd "${DIRNAME}"
+}
+
+
+# get_cxx_flags [$CPU_ARCH]
+# Sets and exports the variable VELOX_CXX_FLAGS with appropriate compiler flags.
+# If $CPU_ARCH is set then we use that else we determine best possible set of flags
+# to use based on current cpu architecture.
+# The goal of this function is to consolidate all architecture specific flags to one
+# location.
+# The values that CPU_ARCH can take are as follows:
+#   arm64  : Target Apple silicon.
+#   aarch64: Target general 64 bit arm cpus.
+#   avx:     Target Intel CPUs with AVX.
+#   sse:     Target Intel CPUs with sse.
+# Echo's the appropriate compiler flags which can be captured as so
+# CXX_FLAGS=$(get_cxx_flags) or
+# CXX_FLAGS=$(get_cxx_flags "avx")
+
+function get_cxx_flags {
+  local CPU_ARCH=$1
+
+  local OS
+  OS=$(uname)
+  local MACHINE
+  MACHINE=$(uname -m)
+
+  if [ -z "$CPU_ARCH" ]; then
+
+    if [ "$OS" = "Darwin" ]; then
+
+      if [ "$MACHINE" = "x86_64" ]; then
+        local CPU_CAPABILITIES
+        CPU_CAPABILITIES=$(sysctl -a | grep machdep.cpu.features | awk '{print tolower($0)}')
+
+        if [[ $CPU_CAPABILITIES =~ "avx" ]]; then
+          CPU_ARCH="avx"
+        else
+          CPU_ARCH="sse"
+        fi
+
+      elif [[ $(sysctl -a | grep machdep.cpu.brand_string) =~ "Apple" ]]; then
+        # Apple silicon.
+        CPU_ARCH="arm64"
+      fi
+    else [ "$OS" = "Linux" ];
+
+      local CPU_CAPABILITIES
+      CPU_CAPABILITIES=$(cat /proc/cpuinfo | grep flags | head -n 1| awk '{print tolower($0)}')
+
+      if [[ "$CPU_CAPABILITIES" =~ "avx" ]]; then
+            CPU_ARCH="avx"
+      elif [[ "$CPU_CAPABILITIES" =~ "sse" ]]; then
+            CPU_ARCH="sse"
+      elif [ "$MACHINE" = "aarch64" ]; then
+            CPU_ARCH="aarch64"
+      fi
+    fi
+  fi
+
+  case $CPU_ARCH in
+
+    "arm64")
+      echo -n "-mcpu=apple-m1+crc -std=c++17"
+    ;;
+
+    "avx")
+      echo -n "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17"
+    ;;
+
+    "sse")
+      echo -n "-msse4.2 -std=c++17"
+    ;;
+
+    "aarch64")
+      echo -n "-march=armv8-a+crc+crypto -std=c++17"
+    ;;
+  *)
+    echo -n "Architecture not supported!"
+  esac
+
+}
+
+function cmake_install {
+  local NAME=$(basename "$(pwd)")
+  local BINARY_DIR=_build
+  if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then
+    rm -rf "${BINARY_DIR}"
+  fi
+  mkdir -p "${BINARY_DIR}"
+  CPU_TARGET="${CPU_TARGET:-avx}"
+  COMPILER_FLAGS=$(get_cxx_flags $CPU_TARGET)
+
+  # CMAKE_POSITION_INDEPENDENT_CODE is required so that Velox can be built into dynamic libraries \
+  cmake -Wno-dev -B"${BINARY_DIR}" \
+    -GNinja \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+    -DCMAKE_CXX_STANDARD=17 \
+    "${INSTALL_PREFIX+-DCMAKE_PREFIX_PATH=}${INSTALL_PREFIX-}" \
+    "${INSTALL_PREFIX+-DCMAKE_INSTALL_PREFIX=}${INSTALL_PREFIX-}" \
+    -DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" \
+    -DBUILD_TESTING=OFF \
+    "$@"
+  ninja -C "${BINARY_DIR}" install
+}
+