diff --git a/README.md b/README.md index 7173e6d85579..c38dc19b4b4a 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,12 @@ subsequent Velox builds can use the installed packages. *You can reuse `DEPENDENCY_INSTALL` and `INSTALL_PREFIX` for Velox clients such as Prestissimo by specifying a common shared directory.`* +The build parallelism for dependencies can be controlled by the `BUILD_THREADS` environment +variable and overrides the default number of parallel processes used for compiling and linking. +The default value is the number of cores on your machine. +This is useful if your machine has lots of cores but no matching memory to process all +compile and link processes in parallel resulting in OOM kills by the kernel. + ### Setting up on macOS On a macOS machine (either Intel or Apple silicon) you can setup and then build like so: diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index cac900631024..6168c51488a1 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -30,7 +30,7 @@ set -efx -o pipefail # so that some low level types are the same size. Also, disable warnings. SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") source $SCRIPTDIR/setup-helper-functions.sh -NPROC=$(getconf _NPROCESSORS_ONLN) +NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)} export CXXFLAGS=$(get_cxx_flags) # Used by boost. export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index 44324ede2516..185a0dae92a2 100755 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -18,6 +18,7 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} OS_CXXFLAGS="" +NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)} function run_and_time { time "$@" || (echo "Failed to run $* ." ; exit 1 ) @@ -214,7 +215,7 @@ function cmake_install { -DBUILD_TESTING=OFF \ "$@" # Exit if the build fails. - cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; } + cmake --build "${BINARY_DIR}" "-j ${NPROC}" || { echo 'build failed' ; exit 1; } ${SUDO} cmake --install "${BINARY_DIR}" } diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index efc3ab0f5621..d38f37acc5ef 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -36,7 +36,7 @@ PYTHON_VENV=${PYTHON_VENV:-"${SCRIPTDIR}/../.venv"} # by tagging the brew packages to be system packages. # This is used during package builds. export OS_CXXFLAGS=" -isystem $(brew --prefix)/include " -NPROC=$(getconf _NPROCESSORS_ONLN) +NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)} BUILD_DUCKDB="${BUILD_DUCKDB:-true}" VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so. diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index c2da4e6dec6b..3d3a7898aeac 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -34,7 +34,7 @@ source $SCRIPTDIR/setup-helper-functions.sh # are the same size. COMPILER_FLAGS=$(get_cxx_flags) export COMPILER_FLAGS -NPROC=$(getconf _NPROCESSORS_ONLN) +NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)} BUILD_DUCKDB="${BUILD_DUCKDB:-true}" export CMAKE_BUILD_TYPE=Release VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so.