misc(build): Add variable to configure dependency build parallelism (#…

…12156)

Summary:
This change adds the BUILD_THREADS environment variable. This controls the parallelism used when building dependencies. By default, the number of cores in the host machine is used but can result in OOM kills. For example, on machines with 8 cores and 16GB ram (plus 16GB swap) OOM kills can be observed. This change allows a user to configure a flexible build automation.

Pull Request resolved: #12156

Reviewed By: kgpai

Differential Revision: D68746594

Pulled By: xiaoxmeng

fbshipit-source-id: 589ae16013d99ad541251f3c8ba32a1337c7a0f7

README.md

@@ -131,6 +131,12 @@ subsequent Velox builds can use the installed packages.
 *You can reuse `DEPENDENCY_INSTALL` and `INSTALL_PREFIX` for Velox clients such as Prestissimo
 by specifying a common shared directory.`*
 
+The build parallelism for dependencies can be controlled by the `BUILD_THREADS` environment
+variable and overrides the default number of parallel processes used for compiling and linking.
+The default value is the number of cores on your machine.
+This is useful if your machine has lots of cores but no matching memory to process all
+compile and link processes in parallel resulting in OOM kills by the kernel.
+
 ### Setting up on macOS
 
 On a macOS machine (either Intel or Apple silicon) you can setup and then build like so:

scripts/setup-centos9.sh

@@ -30,7 +30,7 @@ set -efx -o pipefail
 # so that some low level types are the same size. Also, disable warnings.
 SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
 source $SCRIPTDIR/setup-helper-functions.sh
-NPROC=$(getconf _NPROCESSORS_ONLN)
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 export CXXFLAGS=$(get_cxx_flags) # Used by boost.
 export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"

scripts/setup-helper-functions.sh

@@ -18,6 +18,7 @@
 
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 OS_CXXFLAGS=""
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 
 function run_and_time {
   time "$@" || (echo "Failed to run $* ." ; exit 1 )
@@ -214,7 +215,7 @@ function cmake_install {
     -DBUILD_TESTING=OFF \
     "$@"
   # Exit if the build fails.
-  cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; }
+  cmake --build "${BINARY_DIR}" "-j ${NPROC}" || { echo 'build failed' ; exit 1; }
   ${SUDO} cmake --install "${BINARY_DIR}"
 }
 

scripts/setup-macos.sh

@@ -36,7 +36,7 @@ PYTHON_VENV=${PYTHON_VENV:-"${SCRIPTDIR}/../.venv"}
 # by tagging the brew packages to be system packages.
 # This is used during package builds.
 export OS_CXXFLAGS=" -isystem $(brew --prefix)/include "
-NPROC=$(getconf _NPROCESSORS_ONLN)
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so.

scripts/setup-ubuntu.sh

@@ -34,7 +34,7 @@ source $SCRIPTDIR/setup-helper-functions.sh
 # are the same size.
 COMPILER_FLAGS=$(get_cxx_flags)
 export COMPILER_FLAGS
-NPROC=$(getconf _NPROCESSORS_ONLN)
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 export CMAKE_BUILD_TYPE=Release
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so.