From 1692f5b3c9d26fcf6acfc97b6a0cd80fa02e2180 Mon Sep 17 00:00:00 2001
From: Christian Zentgraf <czentgr@us.ibm.com>
Date: Mon, 27 Jan 2025 21:55:50 -0800
Subject: [PATCH] misc(build): Add variable to configure dependency build
 parallelism (#12156)

Summary:
This change adds the BUILD_THREADS environment variable. This controls the parallelism used when building dependencies. By default, the number of cores in the host machine is used but can result in OOM kills. For example, on machines with 8 cores and 16GB ram (plus 16GB swap) OOM kills can be observed. This change allows a user to configure a flexible build automation.

Pull Request resolved: https://github.com/facebookincubator/velox/pull/12156

Reviewed By: kgpai

Differential Revision: D68746594

Pulled By: xiaoxmeng

fbshipit-source-id: 589ae16013d99ad541251f3c8ba32a1337c7a0f7
---
 README.md                         | 6 ++++++
 scripts/setup-centos9.sh          | 2 +-
 scripts/setup-helper-functions.sh | 3 ++-
 scripts/setup-macos.sh            | 2 +-
 scripts/setup-ubuntu.sh           | 2 +-
 5 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 7173e6d85579..c38dc19b4b4a 100644
--- a/README.md
+++ b/README.md
@@ -131,6 +131,12 @@ subsequent Velox builds can use the installed packages.
 *You can reuse `DEPENDENCY_INSTALL` and `INSTALL_PREFIX` for Velox clients such as Prestissimo
 by specifying a common shared directory.`*
 
+The build parallelism for dependencies can be controlled by the `BUILD_THREADS` environment
+variable and overrides the default number of parallel processes used for compiling and linking.
+The default value is the number of cores on your machine.
+This is useful if your machine has lots of cores but no matching memory to process all
+compile and link processes in parallel resulting in OOM kills by the kernel.
+
 ### Setting up on macOS
 
 On a macOS machine (either Intel or Apple silicon) you can setup and then build like so:
diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh
index cac900631024..6168c51488a1 100755
--- a/scripts/setup-centos9.sh
+++ b/scripts/setup-centos9.sh
@@ -30,7 +30,7 @@ set -efx -o pipefail
 # so that some low level types are the same size. Also, disable warnings.
 SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
 source $SCRIPTDIR/setup-helper-functions.sh
-NPROC=$(getconf _NPROCESSORS_ONLN)
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 export CXXFLAGS=$(get_cxx_flags) # Used by boost.
 export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh
index 44324ede2516..185a0dae92a2 100755
--- a/scripts/setup-helper-functions.sh
+++ b/scripts/setup-helper-functions.sh
@@ -18,6 +18,7 @@
 
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 OS_CXXFLAGS=""
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 
 function run_and_time {
   time "$@" || (echo "Failed to run $* ." ; exit 1 )
@@ -214,7 +215,7 @@ function cmake_install {
     -DBUILD_TESTING=OFF \
     "$@"
   # Exit if the build fails.
-  cmake --build "${BINARY_DIR}" || { echo 'build failed' ; exit 1; }
+  cmake --build "${BINARY_DIR}" "-j ${NPROC}" || { echo 'build failed' ; exit 1; }
   ${SUDO} cmake --install "${BINARY_DIR}"
 }
 
diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh
index efc3ab0f5621..d38f37acc5ef 100755
--- a/scripts/setup-macos.sh
+++ b/scripts/setup-macos.sh
@@ -36,7 +36,7 @@ PYTHON_VENV=${PYTHON_VENV:-"${SCRIPTDIR}/../.venv"}
 # by tagging the brew packages to be system packages.
 # This is used during package builds.
 export OS_CXXFLAGS=" -isystem $(brew --prefix)/include "
-NPROC=$(getconf _NPROCESSORS_ONLN)
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so.
diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh
index c2da4e6dec6b..3d3a7898aeac 100755
--- a/scripts/setup-ubuntu.sh
+++ b/scripts/setup-ubuntu.sh
@@ -34,7 +34,7 @@ source $SCRIPTDIR/setup-helper-functions.sh
 # are the same size.
 COMPILER_FLAGS=$(get_cxx_flags)
 export COMPILER_FLAGS
-NPROC=$(getconf _NPROCESSORS_ONLN)
+NPROC=${BUILD_THREADS:-$(getconf _NPROCESSORS_ONLN)}
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 export CMAKE_BUILD_TYPE=Release
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so.