From 5dc121295082c31b796bea320794df7ec8991bbb Mon Sep 17 00:00:00 2001 From: Christian Zentgraf Date: Thu, 9 Jan 2025 18:48:43 -0500 Subject: [PATCH] refactor(build): Separate thrift and arrow install on Linux This PR addresses two issues. 1. Address downloading reliability from archive.apache.org by switching to github. 2. Remove the bundling of thrift from arrow and make it a separate install. This follows a PR where the bundled version of arrow still bundles thrift but has changed download locations. --- scripts/setup-centos9.sh | 29 ++++++++++++++++++++--------- scripts/setup-ubuntu.sh | 32 ++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 644f44e189ac..cac900631024 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -43,6 +43,8 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} FB_OS_VERSION="v2024.07.01.00" FMT_VERSION="10.1.1" BOOST_VERSION="boost-1.84.0" +THRIFT_VERSION="v0.16.0" +# Note: when updating arrow check if thrift needs an update as well. ARROW_VERSION="15.0.0" STEMMER_VERSION="2.2.0" DUCKDB_VERSION="v0.8.1" @@ -191,8 +193,23 @@ function install_stemmer { ) } +function install_thrift { + wget_and_untar https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift + ( + cd ${DEPENDENCY_DIR}/thrift + ./bootstrap.sh + EXTRA_CXXFLAGS="-O3 -fPIC" + # Clang will generate warnings and they need to be suppressed, otherwise the build will fail. + if [[ ${USE_CLANG} != "false" ]]; then + EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override -Wno-unused-but-set-variable" + fi + ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no --enable-tutorial=no --with-boost=${INSTALL_PREFIX} CXXFLAGS="${EXTRA_CXXFLAGS}" + make "-j${NPROC}" install + ) +} + function install_arrow { - wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow + wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz arrow cmake_install_dir arrow/cpp \ -DARROW_PARQUET=OFF \ -DARROW_WITH_THRIFT=ON \ @@ -208,21 +225,14 @@ function install_arrow { -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_BUILD_TYPE=Release \ -DARROW_BUILD_STATIC=ON \ - -DThrift_SOURCE=BUNDLED \ -DBOOST_ROOT=${INSTALL_PREFIX} - - ( - # Install thrift. - cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build - cmake --install ./ --prefix ${INSTALL_PREFIX} - ) } function install_cuda { # See https://developer.nvidia.com/cuda-downloads dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo local dashed="$(echo $1 | tr '.' '-')" - dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed + dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed } function install_velox_deps { @@ -242,6 +252,7 @@ function install_velox_deps { run_and_time install_fbthrift run_and_time install_duckdb run_and_time install_stemmer + run_and_time install_thrift run_and_time install_arrow } diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 537fc7a0325c..c2da4e6dec6b 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -75,6 +75,8 @@ function install_gcc11_if_needed { FB_OS_VERSION="v2024.07.01.00" FMT_VERSION="10.1.1" BOOST_VERSION="boost-1.84.0" +THRIFT_VERSION="v0.16.0" +# Note: when updating arrow check if thrift needs an update as well. ARROW_VERSION="15.0.0" STEMMER_VERSION="2.2.0" DUCKDB_VERSION="v0.8.1" @@ -94,13 +96,14 @@ function install_build_prerequisites { checkinstall \ git \ pkg-config \ + libtool \ wget - + if [ ! -f ${PYTHON_VENV}/pyvenv.cfg ]; then echo "Creating Python Virtual Environment at ${PYTHON_VENV}" python3 -m venv ${PYTHON_VENV} fi - source ${PYTHON_VENV}/bin/activate; + source ${PYTHON_VENV}/bin/activate; # Install to /usr/local to make it available to all users. ${SUDO} pip3 install cmake==3.28.3 @@ -244,8 +247,23 @@ function install_stemmer { ) } +function install_thrift { + wget_and_untar https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift + ( + cd ${DEPENDENCY_DIR}/thrift + ./bootstrap.sh + EXTRA_CXXFLAGS="-O3 -fPIC" + # Clang will generate warnings and they need to be suppressed, otherwise the build will fail. + if [[ ${USE_CLANG} != "false" ]]; then + EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override -Wno-unused-but-set-variable" + fi + ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no --enable-tutorial=no --with-boost=${INSTALL_PREFIX} CXXFLAGS="${EXTRA_CXXFLAGS}" + make "-j${NPROC}" install + ) +} + function install_arrow { - wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow + wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz arrow cmake_install_dir arrow/cpp \ -DARROW_PARQUET=OFF \ -DARROW_WITH_THRIFT=ON \ @@ -261,14 +279,7 @@ function install_arrow { -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_BUILD_TYPE=Release \ -DARROW_BUILD_STATIC=ON \ - -DThrift_SOURCE=BUNDLED \ -DBOOST_ROOT=${INSTALL_PREFIX} - - ( - # Install thrift. - cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build - $SUDO cmake --install ./ --prefix ${INSTALL_PREFIX} - ) } function install_cuda { @@ -296,6 +307,7 @@ function install_velox_deps { run_and_time install_conda run_and_time install_duckdb run_and_time install_stemmer + run_and_time install_thrift run_and_time install_arrow }