diff --git a/README.md b/README.md index ea043fb..507d70d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ linux Requires [docker](https://www.docker.com/) + Change directories to the root of this repository. -+ `docker run -v "$(pwd):/pantsbuild-binaries" -rm -it --entrypoint /bin/bash python:2.7.13-wheezy && cd /pantsbuild-binaries` to pop yourself in a controlled image back at this repo's root ++ `docker run -v "$(pwd):/pantsbuild-binaries" --rm -it --entrypoint /bin/bash pantsbuild/centos6:latest && cd /pantsbuild-binaries` to pop yourself in a controlled image back at this repo's root + Run the build-\*.sh script corresponding to the binary you wish to build + Manually move the binary from the build tree to its home in build-support/... diff --git a/build-binutils.sh b/build-binutils.sh new file mode 100755 index 0000000..a128fbb --- /dev/null +++ b/build-binutils.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +source ./utils.bash + +set_strict_mode + +function fetch_extract_binutils_source_release { + local -r extracted_dirname="binutils-${BINUTILS_VERSION}" + local -r archive_filename="${extracted_dirname}.tar.xz" + local -r release_url="https://ftpmirror.gnu.org/binutils/${archive_filename}" + + local -r downloaded_archive="$(curl_file_with_fail "$release_url" "$archive_filename")" + extract_for "$downloaded_archive" "$extracted_dirname" +} + +function build_binutils_with_configure { + local -a configure_cmd_line=("$@") + + "${configure_cmd_line[@]}" + + make "-j${MAKE_JOBS}" + + make install +} + +function build_linux { + local -r source_extracted_abs="$(fetch_extract_binutils_source_release)" + + local -r build_dir_abs="$(mkdirp_absolute_path 'binutils-build')" + local -r install_dir_abs="$(mkdirp_absolute_path 'binutils-install')" + + with_pushd >&2 "$build_dir_abs" \ + build_binutils_with_configure \ + "${source_extracted_abs}/configure" \ + --prefix="$install_dir_abs" + + with_pushd "$install_dir_abs" \ + create_gz_package 'binutils' bin +} + +## Interpret arguments and execute build. + +readonly TARGET_PLATFORM="$1" BINUTILS_VERSION="$2" + +MAKE_JOBS="${MAKE_JOBS:-2}" + +case "$TARGET_PLATFORM" in + linux) + with_pushd "$(mkdirp_absolute_path "binutils-${BINUTILS_VERSION}-linux")" \ + build_linux + ;; + *) + die "binutils doesnot support building for '${TARGET_PLATFORM}'" + ;; +esac diff --git a/build-clang.sh b/build-clang.sh new file mode 100755 index 0000000..3f5b35a --- /dev/null +++ b/build-clang.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +source ./utils.bash + +set_strict_mode + +## Build for OSX from LLVM's binary release package. + +function build_osx { + + local -r normal_release_dirname="clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin" + # The top-level directory in the release archive is not always the same across + # releases. The `-final-` part may be to signify that this is the last release + # of that major version. + local -r final_release_dirname="clang+llvm-${LLVM_VERSION}-final-x86_64-apple-darwin" + + local -r archive_filename="${normal_release_dirname}.tar.xz" + local -r release_url="https://releases.llvm.org/${LLVM_VERSION}/${archive_filename}" + + local -r downloaded_archive="$(curl_file_with_fail "$release_url" "$archive_filename")" + local -r extracted_dir="$(extract_for "$downloaded_archive" "$normal_release_dirname" "$final_release_dirname")" + + with_pushd "$extracted_dir" \ + create_gz_package 'clang' +} + + +## Build for Linux from LLVM's multi-part source release packages. + +function fetch_extract_llvm_source_release { + local -r extracted_dirname="$1" + + local -r archive_filename="${extracted_dirname}.tar.xz" + local -r release_url="https://releases.llvm.org/${LLVM_VERSION}/${archive_filename}" + + local -r downloaded_archive="$(curl_file_with_fail "$release_url" "$archive_filename")" + extract_for "$downloaded_archive" "$extracted_dirname" +} + +function build_llvm_with_cmake { + local -r cmake_exe="$1" install_dir="$2" cfe_dir="$3" llvm_dir="$4" + + # This didn't immediately compile right off the bat -- I'd recommend to anyone + # watching that Homebrew formulas are good places to learn about how to provide + # high-quality toolchains on OSX. I found https://llvm.org/docs/CMake.html + # "helpful" for this line as well. + "$cmake_exe" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$install_dir_abs" \ + -DLLVM_EXTERNAL_CLANG_SOURCE_DIR="$cfe_dir" \ + -DLLVM_EXTERNAL_PROJECTS='clang' \ + -DLLVM_TARGETS_TO_BUILD='X86' \ + -DBACKEND_PACKAGE_STRING="Pants-packaged LLVM for ${TARGET_PLATFORM}, version ${LLVM_VERSION}" \ + "$llvm_dir" + + # NB: There appear to be race conditions when running make with any + # parallelism here in a Docker image. + make "-j${MAKE_JOBS}" + + make install +} + +function build_linux { + local -r cmake_exe="$1" + + # Properties of the downloaded release tarball. + local -r llvm_src_dirname="llvm-${LLVM_VERSION}.src" + local -r cfe_src_dirname="cfe-${LLVM_VERSION}.src" + + # Set up an out-of-tree build and install + local -r build_dir_abs="$(mkdirp_absolute_path 'clang-llvm-build')" + local -r install_dir_abs="$(mkdirp_absolute_path 'clang-llvm-install')" + + # LLVM requires you to download the source for LLVM and the Clang frontend + # separately. One alternative is checking out their SVN repo, but that takes + # much longer. + local -r llvm_src_extracted_abs="$(fetch_extract_llvm_source_release "$llvm_src_dirname")" + local -r cfe_src_extracted_abs="$(fetch_extract_llvm_source_release "$cfe_src_dirname")" + + with_pushd >&2 "$build_dir_abs" \ + build_llvm_with_cmake "$cmake_exe" "$install_dir_abs" "$cfe_src_extracted_abs" "$llvm_src_extracted_abs" + + with_pushd "$install_dir_abs" \ + create_gz_package 'clang' +} + +function validate_cmake { + if [[ ! -f "$CMAKE_EXE" ]]; then + die_here < /dev/null - ln -s "../10.7/3.9.5" "3.9.5" + ln -sf "../10.7/3.9.5" "3.9.5" popd > /dev/null done diff --git a/build-cmake.sh b/build-cmake.sh new file mode 100755 index 0000000..73137bf --- /dev/null +++ b/build-cmake.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +source ./utils.bash + +set_strict_mode + +readonly CMAKE_VERSION_PATTERN='^([0-9]+\.[0-9]+)\.[0-9]+$' + +function extract_minor_version { + local -r version_string="$1" + + echo "$version_string" | sed -re "s#${CMAKE_VERSION_PATTERN}#\\1#g" +} + +function fetch_extract_cmake_binary_release { + local -r extracted_dirname="$1" + + local -r cmake_minor_version="$(extract_minor_version "$CMAKE_VERSION")" + + local -r archive_filename="${extracted_dirname}.tar.gz" + local -r release_url="https://cmake.org/files/v${cmake_minor_version}/${archive_filename}" + + local -r downloaded_archive="$(curl_file_with_fail "$release_url" "$archive_filename")" + extract_for "$downloaded_archive" "$extracted_dirname" +} + +function package_cmake { + local -r installed_dir_abs="$1" + + with_pushd "$installed_dir_abs" \ + create_gz_package 'cmake' bin share +} + +function build_osx { + local -r cmake_platform_description="cmake-${CMAKE_VERSION}-Darwin-x86_64" + + local -r extracted_dir="$(fetch_extract_cmake_binary_release "$cmake_platform_description")" + + package_cmake "${extracted_dir}/CMake.app/Contents" +} + +function build_linux { + local -r cmake_platform_description="cmake-${CMAKE_VERSION}-Linux-x86_64" + + local -r extracted_dir="$(fetch_extract_cmake_binary_release "$cmake_platform_description")" + + package_cmake "$extracted_dir" +} + + +## Interpret arguments and execute build. + +readonly TARGET_PLATFORM="$1" CMAKE_VERSION="$2" + +case "$TARGET_PLATFORM" in + osx) + with_pushd "$(mkdirp_absolute_path "cmake-${CMAKE_VERSION}-osx")" \ + build_osx + ;; + linux) + with_pushd "$(mkdirp_absolute_path "cmake-${CMAKE_VERSION}-linux")" \ + build_linux + ;; + *) + die "cmake does not support building for '${TARGET_PLATFORM}'" + ;; +esac diff --git a/build-gcc.sh b/build-gcc.sh new file mode 100755 index 0000000..2888e5b --- /dev/null +++ b/build-gcc.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +source ./utils.bash + +set_strict_mode + +function fetch_extract_gcc_source_release { + local -r extracted_dirname="gcc-${GCC_VERSION}" + local -r archive_filename="${extracted_dirname}.tar.gz" + local -r release_url="https://ftpmirror.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/${archive_filename}" + + local -r downloaded_archive="$(curl_file_with_fail "$release_url" "$archive_filename")" + extract_for "$downloaded_archive" "$extracted_dirname" +} + +function build_gcc_with_configure { + local -a configure_cmd_line=("$@") + + "${configure_cmd_line[@]}" + + make "-j${MAKE_JOBS}" + + make install +} + +function build_gcc_out_of_tree { + local -a configure_args=("$@") + + local -r source_extracted_abs="$(fetch_extract_gcc_source_release)" + + with_pushd >&2 "$source_extracted_abs" \ + ./contrib/download_prerequisites + + local -r build_dir_abs="$(mkdirp_absolute_path 'gcc-build')" + local -r install_dir_abs="$(mkdirp_absolute_path 'gcc-install')" + + with_pushd >&2 "$build_dir_abs" \ + build_gcc_with_configure \ + "${source_extracted_abs}/configure" \ + --prefix="$install_dir_abs" \ + "${configure_args[@]}" + + with_pushd "$install_dir_abs" \ + create_gz_package 'gcc' +} + +function build_osx { + build_gcc_out_of_tree \ + --host='x86_64-apple-darwin' \ + --target='x86_64-apple-darwin' \ + AR="$(which ar)" \ + "${CONFIGURE_BASE_ARGS[@]}" +} + +function build_linux { + build_gcc_out_of_tree \ + "${CONFIGURE_BASE_ARGS[@]}" +} + +## Interpret arguments and execute build. + +readonly TARGET_PLATFORM="$1" GCC_VERSION="$2" + +readonly MAKE_JOBS="${MAKE_JOBS:-2}" + +readonly SUPPORTED_LANGS='c,c++,objc,obj-c++,fortran' + +readonly -a CONFIGURE_BASE_ARGS=( + --disable-multilib + --enable-languages="${SUPPORTED_LANGS}" + --enable-checking='release' + --with-pkgversion="Pants-packaged GCC (${GCC_VERSION})" + --with-bugurl='https://github.com/pantsbuild/pants/issues' +) + +case "$TARGET_PLATFORM" in + osx) + with_pushd "$(mkdirp_absolute_path "gcc-${GCC_VERSION}-osx")" \ + build_osx + ;; + linux) + with_pushd "$(mkdirp_absolute_path "gcc-${GCC_VERSION}-linux")" \ + build_linux + ;; + *) + die "gcc does not support building for '${TARGET_PLATFORM}'" + ;; +esac diff --git a/build-support/bin/binutils/linux/x86_64/2.30/binutils.tar.gz b/build-support/bin/binutils/linux/x86_64/2.30/binutils.tar.gz new file mode 100644 index 0000000..64c64b2 Binary files /dev/null and b/build-support/bin/binutils/linux/x86_64/2.30/binutils.tar.gz differ diff --git a/build-support/bin/buildozer/linux/x86_64/0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3/buildozer b/build-support/bin/buildozer/linux/x86_64/0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3/buildozer new file mode 100755 index 0000000..c1dcabc Binary files /dev/null and b/build-support/bin/buildozer/linux/x86_64/0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3/buildozer differ diff --git a/build-support/bin/buildozer/mac/10.13/0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3/buildozer b/build-support/bin/buildozer/mac/10.13/0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3/buildozer new file mode 100755 index 0000000..bafa4f4 Binary files /dev/null and b/build-support/bin/buildozer/mac/10.13/0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3/buildozer differ diff --git a/build-support/bin/cmake/linux/x86_64/3.9.5/cmake.tar.gz b/build-support/bin/cmake/linux/x86_64/3.9.5/cmake.tar.gz index 3456fb8..dfa7bc7 100644 Binary files a/build-support/bin/cmake/linux/x86_64/3.9.5/cmake.tar.gz and b/build-support/bin/cmake/linux/x86_64/3.9.5/cmake.tar.gz differ diff --git a/build-support/bin/cmake/mac/10.7/3.9.5/3.9.5 b/build-support/bin/cmake/mac/10.7/3.9.5/3.9.5 new file mode 120000 index 0000000..5cbd9fb --- /dev/null +++ b/build-support/bin/cmake/mac/10.7/3.9.5/3.9.5 @@ -0,0 +1 @@ +../10.7/3.9.5 \ No newline at end of file diff --git a/build-support/bin/cmake/mac/10.7/3.9.5/cmake.tar.gz b/build-support/bin/cmake/mac/10.7/3.9.5/cmake.tar.gz index 7db43b5..6ec05aa 100644 Binary files a/build-support/bin/cmake/mac/10.7/3.9.5/cmake.tar.gz and b/build-support/bin/cmake/mac/10.7/3.9.5/cmake.tar.gz differ diff --git a/populate-native-toolchain.sh b/populate-native-toolchain.sh new file mode 100755 index 0000000..7c3816a --- /dev/null +++ b/populate-native-toolchain.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +source ./utils.bash +source ./vars.bash + +set_strict_mode + +readonly -a BINUTILS_VERSIONS=( + '2.30' +) + +readonly -a CLANG_VERSIONS=( + '5.0.1' + '6.0.0' +) + +readonly -a GCC_VERSIONS=( + '7.3.0' +) + +readonly CMAKE_VERSION_FOR_CLANG='3.9.5' + +function build_cmake_for_clang_bootstrap { + local -r cmake_linux_archive_abs="$(./build-cmake.sh linux "$CMAKE_VERSION_FOR_CLANG")" + + local -r workdir='cmake-for-linux-packages' + + with_pushd "$(mkdirp_absolute_path "$workdir")" \ + extract_for "$cmake_linux_archive_abs" 'bin/cmake' +} + +function package_binutils { + for binutils_version in "${BINUTILS_VERSIONS[@]}"; do + binutils_linux_pkg="$(./build-binutils.sh linux "$binutils_version")" + get_create_linux_supportdirs_stdout binutils "$binutils_version" \ + | deploy_package_to_dirs_from_stdin "$binutils_linux_pkg" + done +} + +function package_gcc { + for gcc_version in "${GCC_VERSIONS[@]}"; do + gcc_linux_pkg="$(./build-gcc.sh linux "$gcc_version")" + get_create_linux_supportdirs_stdout gcc "$gcc_version" \ + | deploy_package_to_dirs_from_stdin "$gcc_linux_pkg" + + gcc_osx_pkg="$(./build-gcc.sh osx "$gcc_version")" + get_create_osx_supportdirs_stdout gcc "$gcc_version" \ + | deploy_package_to_dirs_from_stdin "$gcc_osx_pkg" + done +} + +function package_clang { + readonly cmake_linux_bin="$(build_cmake_for_clang_bootstrap)" + + for clang_version in "${CLANG_VERSIONS[@]}"; do + clang_osx_pkg="$(./build-clang.sh osx "$clang_version")" + get_create_osx_supportdirs_stdout clang "$clang_version" \ + | deploy_package_to_dirs_from_stdin "$clang_osx_pkg" + + clang_linux_pkg="$(CMAKE_EXE="$cmake_linux_bin" \ + ./build-clang.sh linux "$clang_version")" + get_create_linux_supportdirs_stdout clang "$clang_version" \ + | deploy_package_to_dirs_from_stdin "$clang_linux_pkg" + done +} + +readonly TOOL_NAME="$1" + +case "$TOOL_NAME" in + binutils) + package_binutils + ;; + clang) + package_clang + ;; + gcc) + package_gcc + ;; + *) + die "Tool not recognized: '${TOOL_NAME}'" + ;; +esac diff --git a/utils.bash b/utils.bash new file mode 100644 index 0000000..bdf738a --- /dev/null +++ b/utils.bash @@ -0,0 +1,147 @@ +# Not an executable. Can be sourced by other scripts in this repo for some free +# error checking for common operations. Uses bash-specific features including +# `local`. + +# see bash(1) +function set_strict_mode { + set -euxo pipefail +} + +# Display a message to stderr. +function warn { + echo >&2 "$@" +} + +# Display a message to stderr, then exit with failure. +function die { + echo >&2 "$@" + exit 1 +} + +# Display a message read from stdin to stderr, then exit with failure. This can +# be used with heredocs (<&2 + exit 1 +} + +# Check for the existence of a command, and fail if it's not there. If a command +# has variations in functionality between Linux and OSX, additional checks may +# be required. +function check_cmd_or_err { + local -r cmd_name="$1" + if ! hash "$cmd_name"; then + die_here <&2 -L --fail -O "$url" + get_existing_absolute_path "$expected_outfile" +} + +# Make a new directory (that may already exist) and echo the absolute path. +function mkdirp_absolute_path { + local -r dir_relpath="$1" + mkdir -p "$dir_relpath" + get_existing_absolute_path "$dir_relpath" +} + +# TODO: note that expected_output can be any of the extracted entries in the +# archive, not just a "root dir" or whatever +function do_extract { + local -r archive_path="$1" + + case "$archive_path" in + *.tar.xz) + check_cmd_or_err 'xz' + tar xf "$archive_path" + ;; + *.tar.gz) + tar zxf "$archive_path" + ;; + *) + die "Unrecognized file extension for compressed archive at '${archive_path}'." + ;; + esac +} + +function extract_for { + local -r archive_path="$1" + local -a result_path_candidates=("${@:2}") + + do_extract "$archive_path" + + for result_path in "${result_path_candidates[@]}"; do + if [[ -e "$result_path" ]]; then + get_existing_absolute_path "$result_path" + return 0 + else + warn "note: candidate '${result_path}' was not found" + fi + done + + die "Could not locate the result of extracting '${archive_path}'." +} + +function create_gz_package { + local -r pkg_name="$1" + local -a from_paths=("${@:2}") + + local -r pkg_archive_name="${pkg_name}.tar.gz" + + rm -f "$pkg_archive_name" + + if [[ "${#from_paths[@]}" -eq 0 ]]; then + tar -czf "$pkg_archive_name" * + else + tar -czf "$pkg_archive_name" "${from_paths[@]}" + fi + + get_existing_absolute_path "$pkg_archive_name" +} + +function deploy_package_to_dirs_from_stdin { + local -r pkg_archive_path="$1" + local -r pkg_output_archive_filename="$(basename "$pkg_archive_path")" + + while read dir; do + cp >&2 -v "$pkg_archive_path" "${dir}/${pkg_output_archive_filename}" + done +} + +function with_pushd { + local -r dest="$1" + local -a cmd_line=("${@:2}") + + pushd >&2 "$dest" + "${cmd_line[@]}" + popd >&2 +} diff --git a/vars.bash b/vars.bash new file mode 100644 index 0000000..fae4eca --- /dev/null +++ b/vars.bash @@ -0,0 +1,38 @@ +# Not an executable. Describes some common configuration for the targets we +# build for, and some useful common environment variables. + +readonly BUILD_SUPPORT_BINARIES_DIR='build-support/bin' + +readonly LINUX_SUPPORTED_ARCHS=( + x86_64 +) + +readonly OSX_SUPPORTED_VERSIONS=( + 10.7 + 10.8 + 10.9 + 10.10 + 10.11 + 10.12 + 10.13 +) + +function get_create_linux_supportdirs_stdout { + local -r pkg_name="$1" pkg_ver="$2" + local linux_arch_dir + for linux_arch in "${LINUX_SUPPORTED_ARCHS[@]}"; do + linux_arch_dir="${BUILD_SUPPORT_BINARIES_DIR}/${pkg_name}/linux/${linux_arch}/${pkg_ver}" + mkdir -p "$linux_arch_dir" + echo "$linux_arch_dir" + done +} + +function get_create_osx_supportdirs_stdout { + local -r pkg_name="$1" pkg_ver="$2" + local osx_ver_dir + for osx_ver in "${OSX_SUPPORTED_VERSIONS[@]}"; do + osx_ver_dir="${BUILD_SUPPORT_BINARIES_DIR}/${pkg_name}/mac/${osx_ver}/${pkg_ver}" + mkdir -p "$osx_ver_dir" + echo "$osx_ver_dir" + done +}