Build rust code only once per platform in a CI run

build-support/bin/ci.sh

@@ -21,7 +21,6 @@ Usage: $0 (-h|-3fxbkmrjlpuneycitzs)
  -x           run bootstrap clean-all (assume bootstrapping from a
               fresh clone)
  -b           skip bootstrapping pants from local sources
- -k           run bootstrapped pants self compile check
  -m           run sanity checks of bootstrapped pants and repo BUILD
               files
  -r           run doc generation tests
@@ -55,25 +54,19 @@ EOF
   fi
 }
 
-bootstrap_compile_args=(
-  lint.python-eval
-  --transitive
-)
-
 # No python test sharding (1 shard) by default.
 python_unit_shard="0/1"
 python_contrib_shard="0/1"
 python_intg_shard="0/1"
 python_three="false"
 
-while getopts "h3fxbkmrjlpeasu:ny:ci:tz" opt; do
+while getopts "h3fxbmrjlpeasu:ny:ci:tz" opt; do
   case ${opt} in
     h) usage ;;
     3) python_three="true" ;;
     f) run_pre_commit_checks="true" ;;
     x) run_bootstrap_clean="true" ;;
     b) run_bootstrap="false" ;;
-    k) bootstrap_compile_args=() ;;
     m) run_sanity_checks="true" ;;
     r) run_docs="true" ;;
     j) run_jvm="true" ;;
@@ -111,21 +104,13 @@ esac
 # We're running against a Pants clone.
 export PANTS_DEV=1
 
-set -x
-
-if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then
-  start_travis_section "PreCommit" "Running pre-commit checks"
-  FULL_CHECK=1 ./build-support/bin/pre-commit.sh || exit 1
-  end_travis_section
-fi
-
 if [[ "${run_bootstrap:-true}" == "true" ]]; then
   start_travis_section "Bootstrap" "Bootstrapping pants"
   (
     if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then
       ./build-support/python/clean.sh || die "Failed to clean before bootstrapping pants."
     fi
-    ./pants ${bootstrap_compile_args[@]} binary \
+    ./pants binary \
       src/python/pants/bin:pants_local_binary && \
     mv dist/pants_local_binary.pex pants.pex && \
     ./pants.pex -V
@@ -139,6 +124,12 @@ fi
 # integration tests that shell out to `./pants`, so we set this env var for those cases.
 export RUN_PANTS_FROM_PEX=1
 
+if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then
+  start_travis_section "PreCommit" "Running pre-commit checks"
+  FULL_CHECK=1 ./build-support/bin/pre-commit.sh || exit 1
+  end_travis_section
+fi
+
 # NB: Ordering matters here. We (currently) always bootstrap a Python 2 pex.
 if [[ "${python_three:-false}" == "true" ]]; then
   banner "Setting interpreter constraints for 3!"

build-support/bin/get_ci_bootstrapped_pants_pex.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 BOOTSTRAPPED_PEX_BUCKET=$1
 BOOTSTRAPPED_PEX_KEY=$2
+
 BOOTSTRAPPED_PEX_URL=s3://${BOOTSTRAPPED_PEX_BUCKET}/${BOOTSTRAPPED_PEX_KEY}
 
 # Note that in the aws cli --no-sign-request allows access to public S3 buckets without
@@ -13,7 +14,7 @@ BOOTSTRAPPED_PEX_URL=s3://${BOOTSTRAPPED_PEX_BUCKET}/${BOOTSTRAPPED_PEX_KEY}
 NUM_VERSIONS=$(aws --no-sign-request --region us-east-1 s3api list-object-versions \
   --bucket ${BOOTSTRAPPED_PEX_BUCKET} --prefix ${BOOTSTRAPPED_PEX_KEY} --max-items 2 \
   | jq '.Versions | length')
-[ "${NUM_VERSIONS}" == "1" ] || die "Error: Found ${NUM_VERSIONS} versions for ${BOOTSTRAPPED_PEX_URL}"
+[ "${NUM_VERSIONS}" == "1" ] || (echo "Error: Found ${NUM_VERSIONS} versions for ${BOOTSTRAPPED_PEX_URL}" && exit 1)
 
 # Now fetch the pre-bootstrapped pex, so that the ./pants wrapper script can use it
 # instead of running from sources (and re-bootstrapping).
@@ -23,4 +24,7 @@ chmod 755 ./pants.pex
 # Pants code executing under test expects native_engine.so to be present as a resource
 # in the source tree. Normally it'll be there because we created it there during bootstrapping.
 # So here we have to manually extract it there from the bootstrapped pex.
-unzip -j pants.pex pants/engine/native_engine.so -d src/python/pants/engine/
+# The "|| true" is necessary because unzip returns a non-zero exit code if there were any
+# bytes before the zip magic number (in our case, the pex shebang), even though the unzip
+# operation otherwise succeeds.
+unzip -j pants.pex pants/engine/native_engine.so -d src/python/pants/engine/ || true

build-support/bin/install_aws_cli_for_ci.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Install the AWS CLI in CI jobs.
+
+# This is the fastest, most reliable way to install the AWS CLI on Linux and, particularly, MacOS.
+# Using pip is broken on some systems, and package managers (e.g., brew) must be updated prior
+# to use, which slows down CI jobs significantly. This is also the installation method recommended
+# by AWS, see https://docs.aws.amazon.com/cli/latest/userguide/install-bundle.html.
+
+TMPDIR=$(mktemp -d)
+
+pushd ${TMPDIR}
+
+curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
+unzip awscli-bundle.zip
+sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
+
+popd
+
+# Multipart operations aren't supported for anonymous users, so we set the
+# threshold high to avoid them being used automatically by the aws cli.
+aws configure set default.s3.multipart_threshold 1024MB

build-support/bin/release.sh

@@ -133,14 +133,20 @@ function pants_version_reset() {
   pushd ${ROOT} > /dev/null
     git checkout -- ${VERSION_FILE}
   popd > /dev/null
+  unset _PANTS_VERSION_OVERRIDE
 }
 
 function pants_version_set() {
-  # Mutates `src/python/pants/VERSION` to temporarily override it. Sets a `trap` to restore to
-  # HEAD on exit.
+  # Set the version in the wheels we build by mutating `src/python/pants/VERSION` to temporarily
+  # override it. Sets a `trap` to restore to HEAD on exit.
   local version=$1
   trap pants_version_reset EXIT
   echo "${version}" > "${VERSION_FILE}"
+  # Also set the version reported by the prebuilt pant.pex we use to build the wheels.
+  # This is so that we pass the sanity-check that verifies that the built wheels have the same
+  # version as the pants version used to build them.
+  # TODO: Do we actually need that sanity check?
+  export _PANTS_VERSION_OVERRIDE=${version}
 }
 
 function build_3rdparty_packages() {
@@ -187,8 +193,6 @@ function build_pants_packages() {
 }
 
 function build_fs_util() {
-  local version=$1
-
   start_travis_section "fs_util" "Building fs_util binary"
   # fs_util is a standalone tool which can be used to inspect and manipulate
   # Pants's engine's file store, and interact with content addressable storage
@@ -200,7 +204,7 @@ function build_fs_util() {
     set -e
     RUST_BACKTRACE=1 "${ROOT}/build-support/bin/native/cargo" build --release \
       --manifest-path="${ROOT}/src/rust/engine/Cargo.toml" -p fs_util
-    dst_dir="${DEPLOY_DIR}/bin/fs_util/$("${ROOT}/build-support/bin/get_os.sh")/${version}"
+    dst_dir="${DEPLOY_DIR}/bin/fs_util/$("${ROOT}/build-support/bin/get_os.sh")/${PANTS_UNSTABLE_VERSION}"
     mkdir -p "${dst_dir}"
     cp "${ROOT}/src/rust/engine/target/release/fs_util" "${dst_dir}/"
   ) || die "Failed to build fs_util"
@@ -615,13 +619,14 @@ function usage() {
   echo "PyPi.  Credentials are needed for this as described in the"
   echo "release docs: http://pantsbuild.org/release.html"
   echo
-  echo "Usage: $0 [-d] [-c] (-h|-n|-t|-l|-o|-e|-p)"
+  echo "Usage: $0 [-d] [-c] (-h|-n|-f|-t|-l|-o|-e|-p)"
   echo " -d  Enables debug mode (verbose output, script pauses after venv creation)"
   echo " -h  Prints out this help message."
   echo " -n  Performs a release dry run."
   echo "       All package distributions will be built, installed locally in"
   echo "       an ephemeral virtualenv and exercised to validate basic"
   echo "       functioning."
+  echo " -f  Build the fs_util binary."
   echo " -t  Tests a live release."
   echo "       Ensures the latest packages have been propagated to PyPi"
   echo "       and can be installed in an ephemeral virtualenv."
@@ -640,11 +645,12 @@ function usage() {
   fi
 }
 
-while getopts "hdntcloepqw" opt; do
+while getopts "hdnftcloepqw" opt; do
   case ${opt} in
     h) usage ;;
     d) debug="true" ;;
     n) dry_run="true" ;;
+    f) build_fs_util ; exit $? ;;
     t) test_release="true" ;;
     l) run_packages_script list ; exit $? ;;
     o) run_packages_script list-owners ; exit $? ;;
@@ -666,7 +672,7 @@ if [[ "${dry_run}" == "true" && "${test_release}" == "true" ]]; then
 elif [[ "${dry_run}" == "true" ]]; then
   banner "Performing a dry run release" && \
   (
-    dry_run_install && build_fs_util "${PANTS_UNSTABLE_VERSION}" && \
+    dry_run_install && \
     banner "Dry run release succeeded"
   ) || die "Dry run release failed."
 elif [[ "${test_release}" == "true" ]]; then

build-support/travis/before_install_linux.mustache

@@ -0,0 +1,6 @@
+- PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
+- JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+# Increase the max number of user watches to ensure that watchman is able to watch all
+# files in the working copy.
+- sudo sysctl fs.inotify.max_user_watches=524288
+- ./build-support/bin/install_aws_cli_for_ci.sh

build-support/travis/before_install_osx.mustache

@@ -0,0 +1,3 @@
+- curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq
+- chmod 755 /usr/local/bin/jq
+- ./build-support/bin/install_aws_cli_for_ci.sh

build-support/travis/generate_travis_yml.py

@@ -20,14 +20,19 @@
 
 def generate_travis_yml():
   """Generates content for a .travis.yml file from templates."""
-  template = pkg_resources.resource_string(__name__,
-                                           'travis.yml.mustache').decode('utf-8')
-  before_install = pkg_resources.resource_string(__name__,
-                                                 'before_install.mustache').decode('utf-8')
+  template = pkg_resources.resource_string(
+    __name__, 'travis.yml.mustache').decode('utf-8')
+  before_install_linux = pkg_resources.resource_string(
+    __name__, 'before_install_linux.mustache').decode('utf-8')
+  before_install_osx = pkg_resources.resource_string(
+    __name__, 'before_install_osx.mustache').decode('utf-8')
   context = {
     'header': HEADER,
     'integration_shards': range(0, num_integration_shards),
     'integration_shards_length': num_integration_shards,
   }
-  renderer = pystache.Renderer(partials={'before_install': before_install})
+  renderer = pystache.Renderer(partials={
+    'before_install_linux': before_install_linux,
+    'before_install_osx': before_install_osx
+  })
   print(renderer.render(template, context))