From 0981b664c37d3c5bddd41f229ecd744e6f215af1 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Wed, 16 Jan 2019 20:19:48 +0100 Subject: [PATCH] build: setup bazel remote execution (#14641) --- .circleci/base-rbe-bazelrc | 127 ++++++++++++++++++ .circleci/bazel.rc | 25 ++++ .circleci/config.yml | 32 +++-- .circleci/gcp_token | Bin 0 -> 2368 bytes .../circleci/bazel/setup-remote-execution.sh | 19 +++ 5 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 .circleci/base-rbe-bazelrc create mode 100644 .circleci/gcp_token create mode 100755 scripts/circleci/bazel/setup-remote-execution.sh diff --git a/.circleci/base-rbe-bazelrc b/.circleci/base-rbe-bazelrc new file mode 100644 index 000000000000..0e049a5d0ee4 --- /dev/null +++ b/.circleci/base-rbe-bazelrc @@ -0,0 +1,127 @@ +# ------------------------------ +# This file is taken from the "bazel-toolchains" repository and is used in order to reduce the +# manual configuration overhead. https://github.com/bazelbuild/bazel-toolchains/tree/master/bazelrc +# ------------------------------ + +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Depending on how many machines are in the remote execution instance, setting +# this higher can make builds faster by allowing more jobs to run in parallel. +# Setting it too high can result in jobs that timeout, however, while waiting +# for a remote machine to execute them. +build:remote --jobs=50 + +# Set several flags related to specifying the platform, toolchain and java +# properties. +# These flags are duplicated rather than imported from (for example) +# %workspace%/configs/ubuntu16_04_clang/1.1/toolchain.bazelrc to make this +# bazelrc a standalone file that can be copied more easily. +# These flags should only be used as is for the rbe-ubuntu16-04 container +# and need to be adapted to work with other toolchain containers. +build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 +build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 +build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.18.0/default:toolchain +build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +# Platform flags: +# The toolchain container used for execution is defined in the target indicated +# by "extra_execution_platforms", "host_platform" and "platforms". +# If you are using your own toolchain container, you need to create a platform +# target with "constraint_values" that allow for the toolchain specified with +# "extra_toolchains" to be selected (given constraints defined in +# "exec_compatible_with"). +# More about platforms: https://docs.bazel.build/versions/master/platforms.html +build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.18.0/cpp:cc-toolchain-clang-x86_64-default +build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604 +build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604 +build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604 + +# Set various strategies so that all actions execute remotely. Mixing remote +# and local execution will lead to errors unless the toolchain and remote +# machine exactly match the host machine. +build:remote --spawn_strategy=remote +build:remote --strategy=Javac=remote +build:remote --strategy=Closure=remote +build:remote --genrule_strategy=remote +build:remote --define=EXECUTOR=remote + +# Enable the remote cache so action results can be shared across machines, +# developers, and workspaces. +build:remote --remote_cache=remotebuildexecution.googleapis.com + +# Enable remote execution so actions are performed on the remote systems. +build:remote --remote_executor=remotebuildexecution.googleapis.com + +# Enable encryption. +build:remote --tls_enabled=true + +# Enforce stricter environment rules, which eliminates some non-hermetic +# behavior and therefore improves both the remote cache hit rate and the +# correctness and repeatability of the build. +build:remote --experimental_strict_action_env=true + +# Set a higher timeout value, just in case. +build:remote --remote_timeout=3600 + +# Enable authentication. This will pick up application default credentials by +# default. You can use --auth_credentials=some_file.json to use a service +# account credential instead. +build:remote --auth_enabled=true + +# Set flags for uploading to BES in order to view results in the Bazel Build +# Results UI. +build:results --bes_backend="buildeventservice.googleapis.com" +build:results --bes_timeout=60s +build:results --tls_enabled + +# Output BES results url +build:results --bes_results_url="https://source.cloud.google.com/results/invocations/" + +# Set flags for uploading to BES without Remote Build Execution. +build:results-local --bes_backend="buildeventservice.googleapis.com" +build:results-local --bes_timeout=60s +build:results-local --tls_enabled=true +build:results-local --auth_enabled=true +build:results-local --spawn_strategy=local +build:results-local --remote_cache=remotebuildexecution.googleapis.com +build:results-local --remote_timeout=3600 +build:results-local --bes_results_url="https://source.cloud.google.com/results/invocations/" + +# The following flags are only necessary for local docker sandboxing +# with the rbe-ubuntu16-04 container. Use of these flags is still experimental. +build:docker-sandbox --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 +build:docker-sandbox --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 +build:docker-sandbox --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.18.0/default:toolchain +build:docker-sandbox --experimental_docker_image=gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:9bd8ba020af33edb5f11eff0af2f63b3bcb168cd6566d7b27c6685e717787928 +build:docker-sandbox --spawn_strategy=docker +build:docker-sandbox --strategy=Javac=docker +build:docker-sandbox --strategy=Closure=docker +build:docker-sandbox --genrule_strategy=docker +build:docker-sandbox --define=EXECUTOR=remote +build:docker-sandbox --experimental_docker_verbose +build:docker-sandbox --experimental_enable_docker_sandbox + +# The following flags enable the remote cache so action results can be shared +# across machines, developers, and workspaces. +build:remote-cache --remote_cache=remotebuildexecution.googleapis.com +build:remote-cache --tls_enabled=true +build:remote-cache --experimental_strict_action_env=true +build:remote-cache --remote_timeout=3600 +build:remote-cache --auth_enabled=true +build:remote-cache --spawn_strategy=standalone +build:remote-cache --strategy=Javac=standalone +build:remote-cache --strategy=Closure=standalone +build:remote-cache --genrule_strategy=standalone diff --git a/.circleci/bazel.rc b/.circleci/bazel.rc index 327a880860d8..cda5abecd517 100644 --- a/.circleci/bazel.rc +++ b/.circleci/bazel.rc @@ -5,3 +5,28 @@ # Save downloaded repositories in a location that can be cached by CircleCI. This helps us # speeding up the analysis time significantly with Bazel managed node dependencies on the CI. build --experimental_repository_cache=/home/circleci/bazel_repository_cache + +######################################## +# Remote Build Execution support on CI # +######################################## + +# Load default settings for Remote Build Execution. +import %workspace%/.circleci/base-rbe-bazelrc + +# Custom execution platform defined in the Angular repository. See: +# https://github.com/angular/angular/blob/master/tools/BUILD.bazel#L21 +build:remote --extra_execution_platforms=@angular//tools:rbe_ubuntu1604-angular +build:remote --host_platform=@angular//tools:rbe_ubuntu1604-angular +build:remote --platforms=@angular//tools:rbe_ubuntu1604-angular + +# Increase the amount of parallel jobs. The default RBE base configuration specifies a low +# number of parallel jobs, but our build and testing should be parallelizable. +build:remote --jobs=150 + +# Use the Angular team internal GCP instance for remote execution. +build:remote --remote_instance_name=projects/internal-200822/instances/default_instance + +# We want to accept remote cached build artifacts to speed up our builds incrementally. +# TODO(devversion): Temporary disabled since we don't have much insight on possible security +# issues. Enable this once we investigated this further and talked to the RBE team. +build:remote --remote_accept_cached=false diff --git a/.circleci/config.yml b/.circleci/config.yml index b5b6aba711ee..d0724ae1f2e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,37 +39,39 @@ var_6: &save_cache - "node_modules" - "~/bazel_repository_cache" +# Decryption token that is used to decode the GCP credentials file in ".circleci/gcp_token". +var_7: &gcp_decrypt_token "angular" + # Job step that ensures that the node module dependencies are installed and up-to-date. We use # Yarn with the frozen lockfile option in order to make sure that lock file and package.json are # in sync. Unlike in Travis, we don't need to manually purge the node modules if stale because # CircleCI automatically discards the cache if the checksum of the lock file has changed. -var_7: &yarn_install +var_8: &yarn_install run: yarn install --frozen-lockfile --non-interactive # Copies the Bazel config which is specifically for CircleCI to a location where Bazel picks it # up and merges it with the project-wide bazel configuration (tools/bazel.rc) -var_8: ©_bazel_config +var_9: ©_bazel_config # Set up the CircleCI specific bazel configuration. run: sudo cp ./.circleci/bazel.rc /etc/bazel.bazelrc # Sets up a different Docker image that includes a moe recent Firefox version which # is needed for headless testing. -var_9: &docker-firefox-image +var_10: &docker-firefox-image # TODO(devversion): Temporarily use a image that includes Firefox 62 because the # ngcontainer image does include an old Firefox version that does not support headless. - image: circleci/node:11.4.0-browsers # Attaches the release output which has been stored in the workspace to the current job. # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs -var_10: &attach_release_output +var_11: &attach_release_output attach_workspace: at: dist/ - # Branch filter that we can specify for jobs that should only run on publish branches. This filter # is used to ensure that not all upstream branches will be published as Github builds # (e.g. revert branches, feature branches) -var_11: &publish_branches_filter +var_12: &publish_branches_filter branches: only: - master @@ -83,11 +85,18 @@ var_11: &publish_branches_filter # In order to reduce duplication we use a YAML anchor that just always excludes the "_presubmit" # branch. We don't want to run Circle for the temporary "_presubmit" branch which is reserved # for the caretaker. -var_12: &ignore_presubmit_branch_filter +var_13: &ignore_presubmit_branch_filter branches: ignore: - "_presubmit" +# Runs a script that sets up the Bazel remote execution. This will be used by jobs that run +# Bazel primarily and should benefit from remote caching and execution. +var_14: &setup_bazel_remote_execution + run: + name: "Setup bazel RBE remote execution" + command: ./scripts/circleci/bazel/setup-remote-execution.sh + # ----------------------------- # Container version of CircleCI # ----------------------------- @@ -105,10 +114,13 @@ jobs: bazel_build_test: <<: *job_defaults resource_class: xlarge + environment: + GCP_DECRYPT_TOKEN: *gcp_decrypt_token steps: - *checkout_code - *restore_cache - *copy_bazel_config + - *setup_bazel_remote_execution - run: bazel build src/... --build_tag_filters=-docs-package - run: bazel test src/... --build_tag_filters=-docs-package @@ -122,11 +134,15 @@ jobs: # This job fails whenever an API has been updated but not explicitly approved through goldens. # -------------------------------------------------------------------------------------------- api_golden_checks: - resource_class: xlarge <<: *job_defaults + resource_class: xlarge + environment: + GCP_DECRYPT_TOKEN: *gcp_decrypt_token steps: - *checkout_code - *restore_cache + - *copy_bazel_config + - *setup_bazel_remote_execution - run: bazel test tools/public_api_guard/... diff --git a/.circleci/gcp_token b/.circleci/gcp_token new file mode 100644 index 0000000000000000000000000000000000000000..c77bcf6345231be66581d8f0ef7e0520add36ef7 GIT binary patch literal 2368 zcmV-G3BUGJVQh3|WM5y<7N>s7?N@Nri!1*7OqCin;t7*3C8RzB&@(|8tTy5lvE%Cv zR+T{fy1h;!C|og;)}HyV*}bm`&r#5geDLh~#};AO3g$`R`cE4jzC`F7H6N)MzH%6( zC~C)}sPGcYw%>bbgTD0JgPMqzCrv9rOknU!okC%NxvD5JnE}H4Re#I;v{8;tXm7*=f z2+Z{9wKNnSvwvYh117bcsz={Mp`W*PGd(LYT*gp^nK3FBNe+SCVw&iN1!=0Yrjn1$^hJ4+F{b=smOXc05OihM1ZxIkL)Y%8p#dQk0Fr?7C7 zhOTc}2*g)zh|2Svhj2B(XI{8nP;DN`#FFkS1^-F(qHZA~*?3R0iDtqHV^l%4bd=m{ zg9}qJjKm(d`CMjMO*rwUt1>1P8|myz{7v;)cH$-Yi1R3XU$c})$_(@>k%^PH*6695x2+GI+Q5e{_==|j;~8;2#` z41j?c&IqQg-?>n-)$Fs5l9t+4^BIa6M29^+dnZf3m~BrBD6Ecvx?kj1A5s*uV5gwo zMV+(!&2qx`{1B3=YTUWc8Zwiv$U0|T9J_qOQ@SqM5ZD4=UDnX;ycX-WkdsEbb(%1869T@i-$&nL-8P~ zObgasS*k=jtDR`&A6Rx-JVV&d+yW7daZriLXt-q0n{0y+pVI)095d~t-|pr$P0-7N z!_9W2!e}&|9u=G~89@Fb!t)K{-gsFTXKAznI!&6al>HWC@3UggZg1~j$`K%ES*?4G zn&>pU-W;>2ul+Jbso|sn&SvuMZo4&`+WbvJr&sN8r}d|ppusZbOozflS23o%A-U%W ztQY0UP=~5G=0o8K_6jZ>%BKl9q#B?CjzT!a!5Rm1<45D_@v2^vF+uVwA12dC@2U#o zCo2UZPh!u)?#@zh4VUUv5pyywj9H}MkOITTGP`Z z0*?Xbo7WzHoM28g)<}{(hSMbDl}ahK5)_DUaHoe`gnIjN*8l?G)GL-N7E=9?NQ9=K9wJG=Tfe#|~WQn7@wf6ws{1nYID zw@jkXHqELS0~xU)f*KXh!Wy(y{mpPqLv`w_CGX*vLi~cZ^u-h|tjO9pcf&W-F8yFn z{$34jU}c?@e^KMQ1=Dvrd;U<>0DFHIUq~Y+>IN#9w7vx)^3J($FHi8(*3j4N1Yqzz zqX}W9vOfErvR+_~xNp~r?==dWgNJ6?IpT=iG*xq9NAqVsMUnp$jxOSyl!+79>I3az zh(WH!Ot*JW_{DYX>~tW-B%G`2G{Ym6S+q{Aim&f^Y6ZL1A_VcQ9Y3*ze+{v$iKkzw*b_O&PXr|e{=)>D~l-sUh4#ZgY7 z8$zvzYAyeUki*`2+}0XVn0-o@q$>dNl$lXt$FKeTK0*JqHC4HTo~J2m@%y}$Jqqq} zJD;Y?9RmW{&dyfu!!#iN?;6$ZJ(^yICg@W^L}%AJK9Z{)jR)Go*+Ux|Q@{{4Bh&oD zR0^Dx^skg?f#1H(fXG)@V8qqn@fzG+2-FNa+a9-lHr%$o62TfM{26pE#vu+1Bh}vV zgoAc-gi5Pm9XvhZY#9HZqcNydGumni7lU|L^+sbkJ2%axZTC_p05tn;Un4{3ir|_s zz7wgILj%A1up~%1wR=ymdLwk4Yet@BHqRI}j=D+e$2=rhmNSSvi`_5Mf}4xM%}+^0 z$}eb>3Cv+GIl=pS4g%vF(JCRWI8wQH*yYi4Vcy2C3=qGCi}WlfUIvCD5?V+r@C_j) ziV|GnOTq?XEov?U=WcVVmp7_{?VI2UT$x@kX82d-|yT4_zq@tSFf*a>->Dp>`_ zCOAdFV$SyQMPNPuanumd`WJdC$xXS9#%b1$VzGgqAl_yM(1vHh`7sh3J%Od$xbZO1 z5k}{z!FT)S`N+-f?9P1M5DePzsxTd)An_er63CdjI@Cc0@pfY9m)iez!Q~BG5toL$ z@48PL>k@8{UppxUd5ZyOy{A$}u#9`{0o4QmAxl`|C0G9Y8=o` zCjED^Z12V(q1qBIcl&7g-%OYy`~XAz7m!uYQO?_#dPs{PTbSBc!Hm#T@$CSY!0}om m{sAQ2Ff~xF*QH+*E%kSQC> $BASH_ENV + +# Update the global Bazel configuration to always use remote execution. +sudo bash -c "echo 'build --config=remote' >> /etc/bazel.bazelrc"