diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index aa834255f303..742ece3c574f 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -10,8 +10,10 @@ jobs: CI_TARGET: 'bazel.gcc' compile_time_options: CI_TARGET: 'bazel.compile_time_options' - fuzz: - CI_TARGET: 'bazel.fuzz' + # This will run on every commit/PR and will make sure the corpus generated by the fuzzers as well as fixed crashes + # (on Fuzzit) is not crashing envoy. This will help find bugs BEFORE merging and not after. + fuzzit: + CI_TARGET: 'bazel.fuzzit_regression' dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. timeoutInMinutes: 360 pool: @@ -60,3 +62,39 @@ jobs: pathtoPublish: "$(Build.StagingDirectory)/envoy" artifactName: $(CI_TARGET) condition: always() + +- job: fuzzit_fuzzing + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + timeoutInMinutes: 360 + # this runs on every push to master / merge to master. this will build the fuzzers and will update them on Fuzzit where + # they will run asynchronously. Essentially this will make sure that the latest master version is always being fuzzed + # continuously. + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['Build.Reason'], 'PullRequest')) + pool: + vmImage: 'Ubuntu 16.04' + steps: + - bash: | + echo "disk space at beginning of build:" + df -h + displayName: "Check disk space at beginning" + + - bash: | + sudo mkdir -p /etc/docker + echo '{ + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64" + }' | sudo tee /etc/docker/daemon.json + sudo service docker restart + displayName: "Enable IPv6" + + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh bazel.fuzzit_fuzzing' + workingDirectory: $(Build.SourcesDirectory) + env: + FUZZIT_API_KEY: $(FuzzitApiKey) + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "true" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote --jobs=100 --curses=no" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Fuzzit Regression" diff --git a/README.md b/README.md index abe4e9822462..7455570302d2 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ involved and how Envoy plays a role, read the CNCF [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1266/badge)](https://bestpractices.coreinfrastructure.org/projects/1266) [![Azure Pipelines](https://dev.azure.com/cncf/envoy/_apis/build/status/envoyproxy.envoy.mac?branchName=master)](https://dev.azure.com/cncf/envoy/_build/latest?definitionId=2&branchName=master) [![CircleCI](https://circleci.com/gh/envoyproxy/envoy/tree/master.svg?style=shield)](https://circleci.com/gh/envoyproxy/envoy/tree/master) +[![fuzzit](https://app.fuzzit.dev/badge?org_id=envoyproxy)](https://app.fuzzit.dev/orgs/envoyproxy/dashboard) [![Jenkins](https://img.shields.io/jenkins/s/https/powerci.osuosl.org/job/build-envoy-master/badge/icon/.svg?label=ppc64le%20build)](http://powerci.osuosl.org/job/build-envoy-master/) ## Documentation diff --git a/ci/do_ci.sh b/ci/do_ci.sh index f94d96fc8cdd..82126ab2f22b 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -260,6 +260,22 @@ elif [[ "$CI_TARGET" == "bazel.fuzz" ]]; then echo "Building envoy fuzzers and executing 100 fuzz iterations..." bazel_with_collection test ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS} --test_arg="-runs=10" exit 0 +elif [[ "$CI_TARGET" == "bazel.fuzzit_regression" ]]; then + setup_clang_toolchain + FUZZ_TEST_TARGETS="$(bazel query "attr('tags','fuzzer',${TEST_TARGETS})")" + echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS}" + echo "Building fuzzers and run a regression with corpus from Fuzzit" + bazel_with_collection build ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS} + ./ci/run_fuzzit.sh local-regression + exit 0 +elif [[ "$CI_TARGET" == "bazel.fuzzit_fuzzing" ]]; then + setup_clang_toolchain + FUZZ_TEST_TARGETS="$(bazel query "attr('tags','fuzzer',${TEST_TARGETS})")" + echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS}" + echo "Build fuzzers and push them to Fuzzit servers for continuous fuzzing" + bazel_with_collection build ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS} + ./ci/run_fuzzit.sh fuzzing + exit 0 elif [[ "$CI_TARGET" == "fix_format" ]]; then echo "fix_format..." ./tools/check_format.py fix diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index ede21c5de859..89f2cb6db41b 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -23,9 +23,9 @@ USER_GROUP=root mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" # Since we specify an explicit hash, docker-run will pull from the remote repo if missing. docker run --rm ${DOCKER_TTY_OPTION} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \ - -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build ${GIT_VOLUME_OPTION} \ + -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build -v /var/run/docker.sock:/var/run/docker.sock ${GIT_VOLUME_OPTION} \ -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE \ - -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE \ + -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE -e FUZZIT_API_KEY \ -v "$PWD":/source --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \ /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \ --home-dir /source envoybuild && usermod -a -G pcap envoybuild && su envoybuild -c \"cd source && $*\"" diff --git a/ci/run_fuzzit.sh b/ci/run_fuzzit.sh new file mode 100755 index 000000000000..d51178fe63dd --- /dev/null +++ b/ci/run_fuzzit.sh @@ -0,0 +1,33 @@ +#!/bin/bash -eux + +# Dynamically source fuzzing targets +declare -r FUZZER_TARGETS_CC=$(find . -name *_fuzz_test.cc) +declare -r FUZZER_TARGETS="$(for t in ${FUZZER_TARGETS_CC}; do echo "${t:2:-3}"; done)" + +declare BAZEL_BUILD_TARGETS="" +for t in ${FUZZER_TARGETS} +do + declare BAZEL_PATH="//"$(dirname "$t")":"$(basename "$t") + declare TAGGED=$(bazel query "attr('tags', 'no_fuzz', ${BAZEL_PATH})") + if [ -z "${TAGGED}" ] + then + FILTERED_FUZZER_TARGETS+="$t " + fi +done + + +# run fuzzing regression or upload to Fuzzit for long running fuzzing job ($1 is either local-regression or fuzzing) +wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.55/fuzzit_Linux_x86_64 +chmod a+x fuzzit + +PREFIX=$(realpath /build/tmp/_bazel_bazel/*/execroot/envoy/bazel-out/k8-fastbuild/bin) +for t in ${FILTERED_FUZZER_TARGETS} +do + TARGET_BASE="$(expr "$t" : '.*/\(.*\)_fuzz_test')" + # Fuzzit target names can't contain underscore + FUZZIT_TARGET_NAME=${TARGET_BASE//_/-} + if [ $1 == "fuzzing" ]; then + ./fuzzit create target --skip-if-exists --public-corpus envoyproxy/"${FUZZIT_TARGET_NAME}" + fi + ./fuzzit create job --skip-if-not-exists --type $1 envoyproxy/"${FUZZIT_TARGET_NAME}" "${PREFIX}"/"${t}"_with_libfuzzer +done