Continuous Fuzzing Integration with Fuzzit (envoyproxy#7509)

This will introduce another platform (apart from oss) fuzz that
will run the long-running fuzzers as well as will introduce
"sanity fuzzers" that will run the accumlated corpus and crashes
on every Pull-Request to detect bugs early-on in the development
cycle.

Risk Level: Low - as this will introduce only another step in CircleCI where the fuzzers will be uploaded to Fuzzit and the heavy lifting will be there.

Testing: No code is added just a CI code in Circle

Signed-off-by: Yevgeny Pats <[email protected]>

.azure-pipelines/linux.yml

@@ -10,8 +10,10 @@ jobs:
         CI_TARGET: 'bazel.gcc'
       compile_time_options:
         CI_TARGET: 'bazel.compile_time_options'
-      fuzz:
-        CI_TARGET: 'bazel.fuzz'
+      # This will run on every commit/PR and will make sure the corpus generated by the fuzzers as well as fixed crashes
+      # (on Fuzzit) is not crashing envoy. This will help find bugs BEFORE merging and not after.
+      fuzzit:
+        CI_TARGET: 'bazel.fuzzit_regression'
   dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
   timeoutInMinutes: 360
   pool:
@@ -60,3 +62,39 @@ jobs:
       pathtoPublish: "$(Build.StagingDirectory)/envoy"
       artifactName: $(CI_TARGET)
     condition: always()
+
+- job: fuzzit_fuzzing
+  dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+  timeoutInMinutes: 360
+  # this runs on every push to master / merge to master. this will build the fuzzers and will update them on Fuzzit where
+  # they will run asynchronously. Essentially this will make sure that the latest master version is always being fuzzed
+  # continuously.
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['Build.Reason'], 'PullRequest'))
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  steps:
+    - bash: |
+        echo "disk space at beginning of build:"
+        df -h
+      displayName: "Check disk space at beginning"
+
+    - bash: |
+        sudo mkdir -p /etc/docker
+        echo '{
+          "ipv6": true,
+          "fixed-cidr-v6": "2001:db8:1::/64"
+        }' | sudo tee /etc/docker/daemon.json
+        sudo service docker restart
+      displayName: "Enable IPv6"
+
+    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh bazel.fuzzit_fuzzing'
+      workingDirectory: $(Build.SourcesDirectory)
+      env:
+        FUZZIT_API_KEY: $(FuzzitApiKey)
+        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+        ENVOY_RBE: "true"
+        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote --jobs=100 --curses=no"
+        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+      displayName: "Fuzzit Regression"

README.md

@@ -11,6 +11,7 @@ involved and how Envoy plays a role, read the CNCF
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1266/badge)](https://bestpractices.coreinfrastructure.org/projects/1266)
 [![Azure Pipelines](https://dev.azure.com/cncf/envoy/_apis/build/status/envoyproxy.envoy.mac?branchName=master)](https://dev.azure.com/cncf/envoy/_build/latest?definitionId=2&branchName=master)
 [![CircleCI](https://circleci.com/gh/envoyproxy/envoy/tree/master.svg?style=shield)](https://circleci.com/gh/envoyproxy/envoy/tree/master)
+[![fuzzit](https://app.fuzzit.dev/badge?org_id=envoyproxy)](https://app.fuzzit.dev/orgs/envoyproxy/dashboard)
 [![Jenkins](https://img.shields.io/jenkins/s/https/powerci.osuosl.org/job/build-envoy-master/badge/icon/.svg?label=ppc64le%20build)](http://powerci.osuosl.org/job/build-envoy-master/)
 
 ## Documentation

ci/do_ci.sh

@@ -260,6 +260,22 @@ elif [[ "$CI_TARGET" == "bazel.fuzz" ]]; then
   echo "Building envoy fuzzers and executing 100 fuzz iterations..."
   bazel_with_collection test ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS} --test_arg="-runs=10"
   exit 0
+elif [[ "$CI_TARGET" == "bazel.fuzzit_regression" ]]; then
+  setup_clang_toolchain
+  FUZZ_TEST_TARGETS="$(bazel query "attr('tags','fuzzer',${TEST_TARGETS})")"
+  echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS}"
+  echo "Building fuzzers and run a regression with corpus from Fuzzit"
+  bazel_with_collection build ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS}
+  ./ci/run_fuzzit.sh local-regression
+  exit 0
+elif [[ "$CI_TARGET" == "bazel.fuzzit_fuzzing" ]]; then
+  setup_clang_toolchain
+  FUZZ_TEST_TARGETS="$(bazel query "attr('tags','fuzzer',${TEST_TARGETS})")"
+  echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS}"
+  echo "Build fuzzers and push them to Fuzzit servers for continuous fuzzing"
+  bazel_with_collection build ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS}
+  ./ci/run_fuzzit.sh fuzzing
+  exit 0
 elif [[ "$CI_TARGET" == "fix_format" ]]; then
   echo "fix_format..."
   ./tools/check_format.py fix

ci/run_envoy_docker.sh

@@ -23,9 +23,9 @@ USER_GROUP=root
 mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
 # Since we specify an explicit hash, docker-run will pull from the remote repo if missing.
 docker run --rm ${DOCKER_TTY_OPTION} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \
-  -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build ${GIT_VOLUME_OPTION} \
+  -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build -v /var/run/docker.sock:/var/run/docker.sock ${GIT_VOLUME_OPTION} \
   -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE \
-  -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE \
+  -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE -e FUZZIT_API_KEY \
   -v "$PWD":/source --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \
   /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \
   --home-dir /source envoybuild && usermod -a -G pcap envoybuild && su envoybuild -c \"cd source && $*\""

ci/run_fuzzit.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -eux
+
+# Dynamically source fuzzing targets
+declare -r FUZZER_TARGETS_CC=$(find . -name *_fuzz_test.cc)
+declare -r FUZZER_TARGETS="$(for t in ${FUZZER_TARGETS_CC}; do echo "${t:2:-3}"; done)"
+
+declare BAZEL_BUILD_TARGETS=""
+for t in ${FUZZER_TARGETS}
+do
+  declare BAZEL_PATH="//"$(dirname "$t")":"$(basename "$t")
+  declare TAGGED=$(bazel query "attr('tags', 'no_fuzz', ${BAZEL_PATH})")
+  if [ -z "${TAGGED}" ]
+  then
+    FILTERED_FUZZER_TARGETS+="$t "
+  fi
+done
+
+
+# run fuzzing regression or upload to Fuzzit for long running fuzzing job ($1 is either local-regression or fuzzing)
+wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.55/fuzzit_Linux_x86_64
+chmod a+x fuzzit
+
+PREFIX=$(realpath /build/tmp/_bazel_bazel/*/execroot/envoy/bazel-out/k8-fastbuild/bin)
+for t in ${FILTERED_FUZZER_TARGETS}
+do
+  TARGET_BASE="$(expr "$t" : '.*/\(.*\)_fuzz_test')"
+  # Fuzzit target names can't contain underscore
+  FUZZIT_TARGET_NAME=${TARGET_BASE//_/-}
+  if [ $1 == "fuzzing" ]; then
+    ./fuzzit create target --skip-if-exists --public-corpus envoyproxy/"${FUZZIT_TARGET_NAME}"
+  fi
+  ./fuzzit create job --skip-if-not-exists --type $1 envoyproxy/"${FUZZIT_TARGET_NAME}" "${PREFIX}"/"${t}"_with_libfuzzer
+done