From 521b56952adff167ceb35632afd2443418c49ce9 Mon Sep 17 00:00:00 2001 From: Nathaniel Brough <nathaniel.brough@gmail.com> Date: Mon, 13 Nov 2023 09:15:28 -0800 Subject: [PATCH] pigweed: Fix build by switching to bazel (#11113) --- projects/pigweed/Dockerfile | 8 +-- projects/pigweed/build.sh | 70 +++---------------------- projects/pigweed/extract_pw_fuzzers.py | 60 ---------------------- projects/pigweed/filter_cipd.py | 71 -------------------------- projects/pigweed/project.yaml | 3 +- 5 files changed, 10 insertions(+), 202 deletions(-) delete mode 100644 projects/pigweed/extract_pw_fuzzers.py delete mode 100644 projects/pigweed/filter_cipd.py diff --git a/projects/pigweed/Dockerfile b/projects/pigweed/Dockerfile index fd1d2704e9f4..b6a91c017545 100644 --- a/projects/pigweed/Dockerfile +++ b/projects/pigweed/Dockerfile @@ -15,13 +15,7 @@ ################################################################################ FROM gcr.io/oss-fuzz-base/base-builder -RUN apt-get update && apt-get install -y git build-essential python - -RUN git clone 'https://chromium.googlesource.com/chromium/tools/depot_tools.git' --depth 1 -ENV PATH="${SRC}/depot_tools:${PATH}" - +RUN apt-get update && apt-get install -y git build-essential python python3 RUN git clone 'https://pigweed.googlesource.com/pigweed/pigweed' --depth 1 COPY build.sh $SRC/ -COPY filter_cipd.py $SRC/ -COPY extract_pw_fuzzers.py $SRC/ diff --git a/projects/pigweed/build.sh b/projects/pigweed/build.sh index f2d1f05184b5..05b3d0a0e11f 100644 --- a/projects/pigweed/build.sh +++ b/projects/pigweed/build.sh @@ -15,68 +15,14 @@ # limitations under the License. # ################################################################################ +cd $SRC/pigweed -# If you ran this script as root against a local checkout, you may need to do -# the following to restore the Pigweed build environment before continuing -# development: -# $ cd $PW_ROOT -# $ sudo rm -rf .cipd/ .python3-env/ out/ -# $ git reset --hard -# $ source ./bootstrap.sh - -PW_ROOT="$SRC/pigweed" -BUILDROOT="$PW_ROOT/out/oss-fuzz" -mkdir -p $BUILDROOT - -# Tweak the ensure file to skip downloading a bunch of build environment pieces -# that we won't use and/or that OSS-Fuzz wants to provide itself. -python $SRC/filter_cipd.py \ - --root "$PW_ROOT" \ - --json "$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json" \ - --excludes \ - infra/cmake \ - fuchsia/third_party/bazel \ - fuchsia/third_party/clang \ - infra/go \ - pigweed/third_party/protoc-gen-go \ - pigweed/third_party/openocd \ - fuchsia/rust \ - pigweed/third_party/mingw64-x86_64-win32-seh \ - pigweed/host_tools \ - infra/goma/client \ - fuchsia/third_party/qemu \ - pigweed/third_party/kythe - -# Pigweed checks that it can find these as part of a "sanity check". -mkdir -p "$PW_ROOT/.environment/cipd/pigweed/bin" -for b in arm-none-eabi-gcc bazel bloaty ; do - x="$PW_ROOT/.environment/cipd/pigweed/bin/$b" - if [[ ! -x $x ]] ; then - ln -s "$(which false)" "$x" - fi -done - -# Setup the Pigweed build environemnt -set +u -PW_ENVSETUP_QUIET=1 source "$PW_ROOT/bootstrap.sh" -set -u - -# -stdlib=libc++ conflicts with the -nostdinc++ used on pw_minimal_cpp_stdlib. -EXTRA_CXXFLAGS="-Wno-unused-command-line-argument" - -# Disable UBSan vptr since target built with -fno-rtti. -EXTRA_CXXFLAGS+=" -fno-sanitize=vptr" -EXTRA_CXXFLAGS+=" -fcoverage-compilation-dir=$PW_ROOT" - -# Build! -CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$CXXFLAGS" \ - gn gen "$BUILDROOT" \ - --root="$PW_ROOT" \ - --args="pw_toolchain_OSS_FUZZ_ENABLED=true - pw_toolchain_SANITIZERS=[\"$SANITIZER\"]" -ninja -C "$BUILDROOT" fuzzers - -# Use build-generated metadata to identify available fuzzers -python "$SRC/extract_pw_fuzzers.py" --buildroot "$BUILDROOT" --out "$OUT/" +echo "Building project using Bazel wrapper." +export BAZEL_FUZZ_TEST_QUERY=" +let all_fuzz_tests = attr(tags, \"fuzz-test\", \"//...\") in +let lang_fuzz_tests = attr(generator_function, \"pw_cc_fuzz_test\", \$all_fuzz_tests) in +\$lang_fuzz_tests - attr(tags, \"no-oss-fuzz\", \$lang_fuzz_tests) +" +bazel_build_fuzz_tests diff --git a/projects/pigweed/extract_pw_fuzzers.py b/projects/pigweed/extract_pw_fuzzers.py deleted file mode 100644 index 6361f2dee07d..000000000000 --- a/projects/pigweed/extract_pw_fuzzers.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -"""Script to find fuzzers in a Pigweed build.""" - -import argparse -import json -import os -import shutil -import sys - - -def main(): - """ Use pw_module_tests.testinfo.json files to find and copy fuzzers. """ - parser = argparse.ArgumentParser() - parser.add_argument('--buildroot') - parser.add_argument('--out') - args = parser.parse_args() - print(' buildroot: ' + args.buildroot) - print(' out: ' + args.out) - - testinfo = os.path.join(args.buildroot, 'host_clang_fuzz', - 'obj', - 'pw_module_tests.testinfo.json') - tests = [] - with open(testinfo) as json_file: - tests = json.load(json_file) - for test in tests: - if test['type'] != 'fuzzer': - # Skip unit tests - continue - fuzzer = test['test_name'] - objdir = test['test_directory'] - module = os.path.basename(os.path.dirname(objdir)) - if module == 'pw_fuzzer': - # Skip examples - continue - src = os.path.join(args.buildroot, objdir, fuzzer) - dst = os.path.join(args.out, '{}_{}'.format(module, fuzzer)) - print('Copying {} to {}'.format(src, dst)) - shutil.copy(src, dst) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/projects/pigweed/filter_cipd.py b/projects/pigweed/filter_cipd.py deleted file mode 100644 index b1d7184539d0..000000000000 --- a/projects/pigweed/filter_cipd.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -"""Script for filter Pigweed's CIPD dependencies for OSS Fuzz conflicts.""" - -import argparse -import json -import os -import sys - - -def main(): - """Main entry point of the script.""" - parser = argparse.ArgumentParser() - parser.add_argument('--root', default=os.path.join(os.getcwd(), 'pigweed')) - parser.add_argument( - '--json', default='pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json') - parser.add_argument('--excludes', default='clang', nargs='*') - args = parser.parse_args() - - # Load args.json - cipd_json = {} - try: - with open(args.json, 'r') as json_file: - cipd_json = json.load(json_file) - except Exception: - print('Encountered error attempting to load ' + args.json) - raise - - packages = cipd_json['packages'] - - # Filter out args.excludes - for exclude in args.excludes: - packages[:] = [p for p in packages if exclude not in str(p['path'])] - - cipd_json['packages'] = packages - - # Rename original CIPD JSON file - try: - os.rename(args.json, args.json + '.orig') - except Exception: - print('Encountered error attempting to rename ' + args.json) - raise - - # Save new CIPD JSON file - try: - with open(args.json, 'w') as json_file: - json.dump(cipd_json, json_file, indent=2) - except Exception: - print('Encountered error attempting to write ' + args.json) - raise - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/projects/pigweed/project.yaml b/projects/pigweed/project.yaml index 6614645cc0dd..9b5c007de2eb 100644 --- a/projects/pigweed/project.yaml +++ b/projects/pigweed/project.yaml @@ -15,8 +15,7 @@ auto_ccs: - "tpudlik@google.com" sanitizers: - address + - memory main_repo: 'https://pigweed.googlesource.com/pigweed/pigweed' fuzzing_engines: - - afl - - honggfuzz - libfuzzer