From 763fb0cd28fe7b126e9ff079f59e235c2ceb4fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= Date: Sat, 13 Jul 2024 22:51:12 +0200 Subject: [PATCH] ffmpeg: enable MSAN Numerous changes and improvements have been made: - Build zlib and bzip2 instead of bundling .so files - Remove no longer needed patchelf - Build libass and its dependencies - Remove libva and libvdpau; they are not tested and are unlikely to be tested without a mock driver - Clean installed apt packages in the build image. Remove duplicated packages and unnecessary libraries - Add meson CFLAGS workaround for #12167 - Disable ASM as the code cannot be instrumented - Use the latest build image, possible after the above changes - Enable Centipede --- projects/ffmpeg/Dockerfile | 29 +++++------ projects/ffmpeg/build.sh | 97 ++++++++++++++++++++++-------------- projects/ffmpeg/project.yaml | 11 ++-- 3 files changed, 80 insertions(+), 57 deletions(-) diff --git a/projects/ffmpeg/Dockerfile b/projects/ffmpeg/Dockerfile index ba3bc83c4f98b..db810acf11753 100644 --- a/projects/ffmpeg/Dockerfile +++ b/projects/ffmpeg/Dockerfile @@ -14,26 +14,23 @@ # ################################################################################ -FROM gcr.io/oss-fuzz-base/base-builder@sha256:19782f7fe8092843368894dbc471ce9b30dd6a2813946071a36e8b05f5b1e27e -# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors -RUN apt-get update && apt-get install -y make autoconf libtool build-essential \ - libass-dev:i386 libfreetype6-dev:i386 \ - libvdpau-dev:i386 libxcb1-dev:i386 libxcb-shm0-dev:i386 libdrm-dev:i386 \ - texinfo libbz2-dev:i386 libbz2-1.0:i386 lib32z1 zlib1g:i386 zlib1g-dev:i386 yasm cmake mercurial wget \ - xutils-dev libpciaccess-dev:i386 nasm rsync libvpx-dev:i386 gcc-multilib \ - libass-dev libfreetype6-dev libsdl1.2-dev \ - libvdpau-dev libxcb1-dev libxcb-shm0-dev libdrm-dev \ - pkg-config texinfo libbz2-dev zlib1g zlib1g-dev yasm cmake mercurial wget \ - xutils-dev libpciaccess-dev nasm rsync libvpx-dev chrpath - -RUN curl -LO http://mirrors.kernel.org/ubuntu/pool/main/a/automake-1.16/automake_1.16.5-1.3_all.deb && \ +FROM gcr.io/oss-fuzz-base/base-builder +RUN apt-get update && apt-get install -y nasm pkg-config rsync autoconf libtool gperf +RUN curl -LO https://mirrors.kernel.org/ubuntu/pool/main/a/automake-1.16/automake_1.16.5-1.3_all.deb && \ apt install ./automake_1.16.5-1.3_all.deb +RUN python3 -m pip install --upgrade pip && python3 -m pip install -U meson ninja + RUN git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg -RUN wget https://www.alsa-project.org/files/pub/lib/alsa-lib-1.1.0.tar.bz2 +RUN curl -O https://www.alsa-project.org/files/pub/lib/alsa-lib-1.2.12.tar.bz2 RUN git clone --depth 1 https://github.com/mstorsjo/fdk-aac.git -RUN git clone --depth 1 https://github.com/intel/libva -RUN git clone --depth 1 -b libvdpau-1.2 https://gitlab.freedesktop.org/vdpau/libvdpau +RUN git clone --depth 1 https://gitlab.freedesktop.org/fontconfig/fontconfig.git +RUN git clone --depth 1 https://gitlab.freedesktop.org/freetype/freetype.git +RUN git clone --depth 1 https://github.com/fribidi/fribidi +RUN git clone --depth 1 https://github.com/harfbuzz/harfbuzz +RUN git clone --depth 1 https://github.com/libass/libass +RUN git clone --depth 1 https://github.com/madler/zlib +RUN git clone --depth 1 https://gitlab.com/federicomenaquintero/bzip2 RUN git clone --depth 1 https://chromium.googlesource.com/webm/libvpx RUN git clone --depth 1 https://gitlab.xiph.org/xiph/ogg.git RUN git clone --depth 1 https://gitlab.xiph.org/xiph/opus.git diff --git a/projects/ffmpeg/build.sh b/projects/ffmpeg/build.sh index d504c5f3b3447..d2d89c5a29fea 100755 --- a/projects/ffmpeg/build.sh +++ b/projects/ffmpeg/build.sh @@ -28,18 +28,66 @@ fi export FFMPEG_DEPS_PATH=$SRC/ffmpeg_deps mkdir -p $FFMPEG_DEPS_PATH -export PATH="$FFMPEG_DEPS_PATH/bin:$PATH" -export LD_LIBRARY_PATH="$FFMPEG_DEPS_PATH/lib" -mkdir -p $OUT/lib/ if [[ "$ARCHITECTURE" == i386 ]]; then - cp /usr/lib/i386-linux-gnu/libbz2.so.1.0 $OUT/lib/ - cp /usr/lib/i386-linux-gnu/libz.so.1 $OUT/lib/ + LIBDIR='lib/i386-linux-gnu' + export PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/$LIBDIR/pkgconfig:$FFMPEG_DEPS_PATH/lib/pkgconfig" else - cp /usr/lib/x86_64-linux-gnu/libbz2.so.1.0 $OUT/lib/ - cp /usr/lib/x86_64-linux-gnu/libz.so.1 $OUT/lib/ + LIBDIR='lib/x86_64-linux-gnu' + export PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/$LIBDIR/pkgconfig:$FFMPEG_DEPS_PATH/lib/pkgconfig" fi +# The option `-fuse-ld=gold` can't be passed via `CFLAGS` or `CXXFLAGS` because +# Meson injects `-Werror=ignored-optimization-argument` during compile tests. +# Remove the `-fuse-ld=` and let Meson handle it. +# https://github.com/mesonbuild/meson/issues/6377#issuecomment-575977919 +export MESON_CFLAGS="$CFLAGS" +if [[ "$CFLAGS" == *"-fuse-ld=gold"* ]]; then + export MESON_CFLAGS="${CFLAGS//-fuse-ld=gold/}" + export CC_LD=gold +fi +export MESON_CXXFLAGS="$CXXFLAGS" +if [[ "$CXXFLAGS" == *"-fuse-ld=gold"* ]]; then + export MESON_CXXFLAGS="${CXXFLAGS//-fuse-ld=gold/}" + export CXX_LD=gold +fi + +meson_install() { + cd $SRC/$1 + CFLAGS="$MESON_CFLAGS" CXXFLAGS="$MESON_CXXFLAGS" \ + meson setup build -Dprefix="$FFMPEG_DEPS_PATH" -Ddefault_library=static -Dprefer_static=true \ + --libdir "$LIBDIR" ${2:-} + meson compile -C build + meson install -C build +} + +meson_install bzip2 + +cd $SRC/zlib +./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared +make clean +make -j$(nproc) +make install + +cd $SRC/libxml2 +./autogen.sh --prefix="$FFMPEG_DEPS_PATH" --enable-static \ + --without-debug --without-ftp --without-http \ + --without-legacy --without-python +make clean +make -j$(nproc) +make install + +meson_install freetype +meson_install fribidi "-Ddocs=false -Dtests=false" +meson_install harfbuzz "-Ddocs=disabled -Dtests=disabled" +meson_install fontconfig + +cd $SRC/libass +./autogen.sh +./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared --disable-asm +make -j$(nproc) +make install + cd $SRC bzip2 -f -d alsa-lib-* tar xf alsa-lib-* @@ -58,20 +106,6 @@ make clean make -j$(nproc) all make install -cd $SRC/libva -./autogen.sh -./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared -make clean -make -j$(nproc) all -make install - -cd $SRC/libvdpau -./autogen.sh -./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared -make clean -make -j$(nproc) all -make install - cd $SRC/libvpx if [[ "$ARCHITECTURE" == i386 ]]; then TARGET="--target=x86-linux-gcc" @@ -128,14 +162,6 @@ make clean make -j$(nproc) make install -cd $SRC/libxml2 -./autogen.sh --prefix="$FFMPEG_DEPS_PATH" --enable-static \ - --without-debug --without-ftp --without-http \ - --without-legacy --without-python -make clean -make -j$(nproc) -make install - # Remove shared libraries to avoid accidental linking against them. rm $FFMPEG_DEPS_PATH/lib/*.so rm $FFMPEG_DEPS_PATH/lib/*.so.* @@ -149,7 +175,7 @@ else FFMPEG_BUILD_ARGS='' fi -PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \ +./configure \ --cc=$CC --cxx=$CXX --ld="$CXX $CXXFLAGS -std=c++11" \ --extra-cflags="-I$FFMPEG_DEPS_PATH/include" \ --extra-ldflags="-L$FFMPEG_DEPS_PATH/lib" \ @@ -169,6 +195,7 @@ PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \ --enable-libvpx \ --enable-libxml2 \ --enable-nonfree \ + --disable-asm \ --disable-libdrm \ --disable-muxers \ --disable-protocols \ @@ -206,7 +233,6 @@ for c in $CONDITIONALS; do echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options make tools/target_bsf_${symbol}_fuzzer mv tools/target_bsf_${symbol}_fuzzer $OUT/${fuzzer_name} - patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name done # Build fuzzers for decoders. @@ -221,7 +247,6 @@ for c in $CONDITIONALS; do echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options make tools/target_dec_${symbol}_fuzzer mv tools/target_dec_${symbol}_fuzzer $OUT/${fuzzer_name} - patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name done # Build fuzzers for encoders @@ -237,7 +262,6 @@ for c in $CONDITIONALS; do echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options make tools/target_enc_${symbol}_fuzzer mv tools/target_enc_${symbol}_fuzzer $OUT/${fuzzer_name} - patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name done @@ -246,14 +270,12 @@ fuzzer_name=ffmpeg_SWS_fuzzer echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options make tools/target_sws_fuzzer mv tools/target_sws_fuzzer $OUT/${fuzzer_name} -patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name # Build fuzzer for demuxer fuzzer_name=ffmpeg_DEMUXER_fuzzer echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options make tools/target_dem_fuzzer mv tools/target_dem_fuzzer $OUT/${fuzzer_name} -patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name # We do not need raw reference files for the muxer rm $(find fate-suite -name '*.s16') @@ -267,10 +289,9 @@ zip -r $OUT/ffmpeg_AV_CODEC_ID_HEVC_fuzzer_seed_corpus.zip fate-suite/hevc fate- fuzzer_name=ffmpeg_IO_DEMUXER_fuzzer make tools/target_io_dem_fuzzer mv tools/target_io_dem_fuzzer $OUT/${fuzzer_name} -patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name #Build fuzzers for individual demuxers -PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \ +./configure \ --cc=$CC --cxx=$CXX --ld="$CXX $CXXFLAGS -std=c++11" \ --extra-cflags="-I$FFMPEG_DEPS_PATH/include" \ --extra-ldflags="-L$FFMPEG_DEPS_PATH/lib" \ @@ -281,6 +302,7 @@ PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \ --optflags=-O1 \ --enable-gpl \ --enable-libxml2 \ + --disable-asm \ --disable-libdrm \ --disable-muxers \ --disable-protocols \ @@ -312,7 +334,6 @@ for c in $CONDITIONALS; do symbol=$(echo $c | sed "s/.*/\L\0/") make tools/target_dem_${symbol}_fuzzer mv tools/target_dem_${symbol}_fuzzer $OUT/${fuzzer_name} - patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name done # Find relevant corpus in test samples and archive them for every fuzzer. diff --git a/projects/ffmpeg/project.yaml b/projects/ffmpeg/project.yaml index fe510402f4f1d..0c403d89b409b 100644 --- a/projects/ffmpeg/project.yaml +++ b/projects/ffmpeg/project.yaml @@ -13,8 +13,13 @@ auto_ccs: - "kempfjb@gmail.com" - "jordyzomer@google.com" fuzzing_engines: - - afl - - honggfuzz - - libfuzzer + - afl + - centipede + - honggfuzz + - libfuzzer +sanitizers: + - address + - memory + - undefined selective_unpack: true main_repo: 'https://git.ffmpeg.org/ffmpeg.git'