ffmpeg: enable MSAN

Numerous changes and improvements have been made: - Build zlib and bzip2 instead of bundling .so files - Remove no longer needed patchelf - Build libass and its dependencies - Remove libva and libvdpau; they are not tested and are unlikely to be tested without a mock driver - Clean installed apt packages in the build image. Remove duplicated packages and unnecessary libraries - Add meson CFLAGS workaround for #12167 - Disable ASM, for MSAN, as the code cannot be instrumented - Use the latest build image - Enable Centipede - Disable docs and programs build, not needed
google · Jul 17, 2024 · a85dc84 · a85dc84
1 parent b4407a1
commit a85dc84
Show file tree

Hide file tree

Showing 3 changed files with 98 additions and 57 deletions.
diff --git a/projects/ffmpeg/Dockerfile b/projects/ffmpeg/Dockerfile
@@ -14,26 +14,23 @@
 #
 ################################################################################
 
-FROM gcr.io/oss-fuzz-base/base-builder@sha256:19782f7fe8092843368894dbc471ce9b30dd6a2813946071a36e8b05f5b1e27e
-# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors
-RUN apt-get update && apt-get install -y make autoconf libtool build-essential \
-	libass-dev:i386 libfreetype6-dev:i386 \
-	libvdpau-dev:i386 libxcb1-dev:i386 libxcb-shm0-dev:i386 libdrm-dev:i386 \
-	texinfo libbz2-dev:i386 libbz2-1.0:i386 lib32z1 zlib1g:i386 zlib1g-dev:i386 yasm cmake mercurial wget \
-	xutils-dev libpciaccess-dev:i386 nasm rsync libvpx-dev:i386 gcc-multilib \
-	libass-dev libfreetype6-dev libsdl1.2-dev \
-	libvdpau-dev libxcb1-dev libxcb-shm0-dev libdrm-dev \
-	pkg-config texinfo libbz2-dev zlib1g zlib1g-dev yasm cmake mercurial wget \
-	xutils-dev libpciaccess-dev nasm rsync libvpx-dev chrpath
-
-RUN curl -LO http://mirrors.kernel.org/ubuntu/pool/main/a/automake-1.16/automake_1.16.5-1.3_all.deb && \
+FROM gcr.io/oss-fuzz-base/base-builder
+RUN apt-get update && apt-get install -y nasm pkg-config rsync autoconf libtool gperf
+RUN curl -LO https://mirrors.kernel.org/ubuntu/pool/main/a/automake-1.16/automake_1.16.5-1.3_all.deb && \
     apt install ./automake_1.16.5-1.3_all.deb
+RUN python3 -m pip install --upgrade pip && python3 -m pip install -U meson ninja
+
 RUN git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg
 
-RUN wget https://www.alsa-project.org/files/pub/lib/alsa-lib-1.1.0.tar.bz2
+RUN curl -O https://www.alsa-project.org/files/pub/lib/alsa-lib-1.2.12.tar.bz2
 RUN git clone --depth 1 https://github.com/mstorsjo/fdk-aac.git
-RUN git clone --depth 1 https://github.com/intel/libva
-RUN git clone --depth 1 -b libvdpau-1.2 https://gitlab.freedesktop.org/vdpau/libvdpau
+RUN git clone --depth 1 https://gitlab.freedesktop.org/fontconfig/fontconfig.git
+RUN git clone --depth 1 https://gitlab.freedesktop.org/freetype/freetype.git
+RUN git clone --depth 1 https://github.com/fribidi/fribidi
+RUN git clone --depth 1 https://github.com/harfbuzz/harfbuzz
+RUN git clone --depth 1 https://github.com/libass/libass
+RUN git clone --depth 1 https://github.com/madler/zlib
+RUN git clone --depth 1 https://gitlab.com/federicomenaquintero/bzip2
 RUN git clone --depth 1 https://chromium.googlesource.com/webm/libvpx
 RUN git clone --depth 1 https://gitlab.xiph.org/xiph/ogg.git
 RUN git clone --depth 1 https://gitlab.xiph.org/xiph/opus.git

diff --git a/projects/ffmpeg/build.sh b/projects/ffmpeg/build.sh
@@ -19,6 +19,14 @@
 export CFLAGS="$CFLAGS -fno-sanitize=vptr"
 export CXXFLAGS="$CXXFLAGS -fno-sanitize=vptr"
 
+if [[ "$CXXFLAGS" == *"-fsanitize=address"* ]]; then
+    export CXXFLAGS="$CXXFLAGS -fno-sanitize-address-use-odr-indicator"
+fi
+
+if [[ "$CFLAGS" == *"-fsanitize=address"* ]]; then
+    export CFLAGS="$CFLAGS -fno-sanitize-address-use-odr-indicator"
+fi
+
 if [[ "$ARCHITECTURE" == i386 ]]; then
   export CFLAGS="$CFLAGS -m32"
   export CXXFLAGS="$CXXFLAGS -m32"
@@ -28,18 +36,66 @@ fi
 export FFMPEG_DEPS_PATH=$SRC/ffmpeg_deps
 mkdir -p $FFMPEG_DEPS_PATH
 
-export PATH="$FFMPEG_DEPS_PATH/bin:$PATH"
-export LD_LIBRARY_PATH="$FFMPEG_DEPS_PATH/lib"
 
-mkdir -p $OUT/lib/
 if [[ "$ARCHITECTURE" == i386 ]]; then
-  cp /usr/lib/i386-linux-gnu/libbz2.so.1.0 $OUT/lib/
-  cp /usr/lib/i386-linux-gnu/libz.so.1 $OUT/lib/
+  LIBDIR='lib/i386-linux-gnu'
+  export PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/$LIBDIR/pkgconfig:$FFMPEG_DEPS_PATH/lib/pkgconfig"
 else
-  cp /usr/lib/x86_64-linux-gnu/libbz2.so.1.0 $OUT/lib/
-  cp /usr/lib/x86_64-linux-gnu/libz.so.1 $OUT/lib/
+  LIBDIR='lib/x86_64-linux-gnu'
+  export PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/$LIBDIR/pkgconfig:$FFMPEG_DEPS_PATH/lib/pkgconfig"
+fi
+
+# The option `-fuse-ld=gold` can't be passed via `CFLAGS` or `CXXFLAGS` because
+# Meson injects `-Werror=ignored-optimization-argument` during compile tests.
+# Remove the `-fuse-ld=` and let Meson handle it.
+# https://github.com/mesonbuild/meson/issues/6377#issuecomment-575977919
+export MESON_CFLAGS="$CFLAGS"
+if [[ "$CFLAGS" == *"-fuse-ld=gold"* ]]; then
+    export MESON_CFLAGS="${CFLAGS//-fuse-ld=gold/}"
+    export CC_LD=gold
+fi
+export MESON_CXXFLAGS="$CXXFLAGS"
+if [[ "$CXXFLAGS" == *"-fuse-ld=gold"* ]]; then
+    export MESON_CXXFLAGS="${CXXFLAGS//-fuse-ld=gold/}"
+    export CXX_LD=gold
 fi
 
+meson_install() {
+  cd $SRC/$1
+  CFLAGS="$MESON_CFLAGS" CXXFLAGS="$MESON_CXXFLAGS" \
+  meson setup build -Dprefix="$FFMPEG_DEPS_PATH" -Ddefault_library=static -Dprefer_static=true \
+                    --libdir "$LIBDIR" ${2:-}
+  meson compile -C build
+  meson install -C build
+}
+
+meson_install bzip2
+
+cd $SRC/zlib
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/libxml2
+./autogen.sh --prefix="$FFMPEG_DEPS_PATH" --enable-static \
+      --without-debug --without-ftp --without-http \
+      --without-legacy --without-python
+make clean
+make -j$(nproc)
+make install
+
+meson_install freetype
+meson_install fribidi "-Ddocs=false -Dtests=false"
+meson_install harfbuzz "-Ddocs=disabled -Dtests=disabled"
+meson_install fontconfig
+
+cd $SRC/libass
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared --disable-asm
+make -j$(nproc)
+make install
+
 cd $SRC
 bzip2 -f -d alsa-lib-*
 tar xf alsa-lib-*
@@ -58,20 +114,6 @@ make clean
 make -j$(nproc) all
 make install
 
-cd $SRC/libva
-./autogen.sh
-./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared
-make clean
-make -j$(nproc) all
-make install
-
-cd $SRC/libvdpau
-./autogen.sh
-./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared
-make clean
-make -j$(nproc) all
-make install
-
 cd $SRC/libvpx
 if [[ "$ARCHITECTURE" == i386 ]]; then
       TARGET="--target=x86-linux-gcc"
@@ -128,14 +170,6 @@ make clean
 make -j$(nproc)
 make install
 
-cd $SRC/libxml2
-./autogen.sh --prefix="$FFMPEG_DEPS_PATH" --enable-static \
-      --without-debug --without-ftp --without-http \
-      --without-legacy --without-python
-make clean
-make -j$(nproc)
-make install
-
 # Remove shared libraries to avoid accidental linking against them.
 rm $FFMPEG_DEPS_PATH/lib/*.so
 rm $FFMPEG_DEPS_PATH/lib/*.so.*
@@ -149,7 +183,15 @@ else
       FFMPEG_BUILD_ARGS=''
 fi
 
-PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \
+if [ "$SANITIZER" = "memory" ]; then
+  FFMPEG_BUILD_ARGS="$FFMPEG_BUILD_ARGS --disable-asm"
+fi
+
+if [ "$SANITIZER" = "memory" ] || [ "$FUZZING_ENGINE" = "centipede" ]; then
+  FFMPEG_BUILD_ARGS="$FFMPEG_BUILD_ARGS --disable-asm"
+fi
+
+./configure \
         --cc=$CC --cxx=$CXX --ld="$CXX $CXXFLAGS -std=c++11" \
         --extra-cflags="-I$FFMPEG_DEPS_PATH/include" \
         --extra-ldflags="-L$FFMPEG_DEPS_PATH/lib" \
@@ -175,6 +217,8 @@ PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \
         --disable-demuxer=rtp,rtsp,sdp \
         --disable-devices \
         --disable-shared \
+        --disable-doc \
+        --disable-programs \
         $FFMPEG_BUILD_ARGS
 make clean
 make -j$(nproc) install
@@ -206,7 +250,6 @@ for c in $CONDITIONALS; do
       echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options
       make tools/target_bsf_${symbol}_fuzzer
       mv tools/target_bsf_${symbol}_fuzzer $OUT/${fuzzer_name}
-      patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 done
 
 # Build fuzzers for decoders.
@@ -221,7 +264,6 @@ for c in $CONDITIONALS; do
       echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options
       make tools/target_dec_${symbol}_fuzzer
       mv tools/target_dec_${symbol}_fuzzer $OUT/${fuzzer_name}
-      patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 done
 
 # Build fuzzers for encoders
@@ -237,7 +279,6 @@ for c in $CONDITIONALS; do
       echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options
       make tools/target_enc_${symbol}_fuzzer
       mv tools/target_enc_${symbol}_fuzzer $OUT/${fuzzer_name}
-      patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 done
 
 
@@ -246,14 +287,12 @@ fuzzer_name=ffmpeg_SWS_fuzzer
 echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options
 make tools/target_sws_fuzzer
 mv tools/target_sws_fuzzer $OUT/${fuzzer_name}
-patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 
 # Build fuzzer for demuxer
 fuzzer_name=ffmpeg_DEMUXER_fuzzer
 echo -en "[libfuzzer]\nmax_len = 1000000\n" >$OUT/${fuzzer_name}.options
 make tools/target_dem_fuzzer
 mv tools/target_dem_fuzzer $OUT/${fuzzer_name}
-patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 
 # We do not need raw reference files for the muxer
 rm $(find fate-suite -name '*.s16')
@@ -267,10 +306,9 @@ zip -r $OUT/ffmpeg_AV_CODEC_ID_HEVC_fuzzer_seed_corpus.zip fate-suite/hevc fate-
 fuzzer_name=ffmpeg_IO_DEMUXER_fuzzer
 make tools/target_io_dem_fuzzer
 mv tools/target_io_dem_fuzzer $OUT/${fuzzer_name}
-patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 
 #Build fuzzers for individual demuxers
-PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \
+./configure \
         --cc=$CC --cxx=$CXX --ld="$CXX $CXXFLAGS -std=c++11" \
         --extra-cflags="-I$FFMPEG_DEPS_PATH/include" \
         --extra-ldflags="-L$FFMPEG_DEPS_PATH/lib" \
@@ -299,6 +337,8 @@ PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \
         --disable-cuda_llvm \
         --enable-demuxers \
         --disable-demuxer=rtp,rtsp,sdp \
+        --disable-doc \
+        --disable-programs \
         $FFMPEG_BUILD_ARGS
 
 CONDITIONALS=$(grep 'DEMUXER 1$' config_components.h | sed 's/#define CONFIG_\(.*\)_DEMUXER 1/\1/')
@@ -312,7 +352,6 @@ for c in $CONDITIONALS; do
       symbol=$(echo $c | sed "s/.*/\L\0/")
       make tools/target_dem_${symbol}_fuzzer
       mv tools/target_dem_${symbol}_fuzzer $OUT/${fuzzer_name}
-      patchelf --set-rpath '$ORIGIN/lib' $OUT/$fuzzer_name
 done
 
 # Find relevant corpus in test samples and archive them for every fuzzer.

diff --git a/projects/ffmpeg/project.yaml b/projects/ffmpeg/project.yaml
@@ -13,8 +13,13 @@ auto_ccs:
  - "[email protected]"
  - "[email protected]"
 fuzzing_engines:
-  - afl
-  - honggfuzz
-  - libfuzzer
+ - afl
+ - centipede
+ - honggfuzz
+ - libfuzzer
+sanitizers:
+ - address
+ - memory
+ - undefined
 selective_unpack: true
 main_repo: 'https://git.ffmpeg.org/ffmpeg.git'