Merge pull request #5379 from knst/bc-bp-valgrind-1

backport: bitcoin#17725, bitcoin#18304, bitcoin#18392, bitcoin#18430, bitcoin#18438, bitcoin#18441, bitcoin#18477, bitcoin#18562, bitcoin#18569, bitcoin#18683, bitcoin#18899, bitcoin#18912, bitcoin#19159, bitcoin#19669, partial bitcoin#14794 - asan/valgrind CI changes

.cirrus.yml

@@ -37,7 +37,7 @@ task:
   timeout_in: 60m
   env:
     MAKEJOBS: "-j9"
-    RUN_CI_ON_HOST: "1"
+    DANGER_RUN_CI_ON_HOST: "1"
     CCACHE_SIZE: "200M"
     CCACHE_DIR: "/tmp/ccache_dir"
   ccache_cache:

.gitlab-ci.yml

@@ -252,6 +252,15 @@ linux64_fuzz-build:
   variables:
     BUILD_TARGET: linux64_fuzz
 
+#linux64_asan-build:
+#  extends:
+#    - .build-template
+#    - .skip-in-fast-mode-template
+#  needs:
+#    - x86_64-pc-linux-gnu-debug
+#  variables:
+#    BUILD_TARGET: linux64_asan
+
 linux64_tsan-build:
   extends:
     - .build-template
@@ -279,6 +288,15 @@ linux64_nowallet-build:
   variables:
     BUILD_TARGET: linux64_nowallet
 
+#linux64_valgrind-build:
+#  extends:
+#    - .build-template
+#    - .skip-in-fast-mode-template
+#  needs:
+#    - x86_64-pc-linux-gnu-debug
+#  variables:
+#    BUILD_TARGET: linux64_valgrind
+
 mac-build:
   extends:
     - .build-template
@@ -306,6 +324,15 @@ linux64_sqlite-test:
   variables:
     BUILD_TARGET: linux64_sqlite
 
+#linux64_asan-test:
+#  extends:
+#    - .test-template
+#    - .skip-in-fast-mode-template
+#  needs:
+#    - linux64_asan-build
+#  variables:
+#    BUILD_TARGET: linux64_asan
+
 linux64_tsan-test:
   extends:
     - .test-template
@@ -323,3 +350,12 @@ linux64_ubsan-test:
     - linux64_ubsan-build
   variables:
     BUILD_TARGET: linux64_ubsan
+
+#linux64_valgrind-test:
+#  extends:
+#    - .test-template
+#    - .skip-in-fast-mode-template
+#  needs:
+#    - linux64_valgrind-build
+#  variables:
+#    BUILD_TARGET: linux64_valgrind

.travis.yml

@@ -17,6 +17,8 @@
 # will trigger cache-invalidation and rebuilds as necessary.
 #
 
+version: ~> 1.0
+
 dist: xenial
 
 os: linux
@@ -30,9 +32,6 @@ cache:
   ccache: true
   directories:
     - $BASE_BUILD_DIR/ci/scratch/.ccache
-    # macOS
-    - $HOME/Library/Caches/Homebrew
-    - /usr/local/Homebrew
 before_cache:
   - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi
 env:
@@ -205,27 +204,47 @@ after_success:
 
     - stage: test
       name: 'ARM  [GOAL: install]  [focal]  [unit tests, functional tests]'
-      arch: arm64
+      arch: arm64  # Can disable QEMU_USER_CMD and run the tests natively without qemu
       env: >-
         FILE_ENV="./ci/test/00_setup_env_arm.sh"
-        QEMU_USER_CMD=""  # Can run the tests natively without qemu
+        QEMU_USER_CMD=""
+
+    - stage: test
+      name: 's390x native BE [GOAL: install] [bionic] [no depends, no GUI]'
+      arch: s390x
+      dist: bionic
+      addons:
+        apt:
+          packages:
+            - bsdmainutils
+            - libboost-filesystem-dev
+            - libboost-system-dev
+            - libboost-test-dev
+            - libboost-thread-dev
+            - libdb++-dev
+            - libdb-dev
+            - libevent-dev
+      env: >-
+        DANGER_RUN_CI_ON_HOST=true
+        CI_USE_APT_INSTALL=no
+        FILE_ENV="./ci/test/00_setup_env_s390x_host.sh"
 
 # s390 build was disabled temporarily because of disk space issues on the Travis VM
 #
 #    - stage: test
 #      name: 'S390x  [GOAL: install]  [focal]  [unit tests, functional tests]'
-#      arch: s390x
+#      arch: s390x  # Can disable QEMU_USER_CMD and run the tests natively without qemu
 #      env: >-
 #        FILE_ENV="./ci/test/00_setup_env_s390x.sh"
-#        QEMU_USER_CMD=""  # Can run the tests natively without qemu
+#        QEMU_USER_CMD=""
 
     - stage: test
       name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no functional tests]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_win64.sh"
 
     - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [focal]  [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]'
+      name: 'x86_64 Linux  [GOAL: install]  [focal]  [uses qt5 dev package and some depends packages] [unsigned char]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_qt5.sh"
 # x86_64 Linux (xenial, no depends, only system libs, sanitizers: thread (TSan))
@@ -251,6 +270,7 @@ after_success:
         FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh"
 
     - stage: test
+      name: 'x86_64 Linux  [GOAL: install]  [focal]  [no wallet]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh"
 
@@ -263,8 +283,28 @@ after_success:
       name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]'
       os: osx
       # Use the most recent version:
-      # Xcode 11, macOS 10.14, JDK 12.0.1
+      # Xcode 11.3.1, macOS 10.14, SDK 10.15
       # https://docs.travis-ci.com/user/reference/osx/#macos-version
-      osx_image: xcode11
+      osx_image: xcode11.3
+      cache:
+        directories:
+          - $TRAVIS_BUILD_DIR/ci/scratch/.ccache
+          - $TRAVIS_BUILD_DIR/releases/$HOST
+          - $HOME/Library/Caches/Homebrew
+          - /usr/local/Homebrew
+      addons:
+        homebrew:
+          packages:
+          - libtool
+          - berkeley-db4
+          - boost
+          - miniupnpc
+          - qt
+          - qrencode
+          - python3
+          - ccache
+          - zeromq
       env: >-
+        DANGER_RUN_CI_ON_HOST=true
+        CI_USE_APT_INSTALL=no
         FILE_ENV="./ci/test/00_setup_env_mac_host.sh"

ci/dash/build_src.sh

@@ -56,6 +56,11 @@ cd dashcore-$BUILD_TARGET
 
 make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
 
+if [ -n "$USE_VALGRIND" ]; then
+    echo "valgrind in USE!"
+    ${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh
+fi
+
 if [ "$RUN_SYMBOL_TESTS" = "true" ]; then
   make $MAKEJOBS -C src check-symbols
 fi

ci/dash/matrix.sh

@@ -11,6 +11,8 @@ export LC_ALL=C.UTF-8
 source ./ci/test/00_setup_env.sh
 
 # Configure sanitizers options
+export ASAN_OPTIONS=""
+export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan"
 
@@ -20,6 +22,8 @@ elif [ "$BUILD_TARGET" = "win64" ]; then
   source ./ci/test/00_setup_env_win64.sh
 elif [ "$BUILD_TARGET" = "linux64" ]; then
   source ./ci/test/00_setup_env_native_qt5.sh
+elif [ "$BUILD_TARGET" = "linux64_asan" ]; then
+  source ./ci/test/00_setup_env_native_asan.sh
 elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then
   source ./ci/test/00_setup_env_native_tsan.sh
 elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then
@@ -32,6 +36,8 @@ elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then
   source ./ci/test/00_setup_env_native_sqlite.sh
 elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then
   source ./ci/test/00_setup_env_native_nowallet.sh
+elif [ "$BUILD_TARGET" = "linux64_valgrind" ]; then
+  source ./ci/test/00_setup_env_native_valgrind.sh
 elif [ "$BUILD_TARGET" = "mac" ]; then
   source ./ci/test/00_setup_env_mac.sh
 elif [ "$BUILD_TARGET" = "s390x" ]; then

ci/dash/test_unittests.sh

@@ -11,7 +11,7 @@ set -e
 
 source ./ci/dash/matrix.sh
 
-if [ "$RUN_UNIT_TESTS" != "true" ]; then
+if [ "$RUN_UNIT_TESTS" != "true" ] && [ "$RUN_UNIT_TESTS_SEQUENTIAL" != "true" ]; then
   echo "Skipping unit tests"
   exit 0
 fi
@@ -29,5 +29,9 @@ if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then
   # Inside Docker, binfmt isn't working so we can't trust in make invoking windows binaries correctly
   wine ./src/test/test_dash.exe
 else
-  make $MAKEJOBS check VERBOSE=1
+  if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
+    ./src/test/test_dash --catch_system_errors=no -l test_suite
+  else
+      make $MAKEJOBS check VERBOSE=1
+  fi
 fi

ci/test/00_setup_env_mac_host.sh

@@ -8,9 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos
 export HOST=x86_64-apple-darwin19
-export BREW_PACKAGES="automake berkeley-db4 libtool boost miniupnpc pkg-config qt qrencode python3 ccache zeromq"
 export PIP_PACKAGES="zmq"
-export RUN_CI_ON_HOST=true
 export RUN_UNIT_TESTS=true
 export RUN_INTEGRATION_TESTS=false
 export GOAL="install"

ci/test/00_setup_env_native_asan.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+export NO_DEPENDS=1
+export FUNCTIONAL_TESTS_CONFIG="--exclude wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163)
+export RUN_BENCH=true
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"

ci/test/00_setup_env_native_fuzz.sh

@@ -6,6 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
+export DOCKER_NAME_TAG="ubuntu:20.04"
 export CONTAINER_NAME=ci_native_fuzz
 export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev"
 export DEP_OPTS="NO_UPNP=1 DEBUG=1"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -6,13 +6,13 @@
 
 export LC_ALL=C.UTF-8
 
+export DOCKER_NAME_TAG="ubuntu:20.04"
 export CONTAINER_NAME=ci_native_fuzz_valgrind
-export PACKAGES="clang-8 llvm-8 python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev valgrind"
+export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev valgrind"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export FUZZ_TESTS_CONFIG="--valgrind"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-c++17 CC=clang-8 CXX=clang++-8"
-# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-c++17 CC=clang CXX=clang++"

ci/test/00_setup_env_native_qt5.sh

@@ -11,5 +11,7 @@ export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libhar
 export DEP_OPTS="NO_UPNP=1 DEBUG=1"
 # TODO: we have few rpcs that aren't covered by any test, re-enable the line below once it's fixed
 # export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_pruning,feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
+export RUN_UNIT_TESTS_SEQUENTIAL="true"
+export RUN_UNIT_TESTS="false"
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-zmq --enable-reduce-exports LDFLAGS=-static-libstdc++"

ci/test/00_setup_env_native_valgrind.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev"
+export USE_VALGRIND=1
+export NO_DEPENDS=1
+if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then
+  export TEST_RUNNER_EXTRA="wallet_disable"  # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit
+else
+  export TEST_RUNNER_EXTRA="--exclude rpc_bind"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
+fi
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++"  # TODO enable GUI

ci/test/00_setup_env_s390x_host.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=s390x-linux-gnu
+export NO_DEPENDS=1
+export BITCOIN_CONFIG="--with-incompatible-bdb --enable-reduce-exports"
+export RUN_UNIT_TESTS=true
+export RUN_FUNCTIONAL_TESTS=true
+export GOAL="install"