From 86094b631e443f8195e3991afaf10842faa07f4f Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 17 Dec 2019 11:46:08 -0500 Subject: [PATCH 01/19] Merge #17725: ci: Add valgrind run facb416ad5ba47b01ee52c273a5c5e3709f06f07 ci: Add valgrind run (MarcoFalke) Pull request description: Fixes #17460 ACKs for top commit: practicalswift: ACK facb416ad5ba47b01ee52c273a5c5e3709f06f07 Tree-SHA512: 55396e548a76f976d7b7170b68bc5f93cfd44656162267172f66db7eb549699a2a22d3b1bb0d5f180fe0697931939e652c8cdb86b435e81e7ce572485798009d --- .gitlab-ci.yml | 18 ++++++ .travis.yml | 7 ++- ci/dash/build_src.sh | 5 ++ ci/dash/matrix.sh | 2 + ci/test/00_setup_env_native_valgrind.sh | 15 +++++ ci/test/04_install.sh | 2 +- ci/test/wrap-valgrind.sh | 18 ++++++ contrib/containers/ci/Dockerfile | 1 + contrib/valgrind.supp | 80 ++++++++++++++++++++++++- 9 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 ci/test/00_setup_env_native_valgrind.sh create mode 100755 ci/test/wrap-valgrind.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0527f648ec1a..0f947f32d9a71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -279,6 +279,15 @@ linux64_nowallet-build: variables: BUILD_TARGET: linux64_nowallet +linux64_valgrind-build: + extends: + - .build-template + - .skip-in-fast-mode-template + needs: + - x86_64-pc-linux-gnu-debug + variables: + BUILD_TARGET: linux64_valgrind + mac-build: extends: - .build-template @@ -323,3 +332,12 @@ linux64_ubsan-test: - linux64_ubsan-build variables: BUILD_TARGET: linux64_ubsan + +linux64_valgrind-test: + extends: + - .test-template + - .skip-in-fast-mode-template + needs: + - linux64_valgrind-build + variables: + BUILD_TARGET: linux64_valgrind diff --git a/.travis.yml b/.travis.yml index 309b9cdbb0518..104c3aeeafecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -225,7 +225,7 @@ after_success: FILE_ENV="./ci/test/00_setup_env_win64.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]' + name: 'x86_64 Linux [GOAL: install] [focal] [uses qt5 dev package and some depends packages] [unsigned char]' env: >- FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" # x86_64 Linux (xenial, no depends, only system libs, sanitizers: thread (TSan)) @@ -240,6 +240,11 @@ after_success: env: >- FILE_ENV="./ci/test/00_setup_env_native_asan.sh" + - stage: test + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, valgrind]' + env: >- + FILE_ENV="./ci/test/00_setup_env_native_valgrind.sh" + - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no wallet]' env: >- diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index d4dd37c55c5ac..7dc5a3a03d0a9 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -56,6 +56,11 @@ cd dashcore-$BUILD_TARGET make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) +if [ -n "$USE_VALGRIND" ]; then + echo "valgrind in USE!" + ${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh +fi + if [ "$RUN_SYMBOL_TESTS" = "true" ]; then make $MAKEJOBS -C src check-symbols fi diff --git a/ci/dash/matrix.sh b/ci/dash/matrix.sh index 3f8f35bc4b277..b9ba18b3b9ea3 100755 --- a/ci/dash/matrix.sh +++ b/ci/dash/matrix.sh @@ -32,6 +32,8 @@ elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then source ./ci/test/00_setup_env_native_sqlite.sh elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then source ./ci/test/00_setup_env_native_nowallet.sh +elif [ "$BUILD_TARGET" = "linux64_valgrind" ]; then + source ./ci/test/00_setup_env_native_valgrind.sh elif [ "$BUILD_TARGET" = "mac" ]; then source ./ci/test/00_setup_env_mac.sh elif [ "$BUILD_TARGET" = "s390x" ]; then diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh new file mode 100644 index 0000000000000..65d5e620d7be8 --- /dev/null +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" +export USE_VALGRIND=1 +export NO_DEPENDS=1 +export TEST_RUNNER_EXTRA="p2p_instantsend.py" # Only run one test for now. TODO enable all and bump timeouts +export RUN_FUNCTIONAL_TESTS=true +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index c5983c7d61198..f541970c44c71 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -47,7 +47,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan" export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env +env | grep -E '^(BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh new file mode 100755 index 0000000000000..d2192061dbc90 --- /dev/null +++ b/ci/test/wrap-valgrind.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for b_name in "${BASE_OUTDIR}/bin"/*; do + # shellcheck disable=SC2044 + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + echo "Wrap $b ..." + mv "$b" "${b}_orig" + echo '#!/usr/bin/env bash' > "$b" + echo "valgrind --gen-suppressions=all --quiet --error-exitcode=1 --suppressions=${BASE_ROOT_DIR}/contrib/valgrind.supp \"${b}_orig\" \"\$@\"" >> "$b" + chmod +x "$b" + done +done diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index b74b9466a97f1..f07a79699323c 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -74,6 +74,7 @@ RUN apt-get update && apt-get install $APT_ARGS \ nsis \ python3-zmq \ parallel \ + valgrind \ wine-stable \ wine32 \ wine64 \ diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 06df5636b5844..12b4d8318cd66 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -30,8 +30,6 @@ Memcheck:Cond obj:*/libdb_cxx-*.so fun:__log_put - obj:*/libdb_cxx-*.so - fun:__log_put_record } { Suppress libdb warning @@ -39,8 +37,51 @@ pwrite64(buf) fun:pwrite fun:__os_io +} +{ + Suppress libdb warning + Memcheck:Cond + fun:__log_putr.isra.1 +} +{ + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:__os_io + obj:*/libdb_cxx-*.so +} +{ + Suppress uninitialized bytes warning in compat code + Memcheck:Param + ioctl(TCSET{S,SW,SF}) + fun:tcsetattr +} +{ + Suppress libdb warning + Memcheck:Leak + fun:malloc + ... obj:*/libdb_cxx-*.so } +{ + Suppress leaks on init + Memcheck:Leak + ... + fun:_Z11AppInitMainR11NodeContext +} +{ + Suppress leaks on shutdown + Memcheck:Leak + ... + fun:_Z8ShutdownR11NodeContext +} +{ + Ignore GUI warning + Memcheck:Leak + ... + obj:/usr/lib64/libgdk-3.so.0.2404.7 +} { Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 Memcheck:Leak @@ -56,17 +97,49 @@ ... fun:_ZN7leveldbL14InitDefaultEnvEv } +{ + Suppress leveldb leak + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldb6DBImpl14BackgroundCallEv +} +{ + Suppress leveldb leak + Memcheck:Leak + fun:_Znwm + ... + fun:GetCoin +} { Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626) Memcheck:Addr16 fun:__wcsnlen_sse4_1 fun:wcsnrtombs } +{ + Suppress wcsnrtombs warning (remove after removing boost::fs) + Memcheck:Cond + ... + fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE + fun:unique_path +} +{ + Suppress boost warning + Memcheck:Leak + fun:_Znwm + ... + fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE + fun:_ZN5boost9unit_test9framework3runEmb + fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc + fun:main +} { Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9) Memcheck:Cond fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE - fun:_ZN5boost10filesystem6detail28directory_iterator_constructERNS0_18directory_iteratorERKNS0_4pathEPNS_6system10error_codeE + ... obj:*/libboost_filesystem.so.* } { @@ -74,6 +147,7 @@ Memcheck:Leak match-leak-kinds: reachable fun:_Znwm + ... fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_ } { From c2554ab891d9f50c20a79bd5a99a355211604b57 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 11 Mar 2020 08:38:59 -0400 Subject: [PATCH 02/19] Merge #18304: ci: Enable all functional tests in valgrind 4444edc2e6671d3f73de3725447130f73ecf0375 ci: Enable all functional tests in valgrind (MarcoFalke) Pull request description: The travis timeout for our repo has been bumped to 2h, so we can run all tests in valgrind now ACKs for top commit: practicalswift: ACK 4444edc2e6671d3f73de3725447130f73ecf0375 -- regarding the three disabled cases (`feature_abortnode`, `feature_block` and `rpc_bind`): not a big deal since MSan will take care of those once #18288 is merged. More is more :) Tree-SHA512: ea2f798112911b6d1f3d88cfcdf0a7cdb698687248343703d6fe55da144542c961c15d888bffb41672c10aa76765615cb7c7ff93d468bfad3c51f962f24e7abb --- ci/test/00_setup_env_native_valgrind.sh | 2 +- test/functional/feature_cltv.py | 2 +- test/functional/wallet_createwallet.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 65d5e620d7be8..01c8f5701eb79 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="p2p_instantsend.py" # Only run one test for now. TODO enable all and bump timeouts +export TEST_RUNNER_EXTRA="--exclude feature_abortnode,feature_block,rpc_bind" # Excluded for now export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 82969347fb085..c9e872b377fd2 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -62,7 +62,7 @@ def set_test_params(self): '-acceptnonstdtxn=1', # cltv_invalidate is nonstandard ]] self.setup_clean_chain = True - self.rpc_timeout = 120 + self.rpc_timeout = 480 def test_cltv_info(self, *, is_active): assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip65'], diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 66fb965cd2856..df1d9082e22ca 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -79,7 +79,7 @@ def run_test(self): assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) # Now set a seed and it should work. Wallet should also be encrypted - w4.walletpassphrase('pass', 2) + w4.walletpassphrase('pass', 60) w4.upgradetohd(walletpassphrase='pass') w4.getnewaddress() w4.getrawchangeaddress() From 32b4f8dd65f37301f82e7b6dfdc36772968034af Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 23 Mar 2020 07:37:38 -0400 Subject: [PATCH 03/19] Merge #18392: ci: Run feature_block in valgrind fa92af5af39a08982f785542df5419d6d5a4706d ci: Run feature_block and feature_abortnode in valgrind (MarcoFalke) fa01febeaf801bade77a613e64f18b556ae16d86 test: Remove ci timeout restriction in test_runner (MarcoFalke) Pull request description: Also revert commit 0a4912e46aecef6f04c6287508020ec60424ed24, because some tests take too long for this to be useful anymore. Top commit has no ACKs. Tree-SHA512: 363f14766e1f4a5860ab668a516b41acebc6fbdf11d8defb3a95a772dbf82304ca1f5f14b1dbad97f2029503e03d92e8c69df0466a8872409c20665838f617ed --- ci/test/00_setup_env_native_valgrind.sh | 3 +-- test/functional/feature_abortnode.py | 4 +++- test/functional/feature_block.py | 6 +++--- test/functional/test_runner.py | 14 +++----------- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 01c8f5701eb79..02e215adcf643 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -9,7 +9,6 @@ export LC_ALL=C.UTF-8 export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude feature_abortnode,feature_block,rpc_bind" # Excluded for now -export RUN_FUNCTIONAL_TESTS=true +export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 20304be8fd8f6..48d8ba3b918ec 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -14,11 +14,12 @@ from test_framework.util import wait_until, get_datadir_path import os -class AbortNodeTest(BitcoinTestFramework): +class AbortNodeTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 + self.rpc_timeout = 240 def setup_network(self): self.setup_nodes() @@ -44,5 +45,6 @@ def run_test(self): self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() + if __name__ == '__main__': AbortNodeTest().main() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index c41bd5a10f82b..4182dc16f4c34 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1272,7 +1272,7 @@ def run_test(self): self.save_spendable_output() spend = self.get_spendable_output() - self.send_blocks(blocks, True, timeout=1920) + self.send_blocks(blocks, True, timeout=2440) chain1_tip = i # now create alt chain of same length @@ -1284,14 +1284,14 @@ def run_test(self): # extend alt chain to trigger re-org block = self.next_block("alt" + str(chain1_tip + 1), version=4) - self.send_blocks([block], True, timeout=1920) + self.send_blocks([block], True, timeout=2440) # ... and re-org back to the first chain self.move_tip(chain1_tip) block = self.next_block(chain1_tip + 1, version=4) self.send_blocks([block], False, force_send=True) block = self.next_block(chain1_tip + 2, version=4) - self.send_blocks([block], True, timeout=1920) + self.send_blocks([block], True, timeout=2440) self.log.info("Reject a block with an invalid block header version") b_v1 = self.next_block('b_v1', version=1) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9ed602f6bf45c..f2ce574dc72b1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -19,9 +19,8 @@ import os import time import shutil -import signal -import sys import subprocess +import sys import tempfile import re import logging @@ -416,11 +415,10 @@ def main(): args=passon_args, combined_logs_len=args.combinedlogslen, failfast=args.failfast, - runs_ci=args.ci, use_term_control=args.ansi, ) -def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, attempts=1, enable_coverage=False, args=None, combined_logs_len=0,failfast=False, runs_ci=False, use_term_control): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, attempts=1, enable_coverage=False, args=None, combined_logs_len=0,failfast=False, use_term_control): args = args or [] # Warn if dashd is already running @@ -473,7 +471,6 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, attempts=1, enab tmpdir=tmpdir, test_list=test_list, flags=flags, - timeout_duration= 90 * 60 if runs_ci else float('inf'), # in seconds use_term_control=use_term_control, attempts=attempts, ) @@ -559,12 +556,11 @@ class TestHandler: Trigger the test scripts passed in via the list. """ - def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, timeout_duration, use_term_control, attempts): + def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, use_term_control, attempts): assert num_tests_parallel >= 1 self.num_jobs = num_tests_parallel self.tests_dir = tests_dir self.tmpdir = tmpdir - self.timeout_duration = timeout_duration self.test_list = test_list self.flags = flags self.num_running = 0 @@ -608,10 +604,6 @@ def get_next(self): time.sleep(.5) for job in self.jobs: (name, start_time, proc, testdir, log_out, log_err, portseed, attempt) = job - if int(time.time() - start_time) > self.timeout_duration: - # Timeout individual tests if timeout is specified (to stop - # tests hanging and not providing useful output). - proc.send_signal(signal.SIGINT) if proc.poll() is not None: log_out.seek(0), log_err.seek(0) [stdout, stderr] = [log_file.read().decode('utf-8') for log_file in (log_out, log_err)] From 316f6b1895ead3df918e88ccc554a5c6a3455e76 Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 4 Jun 2020 10:45:21 +0800 Subject: [PATCH 04/19] Merge #19159: test: Make valgrind.supp work on aarch64 fab7d954b261d74b369fe2a3c1785540c4f056b4 test: Make valgrind.supp work on aarch64 (MarcoFalke) Pull request description: Was easy to fix by simply removing a line ACKs for top commit: practicalswift: ACK fab7d954b261d74b369fe2a3c1785540c4f056b4 -- diff looks correct Tree-SHA512: d2d7c6cac453a3177c20e256ec50a03066f8dbf5ae45299077ccf4a2b45a3a40252b1b5fcaf9224a59bb5c3df5bd90ac58af27eb0f47dc87c2640df5b2b460ca --- contrib/valgrind.supp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 12b4d8318cd66..1ee3e08c51c82 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -1,7 +1,5 @@ -# Valgrind suppressions file for Dash. -# -# Includes known Valgrind warnings in our dependencies that cannot be fixed -# in-tree. +# This valgrind suppressions file includes known Valgrind warnings in our +# dependencies that cannot be fixed in-tree. # # Example use: # $ valgrind --suppressions=contrib/valgrind.supp src/test/test_dash @@ -14,6 +12,9 @@ # --error-limit=no src/test/test_dash # # Note that suppressions may depend on OS and/or library versions. +# Tested on: +# * aarch64 (Ubuntu 20.04 system libs, without gui) +# * x86_64 (Ubuntu 18.04 system libs, without gui) { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -47,8 +48,7 @@ Suppress libdb warning Memcheck:Param pwrite64(buf) - fun:pwrite - fun:__os_io + ... obj:*/libdb_cxx-*.so } { From 3d141a8663f2b1bd298f7c7f39834d88e703cdc1 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 6 May 2020 16:12:33 -0400 Subject: [PATCH 05/19] Merge #18899: travis: Remove valgrind fa082d0a57afedca9122fac4aecd6a3070f06b04 travis: Remove valgrind (MarcoFalke) Pull request description: When the valgrind run was added, it took 2 hours. Travis kindly raised the timeout limit to the maximum possible of 3 hours. Today, a full build of Bitcoin Core with all tests takes more than three hours. Thus, it is impossible to run all tests on travis. Moreover, the feedback loop for developers that create a pull request takes at least 2 hours, but in some cases (when the travis queue is full) until the next day. This is unacceptable. Fix both issues by removing the build from travis. Please note that the `ci/test/` configurations are *not* removed. They will stay in the repo and can be executed anywhere (just not on travis). ACKs for top commit: jamesob: ACK https://github.com/bitcoin/bitcoin/pull/18899/commits/fa082d0a57afedca9122fac4aecd6a3070f06b04 jnewbery: utACK fa082d0a57afedca9122fac4aecd6a3070f06b04 Tree-SHA512: 9acaa0e2d3926014fadb7dd2e86c4e01df382e9399f6ae99f989fa609da66a77bdd1b75d6ff42d2686f38f730b8564e6dc722aa597a473290c9d30c2abe7ef0f --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- .travis.yml | 11 +---------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f947f32d9a71..25ac730fbc273 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -279,14 +279,14 @@ linux64_nowallet-build: variables: BUILD_TARGET: linux64_nowallet -linux64_valgrind-build: - extends: - - .build-template - - .skip-in-fast-mode-template - needs: - - x86_64-pc-linux-gnu-debug - variables: - BUILD_TARGET: linux64_valgrind +#linux64_valgrind-build: +# extends: +# - .build-template +# - .skip-in-fast-mode-template +# needs: +# - x86_64-pc-linux-gnu-debug +# variables: +# BUILD_TARGET: linux64_valgrind mac-build: extends: @@ -333,11 +333,11 @@ linux64_ubsan-test: variables: BUILD_TARGET: linux64_ubsan -linux64_valgrind-test: - extends: - - .test-template - - .skip-in-fast-mode-template - needs: - - linux64_valgrind-build - variables: - BUILD_TARGET: linux64_valgrind +#linux64_valgrind-test: +# extends: +# - .test-template +# - .skip-in-fast-mode-template +# needs: +# - linux64_valgrind-build +# variables: +# BUILD_TARGET: linux64_valgrind diff --git a/.travis.yml b/.travis.yml index 104c3aeeafecc..d2da8a5543f44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -240,22 +240,13 @@ after_success: env: >- FILE_ENV="./ci/test/00_setup_env_native_asan.sh" - - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, valgrind]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_valgrind.sh" - - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" - - - stage: test + name: 'x86_64 Linux [GOAL: install] [focal] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" From cd16cf96a404c4a091d1a5073200586fd3039823 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 25 Mar 2020 13:36:47 -0400 Subject: [PATCH 06/19] Merge #18430: ci: Only clone bitcoin-core/qa-assets when fuzzing fae1e992898aa6b36c402cec4085fbf6da9b33ad ci: Only clone bitcoin-core/qa-assets when fuzzing (MarcoFalke) Pull request description: Currently the only content of that repo are some seeds, so we can speed up some ci builds ACKs for top commit: laanwj: ACK fae1e992898aa6b36c402cec4085fbf6da9b33ad (provided this passes travis) Tree-SHA512: ed813738e7f24bb56a2f12aa3b398e414eb4f0ba98379836a33ff3e5602cbf42a28e89aad10e346468191ecddc03e60d5b236097112e27c07cb1c2293533ea58 --- ci/test/04_install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index f541970c44c71..d0bbec9331ff3 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -103,7 +103,9 @@ else fi if [ ! -d ${DIR_QA_ASSETS} ]; then + if [ "$RUN_FUZZ_TESTS" = "true" ]; then DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} + fi fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ From 548d63e9e689fc84643c730f1001bfcb8d0e044e Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 26 Mar 2020 09:10:00 -0400 Subject: [PATCH 07/19] Merge #18438: ci: Use Homebrew addon on native macOS 25c8b73656c90f7b78a90f25c5bcc308f8b7c598 ci: Use Homebrew addon on native macOS (Hennadii Stepanov) 596c627a1eb2b32e2755ae5b9bf32235cf8ce72b ci: Fix brew in Travis (Hennadii Stepanov) Pull request description: Recently almost every macOS image update on Travis breaks our builds: - #17848 - #18436 This PR: - fixes the error caused by the recent [update](https://changelog.travis-ci.com/xcode-11-3-1-xcode-11-2-1-xcode-11-1-and-xcode11-images-updated-142286) from 10.14.4 (18E226) to 10.14.6 (18G3020) on March 25 - leverages [Homebrew addon](https://config.travis-ci.com/ref/job/addons/homebrew) to install packages Homebrew is not told to install `automake` and `pkg-config` packages, as the [docs](https://docs.travis-ci.com/user/reference/osx/#compilers-and-build-toolchain) states that they are pre-installed: > - automake 1.16.1 > - pkg-config 0.29.2 Top commit has no ACKs. Tree-SHA512: 1a70c06468fbe162503081b03dcf54614d67abf8ff0ce07d118b5bb50bbb92c182a76f769bea586c691aa82b9281a29cdef88091acc16895817a2e7cddafec6e --- .travis.yml | 16 ++++++++++++++-- ci/test/00_setup_env_mac_host.sh | 1 - ci/test/04_install.sh | 22 ---------------------- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2da8a5543f44..be3ac3dcf1e46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -259,8 +259,20 @@ after_success: name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]' os: osx # Use the most recent version: - # Xcode 11, macOS 10.14, JDK 12.0.1 + # Xcode 11.3.1, macOS 10.14, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version - osx_image: xcode11 + osx_image: xcode11.3 + addons: + homebrew: + packages: + - libtool + - berkeley-db4 + - boost + - miniupnpc + - qt + - qrencode + - python3 + - ccache + - zeromq env: >- FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 6e23046b79a80..a43d715a65e3f 100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -8,7 +8,6 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos export HOST=x86_64-apple-darwin19 -export BREW_PACKAGES="automake berkeley-db4 libtool boost miniupnpc pkg-config qt qrencode python3 ccache zeromq" export PIP_PACKAGES="zmq" export RUN_CI_ON_HOST=true export RUN_UNIT_TESTS=true diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index d0bbec9331ff3..9a5d4e0242717 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -14,30 +14,8 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then fi if [ "$TRAVIS_OS_NAME" == "osx" ]; then - set +o errexit - pushd /usr/local/Homebrew || exit 1 - git reset --hard origin/master - popd || exit 1 - set -o errexit - ${CI_RETRY_EXE} brew unlink python@2 - ${CI_RETRY_EXE} brew update - # brew upgrade returns an error if any of the packages is already up to date - # Failure is safe to ignore, unless we really need an update. - brew upgrade $BREW_PACKAGES || true - - # install new packages (brew install returns an error if already installed) - for i in $BREW_PACKAGES; do - if ! brew list | grep -q $i; then - ${CI_RETRY_EXE} brew install $i - fi - done - export PATH="/usr/local/opt/ccache/libexec:$PATH" - OPENSSL_PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig" - export PKG_CONFIG_PATH=$OPENSSL_PKG_CONFIG_PATH:$PKG_CONFIG_PATH - ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES - fi mkdir -p "${BASE_SCRATCH_DIR}" From 385b9a87335681ac384c10a1e13b22fed19f296d Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 26 Mar 2020 14:04:38 -0400 Subject: [PATCH 08/19] Merge #18441: ci: Remove misplaced comments from folded block scalar e41e46cee0d9b98869a569b73dafc5452b4272d6 ci: Remove misplaced comments from folded block scalar (Hennadii Stepanov) Pull request description: On master (f3a91ab0edc70e1bcb7e770f0878f03459c399e1) the [build log for ARM](https://travis-ci.org/github/bitcoin/bitcoin/jobs/667247758) contains: ``` $ export FILE_ENV="./ci/test/00_setup_env_arm.sh" $ export QEMU_USER_CMD="" $ export Can= $ export run= $ export the= $ export tests= $ export natively= $ export without= $ export qemu= Setting up build cache ... ``` and ![Screenshot from 2020-03-26 18-58-50](https://user-images.githubusercontent.com/32963518/77674223-e28d2d00-6f93-11ea-82a1-7f805ccb678d.png) With this PR: ``` $ export FILE_ENV="./ci/test/00_setup_env_arm.sh" $ export QEMU_USER_CMD="" Setting up build cache ... ``` and ![Screenshot from 2020-03-26 19-01-56](https://user-images.githubusercontent.com/32963518/77674456-4d3e6880-6f94-11ea-9c83-546c81f1cdf7.png) --- Also Travis [build config validation](https://docs.travis-ci.com/user/build-config-validation) added: ![Screenshot from 2020-03-26 19-04-19](https://user-images.githubusercontent.com/32963518/77674686-9ee6f300-6f94-11ea-900f-fdef8a673113.png) Top commit has no ACKs. Tree-SHA512: a3ea05d543a4d46c65fffd7c67df9bbd4ca8d426fa215c5ce26653de34145edd419a54884da0416608a2c90bf5b421cab3ec32290c8916f10399354b44ddeb99 --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index be3ac3dcf1e46..ebaa556473a3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ # will trigger cache-invalidation and rebuilds as necessary. # +version: ~> 1.0 + dist: xenial os: linux @@ -205,19 +207,19 @@ after_success: - stage: test name: 'ARM [GOAL: install] [focal] [unit tests, functional tests]' - arch: arm64 + arch: arm64 # Can disable QEMU_USER_CMD and run the tests natively without qemu env: >- FILE_ENV="./ci/test/00_setup_env_arm.sh" - QEMU_USER_CMD="" # Can run the tests natively without qemu + QEMU_USER_CMD="" # s390 build was disabled temporarily because of disk space issues on the Travis VM # # - stage: test # name: 'S390x [GOAL: install] [focal] [unit tests, functional tests]' -# arch: s390x +# arch: s390x # Can disable QEMU_USER_CMD and run the tests natively without qemu # env: >- # FILE_ENV="./ci/test/00_setup_env_s390x.sh" -# QEMU_USER_CMD="" # Can run the tests natively without qemu +# QEMU_USER_CMD="" - stage: test name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' From 3ddc17a3b215cc479c1e93b30b831aab8981b7e1 Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 1 Apr 2020 12:27:34 +0800 Subject: [PATCH 09/19] Merge #18477: ci: Use Focal for fuzzers faa9491870bf29583a989c407694d01f49b65bec ci: Use Focal for fuzzers (MarcoFalke) Pull request description: This gives us access to clang-10, as well as a newer version of valgrind ACKs for top commit: fanquake: ACK faa9491870bf29583a989c407694d01f49b65bec - [Clang 10](https://packages.ubuntu.com/focal/clang) and [valgrind 3.15](https://packages.ubuntu.com/focal/valgrind). practicalswift: ACK faa9491870bf29583a989c407694d01f49b65bec -- diff looks correct & contemporary clang is better than vintage clang Tree-SHA512: 0e67232673434c0309db79c1054e3e981115083585945967e346f4d58792635832100f89911428aab928155e44e5f401207a023681ae008fdb5280cf02c4d427 --- ci/test/00_setup_env_native_fuzz.sh | 1 + ci/test/00_setup_env_native_fuzz_with_valgrind.sh | 6 +++--- ci/test/00_setup_env_native_valgrind.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 7aad5ca06b531..44a6ff494f8f4 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,6 +6,7 @@ export LC_ALL=C.UTF-8 +export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev" export DEP_OPTS="NO_UPNP=1 DEBUG=1" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 16539a1df78a4..8e68bd7f318e9 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -6,13 +6,13 @@ export LC_ALL=C.UTF-8 +export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang-8 llvm-8 python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev valgrind" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-c++17 CC=clang-8 CXX=clang++-8" -# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64 +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-c++17 CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 02e215adcf643..f07518e9e77a9 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 From 85be06af393c8d4d698bebdfdb221754e58fd03b Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 9 Apr 2020 20:06:53 +0800 Subject: [PATCH 10/19] Merge #18562: ci: Run unit tests sequential once fa7af33b4cb12315de86315c163393bdb50fad75 ci: Run unit tests sequential once (MarcoFalke) fa68a3e7640fc00c47f03fa565357679bdafb4b2 appveyor: Enable minimal unit test logging to aid debugging (MarcoFalke) Pull request description: Fixes #16976 Top commit has no ACKs. Tree-SHA512: 1f1ee8776a67afa8c1c5a16ef170c9975b6a486c087c8eba12e97d23382befd1c2801622ec70ca8e4cd1fbedce1dec46be67677ceaf07f35f1d3f3bead0200f0 --- ci/dash/test_unittests.sh | 8 ++++++-- ci/test/00_setup_env_native_qt5.sh | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/dash/test_unittests.sh b/ci/dash/test_unittests.sh index e97d234348f19..0b7bbf46db202 100755 --- a/ci/dash/test_unittests.sh +++ b/ci/dash/test_unittests.sh @@ -11,7 +11,7 @@ set -e source ./ci/dash/matrix.sh -if [ "$RUN_UNIT_TESTS" != "true" ]; then +if [ "$RUN_UNIT_TESTS" != "true" ] && [ "$RUN_UNIT_TESTS_SEQUENTIAL" != "true" ]; then echo "Skipping unit tests" exit 0 fi @@ -29,5 +29,9 @@ if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then # Inside Docker, binfmt isn't working so we can't trust in make invoking windows binaries correctly wine ./src/test/test_dash.exe else - make $MAKEJOBS check VERBOSE=1 + if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + ./src/test/test_dash --catch_system_errors=no -l test_suite + else + make $MAKEJOBS check VERBOSE=1 + fi fi diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 38c9921a6bea0..4ee843d25bf66 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -11,5 +11,7 @@ export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libhar export DEP_OPTS="NO_UPNP=1 DEBUG=1" # TODO: we have few rpcs that aren't covered by any test, re-enable the line below once it's fixed # export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_pruning,feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export RUN_UNIT_TESTS_SEQUENTIAL="true" +export RUN_UNIT_TESTS="false" export GOAL="install" export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-zmq --enable-reduce-exports LDFLAGS=-static-libstdc++" From 01fc1af361ab3be444d32cfaf0c5a19f5c1fdc47 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 9 Apr 2020 22:17:20 +0800 Subject: [PATCH 11/19] Merge #18569: ci: Add big endian native s390x build 6136a96cdf631f824fb89cf2720422787439acc9 ci: Rename RUN_CI_ON_HOST to DANGER_RUN_CI_ON_HOST (Hennadii Stepanov) 97ba77aa8e58c3c7ff767e6ca804d0e46370e5dc ci: Add native s390x (Hennadii Stepanov) Pull request description: Unlike the Docker wrapped solution (#17591) this PR suggests running on host system directly. This approach makes builds quick and stable (see: #18106). The excerpt from the Travis log: ``` ... Running on host system without docker wrapper ... Byte Order: Big Endian ... ``` ACKs for top commit: MarcoFalke: ACK 6136a96cdf631f824fb89cf2720422787439acc9 Tree-SHA512: 1b591de13e38d10a35217e1de11cbd648a359d18d16eed166fac18ea5788b58cc9fc6d407086ed342b99e57e479efd951a0ea693710177e500eb116316b9a788 --- .cirrus.yml | 2 +- .travis.yml | 31 +++++++++++++++++++++++++++--- ci/test/00_setup_env_mac_host.sh | 1 - ci/test/00_setup_env_s390x_host.sh | 14 ++++++++++++++ ci/test/04_install.sh | 7 ++++--- 5 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 ci/test/00_setup_env_s390x_host.sh diff --git a/.cirrus.yml b/.cirrus.yml index 506bfceb217c6..2dd109af6a1b8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -37,7 +37,7 @@ task: timeout_in: 60m env: MAKEJOBS: "-j9" - RUN_CI_ON_HOST: "1" + DANGER_RUN_CI_ON_HOST: "1" CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" ccache_cache: diff --git a/.travis.yml b/.travis.yml index ebaa556473a3c..32f7b92fddbd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,9 +32,6 @@ cache: ccache: true directories: - $BASE_BUILD_DIR/ci/scratch/.ccache - # macOS - - $HOME/Library/Caches/Homebrew - - /usr/local/Homebrew before_cache: - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi env: @@ -212,6 +209,26 @@ after_success: FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" + - stage: test + name: 's390x native BE [GOAL: install] [bionic] [no depends, no GUI]' + arch: s390x + dist: bionic + addons: + apt: + packages: + - bsdmainutils + - libboost-filesystem-dev + - libboost-system-dev + - libboost-test-dev + - libboost-thread-dev + - libdb++-dev + - libdb-dev + - libevent-dev + env: >- + DANGER_RUN_CI_ON_HOST=true + CI_USE_APT_INSTALL=no + FILE_ENV="./ci/test/00_setup_env_s390x_host.sh" + # s390 build was disabled temporarily because of disk space issues on the Travis VM # # - stage: test @@ -264,6 +281,12 @@ after_success: # Xcode 11.3.1, macOS 10.14, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version osx_image: xcode11.3 + cache: + directories: + - $TRAVIS_BUILD_DIR/ci/scratch/.ccache + - $TRAVIS_BUILD_DIR/releases/$HOST + - $HOME/Library/Caches/Homebrew + - /usr/local/Homebrew addons: homebrew: packages: @@ -277,4 +300,6 @@ after_success: - ccache - zeromq env: >- + DANGER_RUN_CI_ON_HOST=true + CI_USE_APT_INSTALL=no FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index a43d715a65e3f..dea7925e851a7 100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -9,7 +9,6 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos export HOST=x86_64-apple-darwin19 export PIP_PACKAGES="zmq" -export RUN_CI_ON_HOST=true export RUN_UNIT_TESTS=true export RUN_INTEGRATION_TESTS=false export GOAL="install" diff --git a/ci/test/00_setup_env_s390x_host.sh b/ci/test/00_setup_env_s390x_host.sh new file mode 100644 index 0000000000000..8f3063604ef7d --- /dev/null +++ b/ci/test/00_setup_env_s390x_host.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=s390x-linux-gnu +export NO_DEPENDS=1 +export BITCOIN_CONFIG="--with-incompatible-bdb --enable-reduce-exports" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=true +export GOAL="install" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 9a5d4e0242717..92ef17c302f4e 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -34,7 +34,7 @@ fi export P_CI_DIR="$PWD" -if [ -z "$RUN_CI_ON_HOST" ]; then +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" @@ -65,7 +65,7 @@ fi if [[ $DOCKER_NAME_TAG == centos* ]]; then ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES -elif [ "$TRAVIS_OS_NAME" != "osx" ]; then +elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi @@ -76,6 +76,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + DOCKER_EXEC echo $(lscpu | grep Endian) DOCKER_EXEC echo "Free disk space:" DOCKER_EXEC df -h fi @@ -89,7 +90,7 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [ -z "$RUN_CI_ON_HOST" ]; then +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR fi From 04a707aa47de8ae90a75ea6930cdf92a31036791 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 17 Apr 2020 14:00:00 -0400 Subject: [PATCH 12/19] Merge #18683: ci: Disable valgrind functionl tests on forked repos to avoid timeouts faceeae49ae85fb644996fedb970ad9d0e297ca5 ci: Disable valgrind functionl tests on forked repos to avoid timeouts (MarcoFalke) Pull request description: Allows people to fork our repo and run the tests again Also print more cache stats ACKs for top commit: hebasto: ACK faceeae49ae85fb644996fedb970ad9d0e297ca5, tested on my own repo: https://travis-ci.org/github/hebasto/bitcoin/jobs/676257500 Tree-SHA512: 50e44edf94fcb997438eeaf7308b2b58a0854141ecb1fbb0ba7bf5ed662f19b60899f966f579cca90ad5e789234d0e90122d8c2c854da70339058adc3a475fa6 --- ci/test/00_setup_env_native_valgrind.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index f07518e9e77a9..fc10eea0d0145 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -9,6 +9,10 @@ export LC_ALL=C.UTF-8 export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then + export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit +else + export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +fi export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI From f935747dcc857190c1f51066de9d213a5ece3925 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 25 Jun 2020 08:58:08 -0400 Subject: [PATCH 13/19] Merge #18912: ci: Run fuzz testing test cases (bitcoin-core/qa-assets) under valgrind to catch memory errors 3f686d1a286f88af7baa17c0c9dc439559a66e81 ci: Run fuzz testing test cases (bitcoin-core/qa-assets) under valgrind to catch memory errors (practicalswift) Pull request description: Re-introduce the Travis valgrind fuzzing job which was removed by PR #18899. The removal seems to have been made by accident since the removed job does not appear to be the source of the problem the PR set out to fix. --- Run fuzz testing [test cases (bitcoin-core/qa-assets)](https://github.com/bitcoin-core/qa-assets) under `valgrind`. This would have caught `util: Avoid potential uninitialized read in FormatISO8601DateTime(int64_t) by checking gmtime_s/gmtime_r return value` (#18162) and similar cases. This fuzzing job was introduced in #18166. Top commit has no ACKs. Tree-SHA512: 6e2681eb0ade6af465c5ea91ac163a337465d2130ec9880ba57a36d9af7c25682734586a32977dc25972d4f78483f339d680ea48c0ae13cf1dfa52b617aae401 --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 32f7b92fddbd8..e135852c3958f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -264,6 +264,11 @@ after_success: env: >- FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" + - stage: test + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' + env: >- + FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" + - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no wallet]' env: >- From 50cac39d9df7c1bad53426dda4e24e58f3e47d1e Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 6 Aug 2020 09:48:26 +0800 Subject: [PATCH 14/19] Merge #19669: contrib: Fixup valgrind suppressions file fa5288cf5c50d014f046b7741f6da69529037933 contrib: Fixup valgrind suppressions file (MarcoFalke) Pull request description: I am observing this one on bionic with system boost::fs: ``` { Memcheck:Cond fun:__wcsnlen_avx2 fun:wcsnrtombs fun:_ZNKSt7codecvtIwc11__mbstate_tE6do_outERS0_PKwS4_RS4_PcS6_RS6_ fun:_ZN5boost10filesystem11path_traits7convertEPKwS3_RNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt7codecvtIwc11__mbstate_tE fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE ... ACKs for top commit: practicalswift: ACK fa5288cf5c50d014f046b7741f6da69529037933 -- patch looks correct Tree-SHA512: 067e10e932a7f5b13e516134e0cfd3030265c1b582cdfde1cea97042e31399aa40c4590710a39429854c68ad703a0ae9f0b06e9af4cdd81e3cacb042939a84b6 --- contrib/valgrind.supp | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 1ee3e08c51c82..d2c904743d44e 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -123,7 +123,6 @@ Memcheck:Cond ... fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE - fun:unique_path } { Suppress boost warning From a3c59cc77a1a93804bc94c30c677b7c1d1ddf410 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 18 May 2023 18:16:04 +0700 Subject: [PATCH 15/19] partial merge bitcoin#14794: Add AddressSanitizer (ASan) Travis build --- .gitlab-ci.yml | 18 ++++++++++++++++++ ci/dash/matrix.sh | 4 ++++ ci/test/00_setup_env_native_asan.sh | 14 ++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 ci/test/00_setup_env_native_asan.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25ac730fbc273..9a60351fcc1d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -252,6 +252,15 @@ linux64_fuzz-build: variables: BUILD_TARGET: linux64_fuzz +linux64_asan-build: + extends: + - .build-template + - .skip-in-fast-mode-template + needs: + - x86_64-pc-linux-gnu-debug + variables: + BUILD_TARGET: linux64_asan + linux64_tsan-build: extends: - .build-template @@ -315,6 +324,15 @@ linux64_sqlite-test: variables: BUILD_TARGET: linux64_sqlite +linux64_asan-test: + extends: + - .test-template + - .skip-in-fast-mode-template + needs: + - linux64_asan-build + variables: + BUILD_TARGET: linux64_asan + linux64_tsan-test: extends: - .test-template diff --git a/ci/dash/matrix.sh b/ci/dash/matrix.sh index b9ba18b3b9ea3..26dcba42efc32 100755 --- a/ci/dash/matrix.sh +++ b/ci/dash/matrix.sh @@ -11,6 +11,8 @@ export LC_ALL=C.UTF-8 source ./ci/test/00_setup_env.sh # Configure sanitizers options +export ASAN_OPTIONS="" +export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan" @@ -20,6 +22,8 @@ elif [ "$BUILD_TARGET" = "win64" ]; then source ./ci/test/00_setup_env_win64.sh elif [ "$BUILD_TARGET" = "linux64" ]; then source ./ci/test/00_setup_env_native_qt5.sh +elif [ "$BUILD_TARGET" = "linux64_asan" ]; then + source ./ci/test/00_setup_env_native_asan.sh elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then source ./ci/test/00_setup_env_native_tsan.sh elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh new file mode 100644 index 0000000000000..0e15669347930 --- /dev/null +++ b/ci/test/00_setup_env_native_asan.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export NO_DEPENDS=1 +export FUNCTIONAL_TESTS_CONFIG="--exclude wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163) +export RUN_BENCH=true +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" From 028281ae159b1a1d768e95f5b6df51b120d4a17e Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Thu, 1 Jun 2023 16:22:39 +0700 Subject: [PATCH 16/19] fix: add supression for integer conversion in crc32 library --- test/sanitizer_suppressions/ubsan | 1 + 1 file changed, 1 insertion(+) diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 2232bebc7430e..d69e3e8af8960 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -48,6 +48,7 @@ implicit-integer-sign-change:chain.* implicit-integer-sign-change:coins.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h +implicit-integer-sign-change:crc32c/ implicit-integer-sign-change:crypto/* implicit-integer-sign-change:key.cpp implicit-integer-sign-change:noui.cpp From 6b87c21f75e74c52755cf7768ee8a134cdad1282 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Sun, 21 May 2023 16:23:37 +0700 Subject: [PATCH 17/19] fix: rid of UB in qt/rpcconsole.cpp --- src/qt/rpcconsole.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index b1afeacc13d90..d4e4ff99e01ad 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -795,7 +795,9 @@ void RPCConsole::setFontSize(int newSize) { QSettings settings; - newSize = std::min(std::max(newSize, FONT_RANGE.width()), FONT_RANGE.height()); + //don't allow an insane font size + if (newSize < FONT_RANGE.width() || newSize > FONT_RANGE.height()) + return; // temp. store the console content QString str = ui->messagesWidget->toHtml(); From c35746088e7b026ad1a01bf7cf20eeb11f46729e Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Mon, 22 May 2023 02:28:46 +0700 Subject: [PATCH 18/19] fix: get rid UB from integer overflow in transactionview --- src/qt/transactionview.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 1c90414968bb6..00a86857c76ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -333,7 +333,7 @@ void TransactionView::chooseType(int idx) if(!transactionProxyModel) return; transactionProxyModel->setTypeFilter( - typeWidget->itemData(idx).toInt()); + typeWidget->itemData(idx).toUInt()); // Persist settings QSettings settings; settings.setValue("transactionType", idx); From 844ede2eb258eccaa8af942ae61d0d623fdbb4b3 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Wed, 31 May 2023 04:16:47 +0700 Subject: [PATCH 19/19] fix: disable ASAN because currently it fails with unknown memory leaks ``` ********* Finished testing of TrafficGraphDataTests ********* ~BitcoinApplication : Stopping thread ~BitcoinApplication : Stopped thread ==28475==WARNING: invalid path to external symbolizer! ==28475==WARNING: Failed to use and restart external symbolizer! ================================================================= ==28475==ERROR: LeakSanitizer: detected memory leaks Direct leak of 520 byte(s) in 13 object(s) allocated from: #0 0x55fc08e2353d (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0x779153d) #1 0x7f7a7724bd85 () Direct leak of 432 byte(s) in 2 object(s) allocated from: #0 0x55fc08e2353d (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0x779153d) #1 0x55fc0dc63b63 (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0xc5d1b63) #2 0x8604597ff0e8feff () Direct leak of 176 byte(s) in 2 object(s) allocated from: #0 0x55fc08e52cad (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0x77c0cad) #1 0x55fc0d822372 (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0xc190372) Direct leak of 66 byte(s) in 3 object(s) allocated from: #0 0x55fc08e2353d (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0x779153d) #1 0x7f7a77243633 () Direct leak of 24 byte(s) in 1 object(s) allocated from: #0 0x55fc08e2353d (/builds/dashpay/dash/build-ci/dashcore-linux64_asan/src/qt/test/test_dash-qt+0x779153d) #1 0x7f7a77245449 () ``` --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a60351fcc1d4..0d33c1ab9ae80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -252,14 +252,14 @@ linux64_fuzz-build: variables: BUILD_TARGET: linux64_fuzz -linux64_asan-build: - extends: - - .build-template - - .skip-in-fast-mode-template - needs: - - x86_64-pc-linux-gnu-debug - variables: - BUILD_TARGET: linux64_asan +#linux64_asan-build: +# extends: +# - .build-template +# - .skip-in-fast-mode-template +# needs: +# - x86_64-pc-linux-gnu-debug +# variables: +# BUILD_TARGET: linux64_asan linux64_tsan-build: extends: @@ -324,14 +324,14 @@ linux64_sqlite-test: variables: BUILD_TARGET: linux64_sqlite -linux64_asan-test: - extends: - - .test-template - - .skip-in-fast-mode-template - needs: - - linux64_asan-build - variables: - BUILD_TARGET: linux64_asan +#linux64_asan-test: +# extends: +# - .test-template +# - .skip-in-fast-mode-template +# needs: +# - linux64_asan-build +# variables: +# BUILD_TARGET: linux64_asan linux64_tsan-test: extends: