From aa275507814fe3337a63169bc5d6bb0f8d8a12cc Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Sun, 20 Jun 2021 17:50:39 +0200 Subject: [PATCH 01/43] bench: bench.h fixes and improvements --- src/bench/bench.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/bench/bench.h b/src/bench/bench.h index 22f06d8cb8..c4fcd80e33 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -18,16 +18,19 @@ /* * Usage: -static void CODE_TO_TIME(benchmark::Bench& bench) +static void NameOfYourBenchmarkFunction(benchmark::Bench& bench) { - ... do any setup needed... - nanobench::Config().run([&] { - ... do stuff you want to time... + ...do any setup needed... + + bench.run([&] { + ...do stuff you want to time; refer to src/bench/nanobench.h + for more information and the options that can be passed here... }); - ... do any cleanup needed... + + ...do any cleanup needed... } -BENCHMARK(CODE_TO_TIME); +BENCHMARK(NameOfYourBenchmarkFunction); */ @@ -55,7 +58,8 @@ class BenchRunner static void RunAll(const Args& args); }; -} +} // namespace benchmark + // BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); #define BENCHMARK(n) \ benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); From 518fed2a3e268b2c82a7b5d468d888f650275e6c Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Tue, 22 Jun 2021 13:30:51 +0200 Subject: [PATCH 02/43] bench: bench_bitcoin.cpp help fixups - remove unneeded strprintf - consistent punctuation (no EOL periods) - sort helps by order they are printed (alphabetical order) --- src/bench/bench_bitcoin.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 135659f87f..aab777cac1 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -16,11 +16,11 @@ static void SetupBenchArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-asymptote=n1,n2,n3,...", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-asymptote=n1,n2,n3,...", strprintf("Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark"), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_csv=", "Generate CSV file with the most important benchmark results.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_json=", "Generate JSON file with all benchmark results.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-output_csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-output_json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } // parses a comma separated list like "10,20,30,50" From 9939f5eaecf47db368fd536214f274812283050e Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Sun, 20 Jun 2021 17:50:13 +0200 Subject: [PATCH 03/43] doc: update doc/benchmarking.md --- doc/benchmarking.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/doc/benchmarking.md b/doc/benchmarking.md index b6cd86eafe..84d5f2c444 100644 --- a/doc/benchmarking.md +++ b/doc/benchmarking.md @@ -8,8 +8,10 @@ thread queue, wallet balance. Running --------------------- -For benchmarks purposes you only need to compile `bitcoin_bench`. Beware of configuring without `--enable-debug` as this would impact -benchmarking by unlatching log printers and lock analysis. +For benchmarking, you only need to compile `bitcoin_bench`. The bench runner +warns if you configure with `--enable-debug`, but consider if building without +it will impact the benchmark(s) you are interested in by unlatching log printers +and lock analysis. make -C src bitcoin_bench @@ -19,19 +21,28 @@ After compiling bitcoin-core, the benchmarks can be run with: The output will look similar to: ``` -| ns/byte | byte/s | error % | benchmark -|--------------------:|--------------------:|--------:|:---------------------------------------------- -| 64.13 | 15,592,356.01 | 0.1% | `Base58CheckEncode` -| 24.56 | 40,722,672.68 | 0.2% | `Base58Decode` +| ns/op | op/s | err% | total | benchmark +|--------------------:|--------------------:|--------:|----------:|:---------- +| 57,927,463.00 | 17.26 | 3.6% | 0.66 | `AddrManAdd` +| 677,816.00 | 1,475.33 | 4.9% | 0.01 | `AddrManGetAddr` + +... + +| ns/byte | byte/s | err% | total | benchmark +|--------------------:|--------------------:|--------:|----------:|:---------- +| 127.32 | 7,854,302.69 | 0.3% | 0.00 | `Base58CheckEncode` +| 31.95 | 31,303,226.99 | 0.2% | 0.00 | `Base58Decode` + ... ``` Help --------------------- - src/bench/bench_bitcoin --help + src/bench/bench_bitcoin -? -To print options like scaling factor or per-benchmark filter. +To print the various options, like listing the benchmarks without running them +or using a regex filter to only run certain benchmarks. Notes --------------------- From 97125bf0b6f63b1049f4db76fdcdbf365b842053 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 25 Jun 2021 20:49:33 -0400 Subject: [PATCH 04/43] guix: Rebase toolchain on glibc 2.24 (2.27 for riscv64) Support for riscv64 in glibc landed in 2.27 so it's unavoidable that we use 2.27. Running a Bitcoin build with toolchains based on 2.24 for platforms other than riscv64 seem to produce binaries which do not have 2.17 symbols. So use 2.24 since it's more recent and maintained by Debian Stretch. --- contrib/guix/manifest.scm | 43 +++- ...c-2.24-elfm-loadaddr-dynamic-rewrite.patch | 60 +++++ ...bc-2.24-no-build-time-cxx-header-run.patch | 98 +++++++ ..._include__-to-include-asm-syscalls.h.patch | 70 +++++ contrib/guix/patches/glibc-ldd-x86_64.patch | 10 + .../patches/glibc-versioned-locpath.patch | 240 ++++++++++++++++++ 6 files changed, 519 insertions(+), 2 deletions(-) create mode 100644 contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch create mode 100644 contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch create mode 100644 contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch create mode 100644 contrib/guix/patches/glibc-ldd-x86_64.patch create mode 100644 contrib/guix/patches/glibc-versioned-locpath.patch diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index ba168a2a4a..f4df4855fc 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -135,11 +135,25 @@ chain for " target " development.")) (package-with-extra-patches gcc-8 (search-our-patches "gcc-8-sort-libtool-find-output.patch"))) +;; Building glibc with stack smashing protector first landed in glibc 2.25, use +;; this function to disable for older glibcs +;; +;; From glibc 2.25 changelog: +;; +;; * Most of glibc can now be built with the stack smashing protector enabled. +;; It is recommended to build glibc with --enable-stack-protector=strong. +;; Implemented by Nick Alcock (Oracle). +(define (make-glibc-without-ssp xglibc) + (package-with-extra-configure-variable + (package-with-extra-configure-variable + xglibc "libc_cv_ssp" "no") + "libc_cv_ssp_strong" "no")) + (define* (make-bitcoin-cross-toolchain target #:key (base-gcc-for-libc gcc-7) (base-kernel-headers linux-libre-headers-5.4) - (base-libc glibc) ; glibc 2.31 + (base-libc (make-glibc-without-ssp glibc-2.24)) (base-gcc (make-gcc-rpath-link base-gcc))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." @@ -557,6 +571,28 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) +(define-public glibc-2.24 + (package + (inherit glibc) + (version "2.24") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))) + (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")) + (sha256 + (base32 + "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-versioned-locpath.patch" + "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" + "glibc-2.24-no-build-time-cxx-header-run.patch")))))) + +(define glibc-2.27/bitcoin-patched + (package-with-extra-patches glibc-2.27 + (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) + (packages->manifest (append (list ;; The Basics @@ -606,7 +642,10 @@ inspecting signatures in Mach-O binaries.") (make-nsis-with-sde-support nsis-x86_64) osslsigncode)) ((string-contains target "-linux-") - (list (make-bitcoin-cross-toolchain target))) + (list (cond ((string-contains target "riscv64-") + (make-bitcoin-cross-toolchain target #:base-libc glibc-2.27/bitcoin-patched)) + (else + (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch new file mode 100644 index 0000000000..54a7824345 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch @@ -0,0 +1,60 @@ +commit 6b02af31e9a721bb15a11380cd22d53b621711f8 +Author: Szabolcs Nagy +Date: Wed Oct 18 17:26:23 2017 +0100 + + [AARCH64] Rewrite elf_machine_load_address using _DYNAMIC symbol + + This patch rewrites aarch64 elf_machine_load_address to use special _DYNAMIC + symbol instead of _dl_start. + + The static address of _DYNAMIC symbol is stored in the first GOT entry. + Here is the change which makes this solution work (part of binutils 2.24): + https://sourceware.org/ml/binutils/2013-06/msg00248.html + + i386, x86_64 targets use the same method to do this as well. + + The original implementation relies on a trick that R_AARCH64_ABS32 relocation + being resolved at link time and the static address fits in the 32bits. + However, in LP64, normally, the address is defined to be 64 bit. + + Here is the C version one which should be portable in all cases. + + * sysdeps/aarch64/dl-machine.h (elf_machine_load_address): Use + _DYNAMIC symbol to calculate load address. + +diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h +index e86d8b5b63..5a5b8a5de5 100644 +--- a/sysdeps/aarch64/dl-machine.h ++++ b/sysdeps/aarch64/dl-machine.h +@@ -49,26 +49,11 @@ elf_machine_load_address (void) + /* To figure out the load address we use the definition that for any symbol: + dynamic_addr(symbol) = static_addr(symbol) + load_addr + +- The choice of symbol is arbitrary. The static address we obtain +- by constructing a non GOT reference to the symbol, the dynamic +- address of the symbol we compute using adrp/add to compute the +- symbol's address relative to the PC. +- This depends on 32bit relocations being resolved at link time +- and that the static address fits in the 32bits. */ +- +- ElfW(Addr) static_addr; +- ElfW(Addr) dynamic_addr; +- +- asm (" \n" +-" adrp %1, _dl_start; \n" +-" add %1, %1, #:lo12:_dl_start \n" +-" ldr %w0, 1f \n" +-" b 2f \n" +-"1: \n" +-" .word _dl_start \n" +-"2: \n" +- : "=r" (static_addr), "=r" (dynamic_addr)); +- return dynamic_addr - static_addr; ++ _DYNAMIC sysmbol is used here as its link-time address stored in ++ the special unrelocated first GOT entry. */ ++ ++ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden; ++ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic (); + } + + /* Set up the loaded object described by L so its unrelocated PLT diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch new file mode 100644 index 0000000000..5d7a148c62 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch @@ -0,0 +1,98 @@ +commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 +Author: Florian Weimer +Date: Wed Sep 21 10:45:32 2016 +0200 + + Avoid running $(CXX) during build to obtain header file paths + + This reduces the build time somewhat and is particularly noticeable + during rebuilds with few code changes. + +diff --git a/Makerules b/Makerules +index 7e4077ee50..c338850de5 100644 +--- a/Makerules ++++ b/Makerules +@@ -121,14 +121,10 @@ ifneq (,$(CXX)) + # will be used instead of /usr/include/stdlib.h and /usr/include/math.h. + before-compile := $(common-objpfx)cstdlib $(common-objpfx)cmath \ + $(before-compile) +-cstdlib=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cstdlib:/{s/:$$//;p}") +-$(common-objpfx)cstdlib: $(cstdlib) ++$(common-objpfx)cstdlib: $(c++-cstdlib-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ +-cmath=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cmath:/{s/:$$//;p}") +-$(common-objpfx)cmath: $(cmath) ++$(common-objpfx)cmath: $(c++-cmath-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ + endif +diff --git a/config.make.in b/config.make.in +index 95c6f36876..04a8b3ed7f 100644 +--- a/config.make.in ++++ b/config.make.in +@@ -45,6 +45,8 @@ defines = @DEFINES@ + sysheaders = @sysheaders@ + sysincludes = @SYSINCLUDES@ + c++-sysincludes = @CXX_SYSINCLUDES@ ++c++-cstdlib-header = @CXX_CSTDLIB_HEADER@ ++c++-cmath-header = @CXX_CMATH_HEADER@ + all-warnings = @all_warnings@ + enable-werror = @enable_werror@ + +diff --git a/configure b/configure +index 17625e1041..6ff252744b 100755 +--- a/configure ++++ b/configure +@@ -635,6 +635,8 @@ BISON + INSTALL_INFO + PERL + BASH_SHELL ++CXX_CMATH_HEADER ++CXX_CSTDLIB_HEADER + CXX_SYSINCLUDES + SYSINCLUDES + AUTOCONF +@@ -5054,6 +5056,18 @@ fi + + + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++ ++ ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following +diff --git a/configure.ac b/configure.ac +index 33bcd62180..9938ab0dc2 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1039,6 +1039,18 @@ fi + AC_SUBST(SYSINCLUDES) + AC_SUBST(CXX_SYSINCLUDES) + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++AC_SUBST(CXX_CSTDLIB_HEADER) ++AC_SUBST(CXX_CMATH_HEADER) ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch new file mode 100644 index 0000000000..39c995ffb5 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch @@ -0,0 +1,70 @@ +From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 31 Dec 2018 09:26:42 -0800 +Subject: [PATCH] riscv: Use __has_include__ to include [BZ + #24022] + + has been removed by + +commit 27f8899d6002e11a6e2d995e29b8deab5aa9cc25 +Author: David Abdurachmanov +Date: Thu Nov 8 20:02:39 2018 +0100 + + riscv: add asm/unistd.h UAPI header + + Marcin Juszkiewicz reported issues while generating syscall table for riscv + using 4.20-rc1. The patch refactors our unistd.h files to match some other + architectures. + + - Add asm/unistd.h UAPI header, which has __ARCH_WANT_NEW_STAT only for 64-bit + - Remove asm/syscalls.h UAPI header and merge to asm/unistd.h + - Adjust kernel asm/unistd.h + + So now asm/unistd.h UAPI header should show all syscalls for riscv. + + may be restored by + +Subject: [PATCH] riscv: restore asm/syscalls.h UAPI header +Date: Tue, 11 Dec 2018 09:09:35 +0100 + +UAPI header asm/syscalls.h was merged into UAPI asm/unistd.h header, +which did resolve issue with missing syscalls macros resulting in +glibc (2.28) build failure. It also broke glibc in a different way: +asm/syscalls.h is being used by glibc. I noticed this while doing +Fedora 30/Rawhide mass rebuild. + +The patch returns asm/syscalls.h header and incl. it into asm/unistd.h. +I plan to send a patch to glibc to use asm/unistd.h instead of +asm/syscalls.h + +In the meantime, we use __has_include__, which was added to GCC 5, to +check if exists before including it. Tested with +build-many-glibcs.py for riscv against kernel 4.19.12 and 4.20-rc7. + + [BZ #24022] + * sysdeps/unix/sysv/linux/riscv/flush-icache.c: Check if + exists with __has_include__ before including it. +--- + sysdeps/unix/sysv/linux/riscv/flush-icache.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/sysdeps/unix/sysv/linux/riscv/flush-icache.c b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +index d612ef4c6c..0b2042620b 100644 +--- a/sysdeps/unix/sysv/linux/riscv/flush-icache.c ++++ b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +@@ -21,7 +21,11 @@ + #include + #include + #include +-#include ++#if __has_include__ () ++# include ++#else ++# include ++#endif + + typedef int (*func_type) (void *, void *, unsigned long int); + +-- +2.31.1 + diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch new file mode 100644 index 0000000000..b1b6d5a548 --- /dev/null +++ b/contrib/guix/patches/glibc-ldd-x86_64.patch @@ -0,0 +1,10 @@ +By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas +it's in 'lib/' for us. This patch fixes that. + +--- glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2012-12-25 04:02:13.000000000 +0100 ++++ glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2013-09-15 23:08:03.000000000 +0200 +@@ -1,3 +1,3 @@ + /LD_TRACE_LOADED_OBJECTS=1/a\ + add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" +-s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ ++s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch new file mode 100644 index 0000000000..bc7652127f --- /dev/null +++ b/contrib/guix/patches/glibc-versioned-locpath.patch @@ -0,0 +1,240 @@ +The format of locale data can be incompatible between libc versions, and +loading incompatible data can lead to 'setlocale' returning EINVAL at best +or triggering an assertion failure at worst. See +https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html +for background information. + +To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' +variable, and to look for locale data in version-specific sub-directories of +that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in +/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. + +That way, a single 'GUIX_LOCPATH' setting can work even if different libc +versions coexist on the system. + +--- a/locale/newlocale.c ++++ b/locale/newlocale.c +@@ -30,6 +30,7 @@ + /* Lock for protecting global data. */ + __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) + ++extern error_t compute_locale_search_path (char **, size_t *); + + /* Use this when we come along an error. */ + #define ERROR_RETURN \ +@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + __locale_t result_ptr; + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + int cnt; + size_t names_len; + +@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') +- { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0) +- return NULL; +- +- if (__argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- return NULL; +- } ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) ++ return NULL; + + /* Get the names for the locales we are interested in. We either + allow a composite name or a single name. */ +diff --git a/locale/setlocale.c b/locale/setlocale.c +index ead030d..0c0e314 100644 +--- a/locale/setlocale.c ++++ b/locale/setlocale.c +@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) + } + } + ++/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as ++ a colon-separated list. Return ENOMEN on error, zero otherwise. */ ++error_t ++compute_locale_search_path (char **locale_path, size_t *locale_path_len) ++{ ++ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); ++ char *locpath_var = getenv ("LOCPATH"); ++ ++ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') ++ { ++ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These ++ entries are systematically prefixed with "/X.Y" where "X.Y" is the ++ libc version. */ ++ if (__argz_create_sep (guix_locpath_var, ':', ++ locale_path, locale_path_len) != 0 ++ || __argz_suffix_entries (locale_path, locale_path_len, ++ "/" VERSION) != 0) ++ goto bail_out; ++ } ++ ++ if (locpath_var != NULL && locpath_var[0] != '\0') ++ { ++ char *reg_locale_path = NULL; ++ size_t reg_locale_path_len = 0; ++ ++ if (__argz_create_sep (locpath_var, ':', ++ ®_locale_path, ®_locale_path_len) != 0) ++ goto bail_out; ++ ++ if (__argz_append (locale_path, locale_path_len, ++ reg_locale_path, reg_locale_path_len) != 0) ++ goto bail_out; ++ ++ free (reg_locale_path); ++ } ++ ++ if (*locale_path != NULL) ++ { ++ /* Append the system default locale directory. */ ++ if (__argz_add_sep (locale_path, locale_path_len, ++ _nl_default_locale_path, ':') != 0) ++ goto bail_out; ++ } ++ ++ return 0; ++ ++ bail_out: ++ free (*locale_path); ++ *locale_path = NULL; ++ *locale_path_len = 0; ++ ++ return ENOMEM; ++} ++ + char * + setlocale (int category, const char *locale) + { + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + char *composite; + + /* Sanity check for CATEGORY argument. */ +@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) + { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0 +- || __argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- { +- __libc_rwlock_unlock (__libc_setlocale_lock); +- return NULL; +- } ++ __libc_rwlock_unlock (__libc_setlocale_lock); ++ return NULL; + } + + if (category == LC_ALL) +diff --git a/string/Makefile b/string/Makefile +index 8424a61..f925503 100644 +--- a/string/Makefile ++++ b/string/Makefile +@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ + swab strfry memfrob memmem rawmemchr strchrnul \ + $(addprefix argz-,append count create ctsep next \ + delete extract insert stringify \ +- addsep replace) \ ++ addsep replace suffix) \ + envz basename \ + strcoll_l strxfrm_l string-inlines memrchr \ + xpg-strerror strerror_l +diff --git a/string/argz-suffix.c b/string/argz-suffix.c +new file mode 100644 +index 0000000..505b0f2 +--- /dev/null ++++ b/string/argz-suffix.c +@@ -0,0 +1,56 @@ ++/* Copyright (C) 2015 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ Contributed by Ludovic Courtès . ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++#include ++#include ++#include ++ ++ ++error_t ++__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) ++ ++{ ++ size_t suffix_len = strlen (suffix); ++ size_t count = __argz_count (*argz, *argz_len); ++ size_t new_argz_len = *argz_len + count * suffix_len; ++ char *new_argz = malloc (new_argz_len); ++ ++ if (new_argz) ++ { ++ char *p = new_argz, *entry; ++ ++ for (entry = *argz; ++ entry != NULL; ++ entry = argz_next (*argz, *argz_len, entry)) ++ { ++ p = stpcpy (p, entry); ++ p = stpcpy (p, suffix); ++ p++; ++ } ++ ++ free (*argz); ++ *argz = new_argz; ++ *argz_len = new_argz_len; ++ ++ return 0; ++ } ++ else ++ return ENOMEM; ++} ++weak_alias (__argz_suffix_entries, argz_suffix_entries) +diff --git a/string/argz.h b/string/argz.h +index bb62a31..d276a35 100644 +--- a/string/argz.h ++++ b/string/argz.h +@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, + const char *__restrict __str, + const char *__restrict __with, + unsigned int *__restrict __replace_count); ++ ++/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, ++ and ENOMEN if memory cannot be allocated. */ ++extern error_t __argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++extern error_t argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++ + + /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there + are no more. If entry is NULL, then the first entry is returned. This From 376942e439c7d7c0fe48f1af922660799eb863ce Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 28 Jun 2021 16:55:32 -0400 Subject: [PATCH 05/43] guix: Build depends/qt with our platform definition Our 'bitcoin-linux-g++' definition better integrates with our depends system than the stock linux-g++-64 definition. This fixes a bug whereby Guix builds on x86_64 for x86_64 did not produce a QMinimalIntegrationPlugin and led to bitcoin-qt not being built. --- contrib/guix/libexec/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 6741328473..9351c54229 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -214,6 +214,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 From f8e897327699c192ce141ade21a0fe70412355e8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 2 Jul 2021 14:10:33 -0400 Subject: [PATCH 06/43] guix: Also sort SHA256SUMS.part --- contrib/guix/libexec/build.sh | 1 + contrib/guix/libexec/codesign.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 9351c54229..e457840d15 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -446,5 +446,6 @@ mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ find "$ACTUAL_OUTDIR" -type f } | xargs realpath --relative-base="$PWD" \ | xargs sha256sum \ + | sort -k2 \ | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part ) diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index b1eec686ec..f484ac5774 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -108,5 +108,6 @@ mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ find "$ACTUAL_OUTDIR" -type f } | xargs realpath --relative-base="$PWD" \ | xargs sha256sum \ + | sort -k2 \ | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part ) From a000c9d8b721b6d2cb0e8eaf546cc78e5efdc55e Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 14 Jun 2021 23:37:40 +0300 Subject: [PATCH 07/43] qt: Do not extend recent transaction width to address/label string --- src/qt/overviewpage.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 1fd1ff3142..618e8a771e 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -72,7 +72,6 @@ class TxViewDelegate : public QAbstractItemDelegate painter->setPen(foreground); QRect boundingRect; painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); - int address_rect_min_width = boundingRect.width(); if (index.data(TransactionTableModel::WatchonlyRole).toBool()) { @@ -80,7 +79,6 @@ class TxViewDelegate : public QAbstractItemDelegate QRect watchonlyRect(boundingRect.right() + 5, mainRect.top()+ypad+halfheight, 16, halfheight); iconWatchonly = platformStyle->TextColorIcon(iconWatchonly); iconWatchonly.paint(painter, watchonlyRect); - address_rect_min_width += 5 + watchonlyRect.width(); } if(amount < 0) @@ -109,7 +107,8 @@ class TxViewDelegate : public QAbstractItemDelegate QRect date_bounding_rect; painter->drawText(amountRect, Qt::AlignLeft | Qt::AlignVCenter, GUIUtil::dateTimeStr(date), &date_bounding_rect); - const int minimum_width = std::max(address_rect_min_width, amount_bounding_rect.width() + date_bounding_rect.width()); + // 0.4*date_bounding_rect.width() is used to visually distinguish a date from an amount. + const int minimum_width = 1.4 * date_bounding_rect.width() + amount_bounding_rect.width(); const auto search = m_minimum_width.find(index.row()); if (search == m_minimum_width.end() || search->second != minimum_width) { m_minimum_width[index.row()] = minimum_width; From b80995d2ef118f0d81f88f01fda1f463bb0079c9 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Tue, 15 Jun 2021 00:20:00 +0300 Subject: [PATCH 08/43] qt: Draw "eye" sign at the beginning of watch-only addresses --- src/qt/overviewpage.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 618e8a771e..864a62edc8 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -69,18 +69,18 @@ class TxViewDelegate : public QAbstractItemDelegate foreground = brush.color(); } - painter->setPen(foreground); - QRect boundingRect; - painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); - - if (index.data(TransactionTableModel::WatchonlyRole).toBool()) - { + if (index.data(TransactionTableModel::WatchonlyRole).toBool()) { QIcon iconWatchonly = qvariant_cast(index.data(TransactionTableModel::WatchonlyDecorationRole)); - QRect watchonlyRect(boundingRect.right() + 5, mainRect.top()+ypad+halfheight, 16, halfheight); + QRect watchonlyRect(addressRect.left(), addressRect.top(), 16, addressRect.height()); iconWatchonly = platformStyle->TextColorIcon(iconWatchonly); iconWatchonly.paint(painter, watchonlyRect); + addressRect.setLeft(addressRect.left() + watchonlyRect.width() + 5); } + painter->setPen(foreground); + QRect boundingRect; + painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); + if(amount < 0) { foreground = COLOR_NEGATIVE; From 7ccb87c3a23663e6c9c819c7597346679897c394 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 28 Jun 2021 17:16:48 +0300 Subject: [PATCH 09/43] qt: Emit dataChanged signal to dynamically re-sort Peers table --- src/qt/peertablemodel.cpp | 4 +++- src/qt/peertablemodel.h | 3 --- src/qt/rpcconsole.cpp | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index b324693692..1b7fda6e77 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -179,5 +179,7 @@ void PeerTableModel::refresh() m_peers_data.swap(new_peers_data); } - Q_EMIT changed(); + const auto top_left = index(0, 0); + const auto bottom_right = index(rowCount() - 1, columnCount() - 1); + Q_EMIT dataChanged(top_left, bottom_right); } diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0ff1b5dba7..0d841ebf28 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -73,9 +73,6 @@ class PeerTableModel : public QAbstractTableModel public Q_SLOTS: void refresh(); -Q_SIGNALS: - void changed(); - private: //! Internal peer data structure. QList m_peers_data{}; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index c973fdbe78..9c57816f91 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -35,6 +35,7 @@ #endif #include +#include #include #include #include @@ -686,7 +687,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // peer table signal handling - update peer details when selecting new node connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::updateDetailWidget); - connect(model->getPeerTableModel(), &PeerTableModel::changed, this, &RPCConsole::updateDetailWidget); + connect(model->getPeerTableModel(), &QAbstractItemModel::dataChanged, [this] { updateDetailWidget(); }); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); From 7540757ef050afd949a47e88ff429036de82c63e Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 9 Jun 2021 10:53:13 +0800 Subject: [PATCH 10/43] build: remove --enable-determinism configure option --- configure.ac | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/configure.ac b/configure.ac index abc126b619..63ab88f881 100644 --- a/configure.ac +++ b/configure.ac @@ -318,13 +318,6 @@ AC_ARG_ENABLE([gprof], [enable_gprof=$enableval], [enable_gprof=no]) -dnl Pass compiler & linker flags that make builds deterministic -AC_ARG_ENABLE([determinism], - [AS_HELP_STRING([--enable-determinism], - [Enable compilation flags that make builds deterministic (default is no)])], - [enable_determinism=$enableval], - [enable_determinism=no]) - dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], @@ -931,12 +924,6 @@ if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"],, [[$LDFLAG_WERROR]]) fi -if test x$enable_determinism = xyes; then - if test x$TARGET_OS = xwindows; then - AX_CHECK_LINK_FLAG([[-Wl,--no-insert-timestamp]], [LDFLAGS="$LDFLAGS -Wl,--no-insert-timestamp"],, [[$LDFLAG_WERROR]]) - fi -fi - AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, From 7d9b99d49c3194db77d6a90f5f676c7133e55c3a Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 28 Jun 2021 11:17:18 +0200 Subject: [PATCH 11/43] doc: Remove unused section from release process --- doc/release-process.md | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/doc/release-process.md b/doc/release-process.md index 3ead1181b9..546169df8d 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -278,7 +278,7 @@ The `*-debug*` files generated by the gitian build contain debug symbols for troubleshooting by developers. It is assumed that anyone that is interested in debugging can run gitian to generate the files for themselves. To avoid end-user confusion about which file to pick, as well as save storage -space *do not upload these to the bitcoin.org server, nor put them in the torrent*. +space *do not upload these to the bitcoincore.org server, nor put them in the torrent*. - GPG-sign it, delete the unsigned file: ``` @@ -288,7 +288,7 @@ rm SHA256SUMS (the digest algorithm is forced to sha256 to avoid confusion of the `Hash:` header that GPG adds with the SHA256 used for the files) Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spurious/nonsensical entry. -- Upload zips and installers, as well as `SHA256SUMS.asc` from last step, to the bitcoin.org server +- Upload zips and installers, as well as `SHA256SUMS.asc` from last step, to the bitcoincore.org server into `/var/www/bin/bitcoin-core-${VERSION}` - A `.torrent` will appear in the directory after a few minutes. Optionally help seed this torrent. To get the `magnet:` URI use: @@ -296,24 +296,9 @@ Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spur transmission-show -m ``` Insert the magnet URI into the announcement sent to mailing lists. This permits -people without access to `bitcoin.org` to download the binary distribution. +people without access to `bitcoincore.org` to download the binary distribution. Also put it into the `optional_magnetlink:` slot in the YAML file for -bitcoin.org (see below for bitcoin.org update instructions). - -- Update bitcoin.org version - - - First, check to see if the Bitcoin.org maintainers have prepared a - release: https://github.com/bitcoin-dot-org/bitcoin.org/labels/Core - - - If they have, it will have previously failed their CI - checks because the final release files weren't uploaded. - Trigger a CI rebuild---if it passes, merge. - - - If they have not prepared a release, follow the Bitcoin.org release - instructions: https://github.com/bitcoin-dot-org/bitcoin.org/blob/master/docs/adding-events-release-notes-and-alerts.md#release-notes - - - After the pull request is merged, the website will automatically show the newest version within 15 minutes, as well - as update the OS download links. +bitcoincore.org. - Update other repositories and websites for new version From 3b4a5ad4d32fb0485de196dc10eaf9a77203f1a4 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 4 Jul 2021 14:22:36 +0200 Subject: [PATCH 12/43] fuzz: Move ConsumeTxDestination to cpp file Moving the implementation out of the header will reduce compile time --- src/test/fuzz/util.cpp | 31 +++++++++++++++++++++++++++++++ src/test/fuzz/util.h | 31 +------------------------------ 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index bcf0b0ce72..a71b7e32fd 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -304,3 +304,34 @@ uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept }) : fuzzed_data_provider.ConsumeIntegral(); } + +CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + CTxDestination tx_destination; + CallOneOf( + fuzzed_data_provider, + [&] { + tx_destination = CNoDestination{}; + }, + [&] { + tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + WitnessUnknown witness_unknown{}; + witness_unknown.version = fuzzed_data_provider.ConsumeIntegral(); + const std::vector witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes(40); + witness_unknown.length = witness_unknown_program_1.size(); + std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); + tx_destination = witness_unknown; + }); + return tx_destination; +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 023dcdb3e5..60dc9050fe 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -178,36 +178,7 @@ template return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; } -[[nodiscard]] inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - CTxDestination tx_destination; - CallOneOf( - fuzzed_data_provider, - [&] { - tx_destination = CNoDestination{}; - }, - [&] { - tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; - }, - [&] { - tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral(); - const std::vector witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes(40); - witness_unknown.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); - tx_destination = witness_unknown; - }); - return tx_destination; -} +[[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; template [[nodiscard]] bool MultiplicationOverflow(const T i, const T j) noexcept From 1b38530a71511d6228423a8a002bd3b824f47f0d Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 4 Jul 2021 16:38:51 +0200 Subject: [PATCH 13/43] fuzz: Improve ConsumeTxDestination * Assert when a type is missing * Add missing WitnessV1Taproot * Limit WitnessUnknown to version [2, 16], to avoid abiguity * Limit WitnessUnknown to size [2, 40], to avoid invalid sizes --- src/test/fuzz/util.cpp | 16 ++++++++++++---- src/test/fuzz/util.h | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index a71b7e32fd..ece3214ed5 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include #include @@ -308,7 +309,7 @@ uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept { CTxDestination tx_destination; - CallOneOf( + const size_t call_size{CallOneOf( fuzzed_data_provider, [&] { tx_destination = CNoDestination{}; @@ -325,13 +326,20 @@ CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) no [&] { tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; }, + [&] { + tx_destination = WitnessV1Taproot{XOnlyPubKey{ConsumeUInt256(fuzzed_data_provider)}}; + }, [&] { WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral(); - const std::vector witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes(40); + witness_unknown.version = fuzzed_data_provider.ConsumeIntegralInRange(2, 16); + std::vector witness_unknown_program_1{fuzzed_data_provider.ConsumeBytes(40)}; + if (witness_unknown_program_1.size() < 2) { + witness_unknown_program_1 = {0, 0}; + } witness_unknown.length = witness_unknown_program_1.size(); std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); tx_destination = witness_unknown; - }); + })}; + Assert(call_size == std::variant_size_v); return tx_destination; } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 60dc9050fe..9f09395a9a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -37,7 +37,7 @@ #include template -void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) +size_t CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) { constexpr size_t call_size{sizeof...(callables)}; static_assert(call_size >= 1); @@ -45,6 +45,7 @@ void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) size_t i{0}; ((i++ == call_index ? callables() : void()), ...); + return call_size; } template From c9dbce0c6019fc2123ed484690fb9bdcb3bbf132 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 4 Jul 2021 21:18:59 +0200 Subject: [PATCH 14/43] fuzz: Simplify CTxDestination fuzzing in the script target The WitnessUnknown operators == and < are already called indirectly by the corresponding CTxDestination operators. --- src/test/fuzz/script.cpp | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index b87bcf2ef5..1d5f37d109 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -184,25 +184,12 @@ FUZZ_TARGET_INIT(script, initialize_script) } { - WitnessUnknown witness_unknown_1{}; - witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral(); - const std::vector witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes(40); - witness_unknown_1.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); - - WitnessUnknown witness_unknown_2{}; - witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral(); - const std::vector witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes(40); - witness_unknown_2.length = witness_unknown_program_2.size(); - std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); - - (void)(witness_unknown_1 == witness_unknown_2); - (void)(witness_unknown_1 < witness_unknown_2); - } - - { - const CTxDestination tx_destination_1 = ConsumeTxDestination(fuzzed_data_provider); + const CTxDestination tx_destination_1{ + fuzzed_data_provider.ConsumeBool() ? + DecodeDestination(fuzzed_data_provider.ConsumeRandomLengthString()) : + ConsumeTxDestination(fuzzed_data_provider)}; const CTxDestination tx_destination_2 = ConsumeTxDestination(fuzzed_data_provider); + (void)(tx_destination_1 == tx_destination_2); (void)(tx_destination_1 < tx_destination_2); } From 9114e7958a68731fcb285bc55272355301ecb4b2 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 4 Jul 2021 21:24:27 +0200 Subject: [PATCH 15/43] fuzz: Move CTxDestination fuzzing to script fuzz target No need to split it over several targets --- src/test/fuzz/integer.cpp | 16 ---------------- src/test/fuzz/key_io.cpp | 11 ----------- src/test/fuzz/script.cpp | 19 +++++++++++++++++-- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index e9fa343896..e28e2feb0a 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -16,8 +16,6 @@ #include #include #include -#include -#include