diff --git a/.cirrus.yml b/.cirrus.yml index b595d9ed1e..f9deb803d0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -51,10 +51,6 @@ global_task_template: &GLOBAL_TASK_TEMPLATE << : *BASE_TEMPLATE << : *MAIN_TEMPLATE -depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE - depends_sdk_cache: - folder: "depends/sdk-sources" - compute_credits_template: &CREDITS_TEMPLATE # https://cirrus-ci.org/pricing/#compute-credits # Only use credits for pull requests to the main repo @@ -213,7 +209,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[depends, sanitizers: thread (TSan), no gui] [jammy]' + name: '[TSan, depends, no gui] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:jammy @@ -225,7 +221,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[depends, sanitizers: memory (MSan)] [focal]' + name: '[MSan, depends] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -234,7 +230,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [jammy]' + name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:jammy @@ -243,7 +239,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' + name: '[fuzzer,address,undefined,integer, no depends] [focal]' only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_BASE_BRANCH == $CIRRUS_DEFAULT_BRANCH << : *GLOBAL_TASK_TEMPLATE container: @@ -278,12 +274,16 @@ task: task: name: 'macOS 10.15 [gui, no tests] [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE - << : *GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE + macos_sdk_cache: + folder: "depends/SDKs/$MACOS_SDK" + fingerprint_key: "$MACOS_SDK" + << : *MAIN_TEMPLATE alias: macos container: image: ubuntu:focal env: + MACOS_SDK: "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_mac.sh" copy_artifacts_script: @@ -296,7 +296,7 @@ task: task: name: 'macOS 11 native [gui] [no depends]' brew_install_script: - - brew install boost libevent berkeley-db4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt + - brew install boost libevent berkeley-db@4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt << : *GLOBAL_TASK_TEMPLATE osx_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) @@ -309,13 +309,15 @@ task: task: name: 'ARM64 Android APK [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE << : *BASE_TEMPLATE - alias: android + android_sdk_cache: + folder: "depends/SDKs/android" + fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=22.1.7171670" depends_sources_cache: folder: "depends/sources" fingerprint_script: git rev-list -1 HEAD ./depends << : *MAIN_TEMPLATE + alias: android container: image: ubuntu:focal env: diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h index 5978b9a0d9..57f2f144ff 100644 --- a/build_msvc/libsecp256k1_config.h +++ b/build_msvc/libsecp256k1_config.h @@ -29,4 +29,4 @@ #define ECMULT_GEN_PREC_BITS 4 #define ECMULT_WINDOW_SIZE 15 -#endif /* BITCOIN_LIBSECP256K1_CONFIG_H */ +#endif // BITCOIN_LIBSECP256K1_CONFIG_H diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 251ac76cad..cf203b4940 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -17,6 +17,6 @@ ${CI_RETRY_EXE} pip3 install mypy==0.910 ${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 ${CI_RETRY_EXE} pip3 install vulture==2.3 -SHELLCHECK_VERSION=v0.7.2 +SHELLCHECK_VERSION=v0.8.0 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 702b84d152..d8c23bd26b 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -15,17 +15,22 @@ fi DOCKER_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" -OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" -if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi -if [ -n "$ANDROID_TOOLS_URL" ]; then - ANDROID_TOOLS_PATH=$DEPENDS_DIR/sdk-sources/android-tools.zip - - DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi DOCKER_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools" DOCKER_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools" DOCKER_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" @@ -38,9 +43,6 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" fi -if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set "${HOST}-g++" \$\(which "${HOST}-g++-posix"\) fi diff --git a/configure.ac b/configure.ac index d0b91a7c35..d34a1694ce 100644 --- a/configure.ac +++ b/configure.ac @@ -323,6 +323,11 @@ AC_ARG_ENABLE([external-signer], [use_external_signer=$enableval], [use_external_signer=yes]) +AC_ARG_ENABLE([lto], + [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])], + [enable_lto=$enableval], + [enable_lto=no]) + AC_LANG_PUSH([C++]) dnl Check for a flag to turn compiler warnings into errors. This is helpful for checks which may @@ -370,6 +375,11 @@ if test "x$enable_debug" = xyes; then AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) fi +if test "x$enable_lto" = "xyes"; then + AX_CHECK_COMPILE_FLAG([-flto], [LTO_CXXFLAGS="$LTO_CXXFLAGS -flto"], [AC_MSG_ERROR([compile failed with -flto])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-flto], [LTO_LDFLAGS="$LTO_LDFLAGS -flto"], [AC_MSG_ERROR([link failed with -flto])], [$CXXFLAG_WERROR]) +fi + if test x$use_sanitizers != x; then dnl First check if the compiler accepts flags. If an incompatible pair like dnl -fsanitize=address,thread is used here, this check will fail. This will also @@ -581,7 +591,7 @@ CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" +CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -683,8 +693,8 @@ case $host in dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - if test "x$use_bdb" != xno && $BREW list --versions berkeley-db4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then - bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) + if test "x$use_bdb" != xno && $BREW list --versions berkeley-db@4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then + bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null) dnl This must precede the call to BITCOIN_FIND_BDB48 below. BDB_CFLAGS="-I$bdb_prefix/include" BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" @@ -694,8 +704,8 @@ case $host in export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi - if $BREW list --versions qt5 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix qt5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" + if $BREW list --versions qt@5 >/dev/null; then + export PKG_CONFIG_PATH="$($BREW --prefix qt@5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi case $host in @@ -1044,9 +1054,6 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ [AC_MSG_RESULT([no])] ) -dnl thread_local is currently disabled when building with glibc back compat. -dnl Our minimum supported glibc is 2.17, however support for thread_local -dnl did not arrive in glibc until 2.18. if test "x$use_thread_local" = xyes || test "x$use_thread_local" = xauto; then TEMP_LDFLAGS="$LDFLAGS" LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" @@ -1830,6 +1837,8 @@ AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) +AC_SUBST(LTO_CXXFLAGS) +AC_SUBST(LTO_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) @@ -1931,7 +1940,7 @@ if test x$bitcoin_enable_qt != xno; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" -if test x$enable_fuzz == xno; then +if test x$enable_fuzz = xno; then echo " with test = $use_tests" else echo " with test = not building test_bitcoin because fuzzing is enabled" @@ -1946,6 +1955,7 @@ echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" +echo " LTO = $enable_lto" echo echo " target os = $TARGET_OS" echo " build os = $build_os" @@ -1954,7 +1964,7 @@ echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index ef421aebb1..677557b8fa 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -121,6 +121,21 @@ def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' return binary.has_relocations +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. @@ -177,7 +192,8 @@ def check_control_flow(binary) -> bool: ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION) + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), ], 'MACHO': [ ('PIE', check_PIE), diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 136a9b70c1..15d4e729ac 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -19,35 +19,31 @@ # https://github.com/lief-project/LIEF/pull/562 LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) -# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases +# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6) +# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases +# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases # -# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial§ion=all) -# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial§ion=all) +# - g++ version 5.3.1 +# - libc version 2.23 # -# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General +# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# -# Taking the minimum of these as our target. -# -# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: -# GCC 4.8.5: GCC_4.8.0 -# (glibc) GLIBC_2_17 +# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - lief.ELF.ARCH.i386: (2,17), - lief.ELF.ARCH.x86_64: (2,17), - lief.ELF.ARCH.ARM: (2,17), - lief.ELF.ARCH.AARCH64:(2,17), - lief.ELF.ARCH.PPC64: (2,17), + lief.ELF.ARCH.i386: (2,18), + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,18), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), LIEF_ELF_ARCH_RISCV: (2,27), }, 'LIBATOMIC': (1,0), diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 0af7cdf5e6..01df863ac0 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -70,16 +70,18 @@ def test_PE(self): write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed HIGH_ENTROPY_VA')) + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 5246375fe3..d699e85026 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -44,7 +44,7 @@ def test_ELF(self): self.skipTest("test not available for RISC-V") # nextup was introduced in GLIBC 2.24, so is newer than our supported - # glibc (2.17), and available in our release build environment (2.24). + # glibc (2.18), and available in our release build environment (2.24). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean index 9fa17191e8..9af0a793cf 100755 --- a/contrib/guix/guix-clean +++ b/contrib/guix/guix-clean @@ -34,7 +34,7 @@ check_tools cat mkdir make git guix # under_dir() { local path_residue - path_residue="${2##${1}}" + path_residue="${2##"${1}"}" if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then return 1 else diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e009f97c60..596d0ca1fb 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -238,9 +238,6 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" -case "$HOST" in - *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;; -esac # CFLAGS HOST_CFLAGS="-O2 -g" @@ -263,7 +260,7 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by # avoiding a PowerPC64 optimisation available in glibc 2.22 and later. # https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html case "$HOST" in diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 40ae4b5208..1f287e9bbb 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,10 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=../../shell/realpath.bash +# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash -# shellcheck source=../../shell/git-utils.bash +# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 36a4ea08f6..81c88a2ae7 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -62,6 +62,12 @@ http_get() { sha256_check "${3}" "${2}" } +# Ensure the commands we use exist on the system +if ! check_exists patch; then + echo "Command-line tool 'patch' not found. Install patch and try again." + exit 1 +fi + mkdir -p "${BDB_PREFIX}" http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index 457d8f5e64..ebef1d2db0 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -81,7 +81,7 @@ def run(): print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: - with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: with tarfile.open(mode="w", fileobj=gzf) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index 1f93474fa0..b71ce2f34b 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -234,3 +234,62 @@ Histogram of block connection times in milliseconds (ms). [16, 32) 9 | | [32, 64) 4 | | ``` + +### log_utxocache_flush.py + +A BCC Python script to log the cache and index flushes. Based on the +`utxocache:flush` tracepoint. + +```bash +$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind +``` + +``` +Logging utxocache flushes. Ctrl-C to end... +Duration (µs) Mode Coins Count Memory Usage Prune Full Flush +0 PERIODIC 28484 3929.87 kB False False +1 PERIODIC 28485 3930.00 kB False False +0 PERIODIC 28489 3930.51 kB False False +1 PERIODIC 28490 3930.64 kB False False +0 PERIODIC 28491 3930.77 kB False False +0 PERIODIC 28491 3930.77 kB False False +0 PERIODIC 28496 3931.41 kB False False +1 PERIODIC 28496 3931.41 kB False False +0 PERIODIC 28497 3931.54 kB False False +1 PERIODIC 28497 3931.54 kB False False +1 PERIODIC 28499 3931.79 kB False False +. +. +. +53788 ALWAYS 30076 4136.27 kB False False +7463 ALWAYS 0 245.84 kB False False +``` + +### log_utxos.bt + +A `bpftrace` script to log information about the coins that are added, spent, or +uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and +`utxocache:uncache` tracepoints. + +```bash +$ bpftrace contrib/tracing/log_utxos.bt +``` + +It should produce an output similar to the following. + +```bash +Attaching 4 probes... +OP Outpoint Value Height Coinbase +Added 6ba9ad857e1ef2eb2a2c94f06813c414c7ab273e3d6bd7ad64e000315a887e7c:1 10000 2094512 No +Spent fa7dc4db56637a151f6649d8f26732956d1c5424c82aae400a83d02b2cc2c87b:0 182264897 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:0 10000 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:1 182254756 2094512 No +Added a0c7f4ec9cccef2d89672a624a4e6c8237a17572efdd4679eea9e9ee70d2db04:0 10072679 2094513 Yes +Spent 25e0df5cc1aeb1b78e6056bf403e5e8b7e41f138060ca0a50a50134df0549a5e:2 540 2094508 No +Spent 42f383c04e09c26a2378272ec33aa0c1bf4883ca5ab739e8b7e06be5a5787d61:1 3848399 2007724 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:0 3788399 2094513 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:2 540 2094513 No +Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 5591281046 2094511 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No +``` diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py new file mode 100755 index 0000000000..df27dc193a --- /dev/null +++ b/contrib/tracing/log_utxocache_flush.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +import sys +import ctypes +from bcc import BPF, USDT + +"""Example logging Bitcoin Core utxo set cache flushes utilizing + the utxocache:flush tracepoint.""" + +# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +# include +struct data_t +{ + u64 duration; + u32 mode; + u64 coins_count; + u64 coins_mem_usage; + bool is_flush_prune; + bool is_full_flush; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(flush); + +int trace_flush(struct pt_regs *ctx) { + struct data_t data = {}; + bpf_usdt_readarg(1, ctx, &data.duration); + bpf_usdt_readarg(2, ctx, &data.mode); + bpf_usdt_readarg(3, ctx, &data.coins_count); + bpf_usdt_readarg(4, ctx, &data.coins_mem_usage); + bpf_usdt_readarg(5, ctx, &data.is_flush_prune); + bpf_usdt_readarg(5, ctx, &data.is_full_flush); + flush.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +""" + +FLUSH_MODES = [ + 'NONE', + 'IF_NEEDED', + 'PERIODIC', + 'ALWAYS' +] + + +class Data(ctypes.Structure): + # define output data structure corresponding to struct data_t + _fields_ = [ + ("duration", ctypes.c_uint64), + ("mode", ctypes.c_uint32), + ("coins_count", ctypes.c_uint64), + ("coins_mem_usage", ctypes.c_uint64), + ("is_flush_prune", ctypes.c_bool), + ("is_full_flush", ctypes.c_bool) + ] + + +def print_event(event): + print("%-15d %-10s %-15d %-15s %-8s %-8s" % ( + event.duration, + FLUSH_MODES[event.mode], + event.coins_count, + "%.2f kB" % (event.coins_mem_usage/1000), + event.is_flush_prune, + event.is_full_flush + )) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="flush", fn_name="trace_flush") + b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + def handle_flush(_, data, size): + """ Coins Flush handler. + Called each time coin caches and indexes are flushed.""" + event = ctypes.cast(data, ctypes.POINTER(Data)).contents + print_event(event) + + b["flush"].open_perf_buffer(handle_flush) + print("Logging utxocache flushes. Ctrl-C to end...") + print("%-15s %-10s %-15s %-15s %-8s %-8s" % ("Duration (µs)", "Mode", + "Coins Count", "Memory Usage", + "Prune", "Full Flush")) + + while True: + try: + b.perf_buffer_poll() + except KeyboardInterrupt: + exit(0) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt new file mode 100755 index 0000000000..0d47f3d62b --- /dev/null +++ b/contrib/tracing/log_utxos.bt @@ -0,0 +1,86 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/log_utxos.bt + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'utxochache' tracepoints. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + NOTE: requires bpftrace v0.12.0 or above. +*/ + +BEGIN +{ + printf("%-7s %-71s %16s %7s %8s\n", + "OP", "Outpoint", "Value", "Height", "Coinbase"); +} + +/* + Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:add +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Added "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:spent +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Spent "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:uncache +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Uncache "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} diff --git a/depends/README.md b/depends/README.md index bd69768167..5c00807473 100644 --- a/depends/README.md +++ b/depends/README.md @@ -87,14 +87,6 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu -### Install the required dependencies: M1-based macOS - -To be able to build the `qt` package, ensure that Rosetta 2 is installed: - -``` -softwareupdate --install-rosetta -``` - ### Dependency Options The following can be set when running make: `make FOO=bar` diff --git a/depends/config.site.in b/depends/config.site.in index 5cf107f19b..87d0011b1c 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -72,7 +72,6 @@ fi if test "@host_os@" = darwin; then BREW=no - PORT=no fi PATH="${depends_prefix}/native/bin:${PATH}" diff --git a/depends/packages.md b/depends/packages.md index 7ed20ea129..4158b46d28 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -178,8 +178,8 @@ not sufficient to just say `libprimary`. For us, it's much easier to just link a static `libsecondary` into a shared `libprimary`. Especially because in our case, we are linking against a dummy `libprimary` anyway that we'll throw away. We don't care if the end-user has a -static or dynamic `libseconday`, that's not our concern. With a static -`libseconday`, when we need to link `libprimary` into our executable, there's no +static or dynamic `libsecondary`, that's not our concern. With a static +`libsecondary`, when we need to link `libprimary` into our executable, there's no dependency chain to worry about as `libprimary` has all the symbols. ## Build targets: diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index ab29742b55..5fe2b2bbb8 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -27,6 +27,7 @@ $(package)_cxxflags+=-std=c++17 $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC $(package)_cxxflags_x86_64_darwin=-fcf-protection=full +$(package)_cxxflags_mingw32=-fcf-protection=full endef define $(package)_preprocess_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index e16ed4f4ad..30e41f61b1 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -10,9 +10,10 @@ $(package)_linguist_tools = lrelease lupdate lconvert $(package)_patches = qt.pro qttools_src.pro $(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch $(package)_patches += support_new_android_ndks.patch fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches+= no_sdk_version_check.patch +$(package)_patches += dont_hardcode_x86_64.patch $(package)_patches+= fix_lib_paths.patch fix_android_pch.patch $(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches+= fix_montery_include.patch $(package)_patches += fix_qml_python.patch riscv_detection.patch fix_riscv_atomic.patch $(package)_qtdeclarative_file_name = qtdeclarative-$($(package)_suffix) @@ -140,8 +141,10 @@ $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif -# for macOS on Apple Silicon (ARM) see https://bugreports.qt.io/browse/QTBUG-85279 +ifneq ($(build_arch),$(host_arch)) $(package)_config_opts_aarch64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=arm64 +$(package)_config_opts_x86_64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=x86_64 +endif $(package)_config_opts_linux = -qt-xcb $(package)_config_opts_linux += -no-xcb-xlib @@ -254,10 +257,11 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qml_python.patch && \ patch -p1 -i $($(package)_patch_dir)/riscv_detection.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_riscv_atomic.patch && \ @@ -277,6 +281,7 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ + export QT_MAC_SDK_NO_VERSION_CHECK=1 && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch new file mode 100644 index 0000000000..0e1ca6acda --- /dev/null +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -0,0 +1,123 @@ +macOS: Don't hard-code x86_64 as the architecture when using qmake + +Upstream commit: + - Qt 6.1: 9082cc8e8d5a6441dabe5e7a95bc0cd9085b95fe + +For other Qt branches see +https://codereview.qt-project.org/q/I70db7e4c27f0d3da5d0af33cb491d72c312d3fa8 + + +--- old/qtbase/configure.json ++++ new/qtbase/configure.json +@@ -208,11 +208,18 @@ + + "testTypeDependencies": { + "linkerSupportsFlag": [ "use_gold_linker" ], +- "verifySpec": [ "shared", "use_gold_linker", "compiler-flags", "qmakeargs", "commit" ], ++ "verifySpec": [ ++ "shared", ++ "use_gold_linker", ++ "compiler-flags", "qmakeargs", ++ "simulator_and_device", ++ "thread", ++ "commit" ], + "compile": [ "verifyspec" ], + "detectPkgConfig": [ "cross_compile", "machineTuple" ], + "library": [ "pkg-config", "compiler-flags" ], +- "getPkgConfigVariable": [ "pkg-config" ] ++ "getPkgConfigVariable": [ "pkg-config" ], ++ "architecture" : [ "verifyspec" ] + }, + + "testTypeAliases": { +@@ -653,7 +660,7 @@ + }, + "architecture": { + "label": "Architecture", +- "output": [ "architecture" ] ++ "output": [ "architecture", "commitConfig" ] + }, + "pkg-config": { + "label": "Using pkg-config", +diff --git a/configure.pri b/configure.pri +index 33c90a8c2f..71767e29d6 100644 + +--- old/qtbase/configure.pri ++++ new/qtbase/configure.pri +@@ -642,6 +642,13 @@ defineTest(qtConfOutput_commitOptions) { + write_file($$QT_BUILD_TREE/mkspecs/qdevice.pri, $${currentConfig}.output.devicePro)|error() + } + ++# Output is written after configuring each Qt module, ++# but some tests within a module might depend on the ++# configuration output of previous tests. ++defineTest(qtConfOutput_commitConfig) { ++ qtConfProcessOutput() ++} ++ + # type (empty or 'host'), option name, default value + defineTest(processQtPath) { + out_var = config.rel_input.$${2} +diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf +index 7d4a406134..de96c12fc9 100644 + +--- old/qtbase/mkspecs/common/macx.conf ++++ new/qtbase/mkspecs/common/macx.conf +@@ -6,7 +6,6 @@ QMAKE_PLATFORM += macos osx macx + QMAKE_MAC_SDK = macosx + + QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.12 +-QMAKE_APPLE_DEVICE_ARCHS = x86_64 + + QT_MAC_SDK_VERSION_MIN = 10.13 + QT_MAC_SDK_VERSION_MAX = 11.0 +diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/default_post.prf +index d052808c14..0a89effe87 100644 + +--- old/qtbase/mkspecs/features/mac/default_post.prf ++++ new/qtbase/mkspecs/features/mac/default_post.prf +@@ -89,6 +89,11 @@ app_extension_api_only { + QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION + } + ++# Non-universal builds do not set QMAKE_APPLE_DEVICE_ARCHS, ++# so we pick it up from what the arch test resolved instead. ++isEmpty(QMAKE_APPLE_DEVICE_ARCHS): \ ++ QMAKE_APPLE_DEVICE_ARCHS = $$QT_ARCH ++ + macx-xcode { + qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO + !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ +@@ -144,9 +149,6 @@ macx-xcode { + simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS + VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS + +- isEmpty(VALID_ARCHS): \ +- error("QMAKE_APPLE_DEVICE_ARCHS or QMAKE_APPLE_SIMULATOR_ARCHS must contain at least one architecture") +- + single_arch: VALID_ARCHS = $$first(VALID_ARCHS) + + ACTIVE_ARCHS = $(filter $(EXPORT_VALID_ARCHS), $(ARCHS)) +diff --git a/mkspecs/features/toolchain.prf b/mkspecs/features/toolchain.prf +index 5003679bd0..c7c080cb07 100644 + +--- old/qtbase/mkspecs/features/toolchain.prf ++++ new/qtbase/mkspecs/features/toolchain.prf +@@ -182,9 +182,14 @@ isEmpty($${target_prefix}.INCDIRS) { + # UIKit simulator platforms will see the device SDK's sysroot in + # QMAKE_DEFAULT_*DIRS, because they're handled in a single build pass. + darwin { +- # Clang doesn't pick up the architecture from the sysroot, and will +- # default to the host architecture, so we need to manually set it. +- cxx_flags += -arch $$QMAKE_APPLE_DEVICE_ARCHS ++ uikit { ++ # Clang doesn't automatically pick up the architecture, just because ++ # we're passing the iOS sysroot below, and we will end up building the ++ # test for the host architecture, resulting in linker errors when ++ # linking against the iOS libraries. We work around this by passing ++ # the architecture explicitly. ++ cxx_flags += -arch $$first(QMAKE_APPLE_DEVICE_ARCHS) ++ } + + uikit:macx-xcode: \ + cxx_flags += -isysroot $$sdk_path_device.value diff --git a/depends/patches/qt/fix_montery_include.patch b/depends/patches/qt/fix_montery_include.patch new file mode 100644 index 0000000000..38b700addf --- /dev/null +++ b/depends/patches/qt/fix_montery_include.patch @@ -0,0 +1,21 @@ +From dece6f5840463ae2ddf927d65eb1b3680e34a547 +From: Øystein Heskestad +Date: Wed, 27 Oct 2021 13:07:46 +0200 +Subject: [PATCH] Add missing macOS header file that was indirectly included before + +See: https://bugreports.qt.io/browse/QTBUG-97855 + +Upstream Commits: + - Qt 6.2: c884bf138a21dd7320e35cef34d24e22e74d7ce0 + +diff --git a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +index e070ba97..07c75b04 100644 +--- a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h ++++ b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +@@ -40,6 +40,7 @@ + #ifndef QIOSURFACEGRAPHICSBUFFER_H + #define QIOSURFACEGRAPHICSBUFFER_H + ++#include + #include + #include diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 190ab7a160..e4bfaa1463 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -13,7 +13,6 @@ QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony -QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS diff --git a/depends/patches/qt/no_sdk_version_check.patch b/depends/patches/qt/no_sdk_version_check.patch deleted file mode 100644 index b16635b572..0000000000 --- a/depends/patches/qt/no_sdk_version_check.patch +++ /dev/null @@ -1,20 +0,0 @@ -commit f5eb142cd04be2bc4ca610ed3b5b7e8ce3520ee3 -Author: fanquake -Date: Tue Jan 5 16:08:49 2021 +0800 - - Don't invoke macOS SDK version checking - - This tries to use xcrun which is not available when cross-compiling. - -diff --git a/qtbase/mkspecs/features/mac/default_post.prf b/qtbase/mkspecs/features/mac/default_post.prf -index 92a9112bca6..447e186eb26 100644 ---- a/qtbase/mkspecs/features/mac/default_post.prf -+++ b/qtbase/mkspecs/features/mac/default_post.prf -@@ -8,7 +8,6 @@ contains(TEMPLATE, .*app) { - !macx-xcode:if(isEmpty(BUILDS)|build_pass) { - # Detect changes to the platform SDK - QMAKE_EXTRA_VARIABLES += QMAKE_MAC_SDK QMAKE_MAC_SDK_VERSION QMAKE_XCODE_DEVELOPER_PATH -- QMAKE_EXTRA_INCLUDES += $$shell_quote($$PWD/sdk.mk) - } - - # Detect incompatible SDK versions diff --git a/doc/dependencies.md b/doc/dependencies.md index 0c1fd6ba98..6ccb53f6b3 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -12,7 +12,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | -| glibc | | [2.17](https://www.gnu.org/software/libc/) | | | | | +| glibc | | [2.18](https://www.gnu.org/software/libc/) | | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 7ff1d36442..1888897856 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1254,6 +1254,12 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: User-facing consistency. +- Use `fs::path::u8string()` and `fs::u8path()` functions when converting path + to JSON strings, not `fs::PathToString` and `fs::PathFromString` + + - *Rationale*: JSON strings are Unicode strings, not byte strings, and + RFC8259 requires JSON to be encoded as UTF-8. + Internal interface guidelines ----------------------------- diff --git a/doc/release-notes-16807.md b/doc/release-notes-16807.md new file mode 100644 index 0000000000..5027550a99 --- /dev/null +++ b/doc/release-notes-16807.md @@ -0,0 +1,6 @@ +Updated RPCs +------------ + +- The `validateaddress` RPC now optionally returns an `error_locations` array, with the indices of +invalid characters in the address. For example, this will return the locations of up to two Bech32 +errors. \ No newline at end of file diff --git a/doc/release-notes.md b/doc/release-notes.md index b460cd3eb2..4483dee1dd 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -127,6 +127,10 @@ Updated settings mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` and `-nopersistmempool` is unaffected. (#23061) +- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. + E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. + Default is `M` if no suffix provided. (#23249) + Tools and Utilities ------------------- diff --git a/doc/tracing.md b/doc/tracing.md index 57104c43a0..5b9ba09c2f 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -108,6 +108,55 @@ Arguments passed: 5. SigOps in the Block (excluding coinbase SigOps) `uint64` 6. Time it took to connect the Block in microseconds (µs) as `uint64` +### Context `utxocache` + +#### Tracepoint `utxocache:flush` + +Is called *after* the caches and indexes are flushed depending on the mode +`CChainState::FlushStateToDisk` is called with. + +Arguments passed: +1. Duration in microseconds as `int64` +2. Flush state mode as `uint32`. It's an enumerator class with values `0` + (`NONE`), `1` (`IF_NEEDED`), `2` (`PERIODIC`), `3` (`ALWAYS`) +3. Number of coins flushed as `uint64` +4. Memory usage in bytes as `uint64` +5. If the flush was pruned as `bool` +6. If it was full flush as `bool` + +#### Tracepoint `utxocache:add` + +It is called when a new coin is added to the UTXO cache. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was added to the UTXO-set as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:spent` + +It is called when a coin is spent from the UTXO cache. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was spent, as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:uncache` + +It is called when the UTXO with the given outpoint is removed from the cache. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was uncached, as `uint32` +4. Value of the coin as `int64` +. If the coin is a coinbase as `bool` + ## Adding tracepoints to Bitcoin Core To add a new tracepoint, `#include ` in the compilation unit where diff --git a/src/Makefile.am b/src/Makefile.am index 25cd5c03d9..6846eb9693 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,8 +8,8 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) @@ -166,7 +166,6 @@ BITCOIN_CORE_H = \ mapport.h \ memusage.h \ merkleblock.h \ - miner.h \ net.h \ net_permissions.h \ net_processing.h \ @@ -178,6 +177,7 @@ BITCOIN_CORE_H = \ node/coin.h \ node/coinstats.h \ node/context.h \ + node/miner.h \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ @@ -246,6 +246,7 @@ BITCOIN_CORE_H = \ util/macros.h \ util/message.h \ util/moneystr.h \ + util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ util/serfloat.h \ @@ -334,7 +335,6 @@ libbitcoin_server_a_SOURCES = \ index/txindex.cpp \ init.cpp \ mapport.cpp \ - miner.cpp \ net.cpp \ net_processing.cpp \ node/blockstorage.cpp \ @@ -342,6 +342,7 @@ libbitcoin_server_a_SOURCES = \ node/coinstats.cpp \ node/context.cpp \ node/interfaces.cpp \ + node/miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ @@ -408,6 +409,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/interfaces.cpp \ wallet/load.cpp \ wallet/receive.cpp \ + wallet/rpc/signmessage.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/scriptpubkeyman.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 88a1ef798e..06d195aaaf 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -51,6 +51,7 @@ FUZZ_SUITE_LD_COMMON = \ $(BOOST_LIBS) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ + $(MINISKETCH_LIBS) \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) @@ -160,6 +161,7 @@ BITCOIN_TESTS += \ wallet/test/wallet_tests.cpp \ wallet/test/walletdb_tests.cpp \ wallet/test/wallet_crypto_tests.cpp \ + wallet/test/wallet_transaction_tests.cpp \ wallet/test/coinselector_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ @@ -259,6 +261,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ diff --git a/src/banman.h b/src/banman.h index f495dab49d..6cb6304744 100644 --- a/src/banman.h +++ b/src/banman.h @@ -95,4 +95,4 @@ class BanMan CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; -#endif +#endif // BITCOIN_BANMAN_H diff --git a/src/bech32.cpp b/src/bech32.cpp index 9da2488ef2..ea44480a6c 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -30,6 +31,183 @@ const int8_t CHARSET_REV[128] = { 1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1 }; +// We work with the finite field GF(1024) defined as a degree 2 extension of the base field GF(32) +// The defining polynomial of the extension is x^2 + 9x + 23 +// Let (e) be a primitive element of GF(1024), that is, a generator of the field. +// Every non-zero element of the field can then be represented as (e)^k for some power k. +// The array GF1024_EXP contains all these powers of (e) - GF1024_EXP[k] = (e)^k in GF(1024). +// Conversely, GF1024_LOG contains the discrete logarithms of these powers, so +// GF1024_LOG[GF1024_EXP[k]] == k +// Each element v of GF(1024) is encoded as a 10 bit integer in the following way: +// v = v1 || v0 where v0, v1 are 5-bit integers (elements of GF(32)). +// +// The element (e) is encoded as 9 || 15. Given (v), we compute (e)*(v) by multiplying in the following way: +// v0' = 27*v1 + 15*v0 +// v1' = 6*v1 + 9*v0 +// e*v = v1' || v0' +// +// The following sage code can be used to reproduce both _EXP and _LOG arrays +// GF1024_LOG = [-1] + [0] * 1023 +// GF1024_EXP = [1] * 1024 +// v = 1 +// for i in range(1, 1023): +// v0 = v & 31 +// v1 = v >> 5 +// v0n = F.fetch_int(27)*F.fetch_int(v1) + F.fetch_int(15)*F.fetch_int(v0) +// v1n = F.fetch_int(6)*F.fetch_int(v1) + F.fetch_int(9)*F.fetch_int(v0) +// v = v1n.integer_representation() << 5 | v0n.integer_representation() +// GF1024_EXP[i] = v +// GF1024_LOG[v] = i + +const int16_t GF1024_EXP[] = { + 1, 303, 635, 446, 997, 640, 121, 142, 959, 420, 350, 438, 166, 39, 543, + 335, 831, 691, 117, 632, 719, 97, 107, 374, 558, 797, 54, 150, 858, 877, + 724, 1013, 294, 23, 354, 61, 164, 633, 992, 538, 469, 659, 174, 868, 184, + 809, 766, 563, 866, 851, 257, 520, 45, 770, 535, 524, 408, 213, 436, 760, + 472, 330, 933, 799, 616, 361, 15, 391, 756, 814, 58, 608, 554, 680, 993, + 821, 942, 813, 843, 484, 193, 935, 321, 919, 572, 741, 423, 559, 562, + 589, 296, 191, 493, 685, 891, 665, 435, 60, 395, 2, 606, 511, 853, 746, + 32, 219, 284, 631, 840, 661, 837, 332, 78, 311, 670, 887, 111, 195, 505, + 190, 194, 214, 709, 380, 819, 69, 261, 957, 1018, 161, 739, 588, 7, 708, + 83, 328, 507, 736, 317, 899, 47, 348, 1000, 345, 882, 245, 367, 996, 943, + 514, 304, 90, 804, 295, 312, 793, 387, 833, 249, 921, 660, 618, 823, 496, + 722, 30, 782, 225, 892, 93, 480, 372, 112, 738, 867, 636, 890, 950, 968, + 386, 622, 642, 551, 369, 234, 846, 382, 365, 442, 592, 343, 986, 122, + 1023, 59, 847, 81, 790, 4, 437, 983, 931, 244, 64, 415, 529, 487, 944, + 35, 938, 664, 156, 583, 53, 999, 222, 390, 987, 341, 388, 389, 170, 721, + 879, 138, 522, 627, 765, 322, 230, 440, 14, 168, 143, 656, 991, 224, 595, + 550, 94, 657, 752, 667, 1005, 451, 734, 744, 638, 292, 585, 157, 872, + 590, 601, 827, 774, 930, 475, 571, 33, 500, 871, 969, 173, 21, 828, 450, + 1009, 147, 960, 705, 201, 228, 998, 497, 1021, 613, 688, 772, 508, 36, + 366, 715, 468, 956, 725, 730, 861, 425, 647, 701, 221, 759, 95, 958, 139, + 805, 8, 835, 679, 614, 449, 128, 791, 299, 974, 617, 70, 628, 57, 273, + 430, 67, 750, 405, 780, 703, 643, 776, 778, 340, 171, 1022, 276, 308, + 495, 243, 644, 460, 857, 28, 336, 286, 41, 695, 448, 431, 364, 149, 43, + 233, 63, 762, 902, 181, 240, 501, 584, 434, 275, 1008, 444, 443, 895, + 812, 612, 927, 383, 66, 961, 1006, 690, 346, 3, 881, 900, 747, 271, 672, + 162, 402, 456, 748, 971, 755, 490, 105, 808, 977, 72, 732, 182, 897, 625, + 163, 189, 947, 850, 46, 115, 403, 231, 151, 629, 278, 874, 16, 934, 110, + 492, 898, 256, 807, 598, 700, 498, 140, 481, 91, 523, 860, 134, 252, 771, + 824, 119, 38, 816, 820, 641, 342, 757, 513, 577, 990, 463, 40, 920, 955, + 17, 649, 533, 82, 103, 896, 862, 728, 259, 86, 466, 87, 253, 556, 323, + 457, 963, 432, 845, 527, 745, 849, 863, 1015, 888, 488, 567, 727, 132, + 674, 764, 109, 669, 6, 1003, 552, 246, 542, 96, 324, 781, 912, 248, 694, + 239, 980, 210, 880, 683, 144, 177, 325, 546, 491, 326, 339, 623, 941, 92, + 207, 783, 462, 263, 483, 517, 1012, 9, 620, 220, 984, 548, 512, 878, 421, + 113, 973, 280, 962, 159, 310, 945, 268, 465, 806, 889, 199, 76, 873, 865, + 34, 645, 227, 290, 418, 693, 926, 80, 569, 639, 11, 50, 291, 141, 206, + 544, 949, 185, 518, 133, 909, 135, 467, 376, 646, 914, 678, 841, 954, + 318, 242, 939, 951, 743, 1017, 976, 359, 167, 264, 100, 241, 218, 51, 12, + 758, 368, 453, 309, 192, 648, 826, 553, 473, 101, 478, 673, 397, 1001, + 118, 265, 331, 650, 356, 982, 652, 655, 510, 634, 145, 414, 830, 924, + 526, 966, 298, 737, 18, 504, 401, 697, 360, 288, 1020, 842, 203, 698, + 537, 676, 279, 581, 619, 536, 907, 876, 1019, 398, 152, 1010, 994, 68, + 42, 454, 580, 836, 99, 565, 137, 379, 503, 22, 77, 582, 282, 412, 352, + 611, 347, 300, 266, 570, 270, 911, 729, 44, 557, 108, 946, 637, 597, 461, + 630, 615, 238, 763, 681, 718, 334, 528, 200, 459, 413, 79, 24, 229, 713, + 906, 579, 384, 48, 893, 370, 923, 202, 917, 98, 794, 754, 197, 530, 662, + 52, 712, 677, 56, 62, 981, 509, 267, 789, 885, 561, 316, 684, 596, 226, + 13, 985, 779, 123, 720, 576, 753, 948, 406, 125, 315, 104, 519, 426, 502, + 313, 566, 1016, 767, 796, 281, 749, 740, 136, 84, 908, 424, 936, 198, + 355, 274, 735, 967, 5, 154, 428, 541, 785, 704, 486, 671, 600, 532, 381, + 540, 574, 187, 88, 378, 216, 621, 499, 419, 922, 485, 494, 476, 255, 114, + 188, 668, 297, 400, 918, 787, 158, 25, 458, 178, 564, 422, 768, 73, 1011, + 717, 575, 404, 547, 196, 829, 237, 394, 301, 37, 65, 176, 106, 89, 85, + 675, 979, 534, 803, 995, 363, 593, 120, 417, 452, 26, 699, 822, 223, 169, + 416, 235, 609, 773, 211, 607, 208, 302, 852, 965, 603, 357, 761, 247, + 817, 539, 250, 232, 272, 129, 568, 848, 624, 396, 710, 525, 183, 686, 10, + 285, 856, 307, 811, 160, 972, 55, 441, 289, 723, 305, 373, 351, 153, 733, + 409, 506, 975, 838, 573, 970, 988, 913, 471, 205, 337, 49, 594, 777, 549, + 815, 277, 27, 916, 333, 353, 844, 800, 146, 751, 186, 375, 769, 358, 392, + 883, 474, 788, 602, 74, 130, 329, 212, 155, 131, 102, 687, 293, 870, 742, + 726, 427, 217, 834, 904, 29, 127, 869, 407, 338, 832, 470, 482, 810, 399, + 439, 393, 604, 929, 682, 447, 714, 251, 455, 875, 319, 477, 464, 521, + 258, 377, 937, 489, 792, 172, 314, 327, 124, 20, 531, 953, 591, 886, 320, + 696, 71, 859, 578, 175, 587, 707, 663, 283, 179, 795, 989, 702, 940, 371, + 692, 689, 555, 903, 410, 651, 75, 429, 818, 362, 894, 515, 31, 545, 666, + 706, 952, 864, 269, 254, 349, 711, 802, 716, 784, 1007, 925, 801, 445, + 148, 260, 658, 385, 287, 262, 204, 126, 586, 1004, 236, 165, 854, 411, + 932, 560, 19, 215, 1002, 775, 653, 928, 901, 964, 884, 798, 839, 786, + 433, 610, 116, 855, 180, 479, 910, 1014, 599, 915, 905, 306, 516, 731, + 626, 978, 825, 344, 605, 654, 209 +}; +// As above, GF1024_EXP contains all elements of GF(1024) except 0 +static_assert(std::size(GF1024_EXP) == 1023, "GF1024_EXP length should be 1023"); + +const int16_t GF1024_LOG[] = { + -1, 0, 99, 363, 198, 726, 462, 132, 297, 495, 825, 528, 561, 693, 231, + 66, 396, 429, 594, 990, 924, 264, 627, 33, 660, 759, 792, 858, 330, 891, + 165, 957, 104, 259, 518, 208, 280, 776, 416, 13, 426, 333, 618, 339, 641, + 52, 388, 140, 666, 852, 529, 560, 678, 213, 26, 832, 681, 309, 70, 194, + 97, 35, 682, 341, 203, 777, 358, 312, 617, 125, 307, 931, 379, 765, 875, + 951, 515, 628, 112, 659, 525, 196, 432, 134, 717, 781, 438, 440, 740, + 780, 151, 408, 487, 169, 239, 293, 467, 21, 672, 622, 557, 571, 881, 433, + 704, 376, 779, 22, 643, 460, 398, 116, 172, 503, 751, 389, 1004, 18, 576, + 415, 789, 6, 192, 696, 923, 702, 981, 892, 302, 816, 876, 880, 457, 537, + 411, 539, 716, 624, 224, 295, 406, 531, 7, 233, 478, 586, 864, 268, 974, + 338, 27, 392, 614, 839, 727, 879, 211, 250, 758, 507, 830, 129, 369, 384, + 36, 985, 12, 555, 232, 796, 221, 321, 920, 263, 42, 934, 778, 479, 761, + 939, 1006, 344, 381, 823, 44, 535, 866, 739, 752, 385, 119, 91, 566, 80, + 120, 117, 771, 675, 721, 514, 656, 271, 670, 602, 980, 850, 532, 488, + 803, 1022, 475, 801, 878, 57, 121, 991, 742, 888, 559, 105, 497, 291, + 215, 795, 236, 167, 692, 520, 272, 661, 229, 391, 814, 340, 184, 798, + 984, 773, 650, 473, 345, 558, 548, 326, 202, 145, 465, 810, 471, 158, + 813, 908, 412, 441, 964, 750, 401, 50, 915, 437, 975, 126, 979, 491, 556, + 577, 636, 685, 510, 963, 638, 367, 815, 310, 723, 349, 323, 857, 394, + 606, 505, 713, 630, 938, 106, 826, 332, 978, 599, 834, 521, 530, 248, + 883, 32, 153, 90, 754, 592, 304, 635, 775, 804, 1, 150, 836, 1013, 828, + 324, 565, 508, 113, 154, 708, 921, 703, 689, 138, 547, 911, 929, 82, 228, + 443, 468, 480, 483, 922, 135, 877, 61, 578, 111, 860, 654, 15, 331, 851, + 895, 484, 320, 218, 420, 190, 1019, 143, 362, 634, 141, 965, 10, 838, + 632, 861, 34, 722, 580, 808, 869, 554, 598, 65, 954, 787, 337, 187, 281, + 146, 563, 183, 668, 944, 171, 837, 23, 867, 541, 916, 741, 625, 123, 736, + 186, 357, 665, 977, 179, 156, 219, 220, 216, 67, 870, 902, 774, 98, 820, + 574, 613, 900, 755, 596, 370, 390, 769, 314, 701, 894, 56, 841, 949, 987, + 631, 658, 587, 204, 797, 790, 522, 745, 9, 502, 763, 86, 719, 288, 706, + 887, 728, 952, 311, 336, 446, 1002, 348, 96, 58, 199, 11, 901, 230, 833, + 188, 352, 351, 973, 3, 906, 335, 301, 266, 244, 791, 564, 619, 909, 371, + 444, 760, 657, 328, 647, 490, 425, 913, 511, 439, 540, 283, 40, 897, 849, + 60, 570, 872, 257, 749, 912, 572, 1007, 170, 407, 898, 492, 79, 747, 732, + 206, 454, 918, 375, 482, 399, 92, 748, 325, 163, 274, 405, 744, 260, 346, + 707, 626, 595, 118, 842, 136, 279, 684, 584, 101, 500, 422, 149, 956, + 1014, 493, 536, 705, 51, 914, 225, 409, 55, 822, 590, 448, 655, 205, 676, + 925, 735, 431, 784, 54, 609, 604, 39, 812, 737, 729, 466, 14, 533, 958, + 481, 770, 499, 855, 238, 182, 464, 569, 72, 947, 442, 642, 24, 87, 989, + 688, 88, 47, 762, 623, 709, 455, 817, 526, 637, 258, 84, 845, 738, 768, + 698, 423, 933, 664, 620, 607, 629, 212, 347, 249, 982, 935, 131, 89, 252, + 927, 189, 788, 853, 237, 691, 646, 403, 1010, 734, 253, 874, 807, 903, + 1020, 100, 802, 71, 799, 1003, 633, 355, 276, 300, 649, 64, 306, 161, + 608, 496, 743, 180, 485, 819, 383, 1016, 226, 308, 393, 648, 107, 19, 37, + 585, 2, 175, 645, 247, 527, 5, 419, 181, 317, 327, 519, 542, 289, 567, + 430, 579, 950, 582, 994, 1021, 583, 234, 240, 976, 41, 160, 109, 677, + 937, 210, 95, 959, 242, 753, 461, 114, 733, 368, 573, 458, 782, 605, 680, + 544, 299, 73, 652, 905, 477, 690, 93, 824, 882, 277, 946, 361, 17, 945, + 523, 472, 334, 930, 597, 603, 793, 404, 290, 942, 316, 731, 270, 960, + 936, 133, 122, 821, 966, 679, 662, 907, 282, 968, 767, 653, 20, 697, 222, + 164, 835, 30, 285, 886, 456, 436, 640, 286, 1015, 380, 840, 245, 724, + 137, 593, 173, 130, 715, 85, 885, 551, 246, 449, 103, 366, 372, 714, 313, + 865, 241, 699, 674, 374, 68, 421, 562, 292, 59, 809, 342, 651, 459, 227, + 46, 711, 764, 868, 53, 413, 278, 800, 255, 993, 318, 854, 319, 695, 315, + 469, 166, 489, 969, 730, 1001, 757, 873, 686, 197, 303, 919, 155, 673, + 940, 712, 25, 999, 63, 863, 972, 967, 785, 152, 296, 512, 402, 377, 45, + 899, 829, 354, 77, 69, 856, 417, 811, 953, 124, 418, 75, 794, 162, 414, + 1018, 568, 254, 265, 772, 588, 16, 896, 157, 889, 298, 621, 110, 844, + 1000, 108, 545, 601, 78, 862, 447, 185, 195, 818, 450, 387, 49, 805, 102, + 986, 1005, 827, 329, 28, 932, 410, 287, 435, 451, 962, 517, 48, 174, 43, + 893, 884, 261, 251, 516, 395, 910, 611, 29, 501, 223, 476, 364, 144, 871, + 998, 687, 928, 115, 453, 513, 176, 94, 168, 667, 955, 353, 434, 382, 400, + 139, 365, 996, 343, 948, 890, 1012, 663, 610, 718, 538, 1008, 639, 470, + 848, 543, 1011, 859, 671, 756, 83, 427, 159, 746, 669, 589, 971, 524, + 356, 995, 904, 256, 201, 988, 62, 397, 81, 720, 917, 209, 549, 943, 486, + 76, 148, 207, 509, 644, 386, 700, 534, 177, 550, 961, 926, 546, 428, 284, + 127, 294, 8, 269, 359, 506, 445, 997, 806, 591, 725, 178, 262, 846, 373, + 831, 504, 305, 843, 553, 378, 1017, 783, 474, 683, 581, 200, 498, 694, + 191, 217, 847, 941, 424, 235, 38, 74, 616, 786, 147, 4, 273, 214, 142, + 575, 992, 463, 983, 243, 360, 970, 350, 267, 615, 766, 494, 31, 1009, + 452, 710, 552, 128, 612, 600, 275, 322, 193 +}; +static_assert(std::size(GF1024_LOG) == 1024, "GF1024_EXP length should be 1024"); + /* Determine the final constant to use for the specified encoding. */ uint32_t EncodingConstant(Encoding encoding) { assert(encoding == Encoding::BECH32 || encoding == Encoding::BECH32M); @@ -127,12 +305,116 @@ uint32_t PolyMod(const data& v) return c; } +/** Syndrome computes the values s_j = R(e^j) for j in [997, 998, 999]. As described above, the + * generator polynomial G is the LCM of the minimal polynomials of (e)^997, (e)^998, and (e)^999. + * + * Consider a codeword with errors, of the form R(x) = C(x) + E(x). The residue is the bit-packed + * result of computing R(x) mod G(X), where G is the generator of the code. Because C(x) is a valid + * codeword, it is a multiple of G(X), so the residue is in fact just E(x) mod G(x). Note that all + * of the (e)^j are roots of G(x) by definition, so R((e)^j) = E((e)^j). + * + * Syndrome returns the three values packed into a 30-bit integer, where each 10 bits is one value. + */ +uint32_t Syndrome(const uint32_t residue) { + // Let R(x) = r1*x^5 + r2*x^4 + r3*x^3 + r4*x^2 + r5*x + r6 + // low is the first 5 bits, corresponding to the r6 in the residue + // (the constant term of the polynomial). + + uint32_t low = residue & 0x1f; + + // Recall that XOR corresponds to addition in a characteristic 2 field. + // + // To compute R((e)^j), we are really computing: + // r1*(e)^(j*5) + r2*(e)^(j*4) + r3*(e)^(j*3) + r4*(e)^(j*2) + r5*(e)^j + r6 + // Now note that all of the (e)^(j*i) for i in [5..0] are constants and can be precomputed + // for efficiency. But even more than that, we can consider each coefficient as a bit-string. + // For example, take r5 = (b_5, b_4, b_3, b_2, b_1) written out as 5 bits. Then: + // r5*(e)^j = b_1*(e)^j + b_2*(2*(e)^j) + b_3*(4*(e)^j) + b_4*(8*(e)^j) + b_5*(16*(e)^j) + // where all the (2^i*(e)^j) are constants and can be precomputed. Then we just add each + // of these corresponding constants to our final value based on the bit values b_i. + // This is exactly what is done below. Note that all three values of s_j for j in (997, 998, + // 999) are computed simultaneously. + // + // We begin by setting s_j = low = r6 for all three values of j, because these are unconditional. + // Then for each following bit, we add the corresponding precomputed constant if the bit is 1. + // For example, 0x31edd3c4 is 1100011110 1101110100 1111000100 when unpacked in groups of 10 + // bits, corresponding exactly to a^999 || a^998 || a^997 (matching the corresponding values in + // GF1024_EXP above). + // + // The following sage code reproduces these constants: + // for k in range(1, 6): + // for b in [1,2,4,8,16]: + // c0 = GF1024_EXP[(997*k + GF1024_LOG[b]) % 1023] + // c1 = GF1024_EXP[(998*k + GF1024_LOG[b]) % 1023] + // c2 = GF1024_EXP[(999*k + GF1024_LOG[b]) % 1023] + // c = c2 << 20 | c1 << 10 | c0 + // print("0x%x" % c) + + return low ^ (low << 10) ^ (low << 20) ^ + ((residue >> 5) & 1 ? 0x31edd3c4 : 0) ^ + ((residue >> 6) & 1 ? 0x335f86a8 : 0) ^ + ((residue >> 7) & 1 ? 0x363b8870 : 0) ^ + ((residue >> 8) & 1 ? 0x3e6390c9 : 0) ^ + ((residue >> 9) & 1 ? 0x2ec72192 : 0) ^ + ((residue >> 10) & 1 ? 0x1046f79d : 0) ^ + ((residue >> 11) & 1 ? 0x208d4e33 : 0) ^ + ((residue >> 12) & 1 ? 0x130ebd6f : 0) ^ + ((residue >> 13) & 1 ? 0x2499fade : 0) ^ + ((residue >> 14) & 1 ? 0x1b27d4b5 : 0) ^ + ((residue >> 15) & 1 ? 0x04be1eb4 : 0) ^ + ((residue >> 16) & 1 ? 0x0968b861 : 0) ^ + ((residue >> 17) & 1 ? 0x1055f0c2 : 0) ^ + ((residue >> 18) & 1 ? 0x20ab4584 : 0) ^ + ((residue >> 19) & 1 ? 0x1342af08 : 0) ^ + ((residue >> 20) & 1 ? 0x24f1f318 : 0) ^ + ((residue >> 21) & 1 ? 0x1be34739 : 0) ^ + ((residue >> 22) & 1 ? 0x35562f7b : 0) ^ + ((residue >> 23) & 1 ? 0x3a3c5bff : 0) ^ + ((residue >> 24) & 1 ? 0x266c96f7 : 0) ^ + ((residue >> 25) & 1 ? 0x25c78b65 : 0) ^ + ((residue >> 26) & 1 ? 0x1b1f13ea : 0) ^ + ((residue >> 27) & 1 ? 0x34baa2f4 : 0) ^ + ((residue >> 28) & 1 ? 0x3b61c0e1 : 0) ^ + ((residue >> 29) & 1 ? 0x265325c2 : 0); +} + /** Convert to lower case. */ inline unsigned char LowerCase(unsigned char c) { return (c >= 'A' && c <= 'Z') ? (c - 'A') + 'a' : c; } +void push_range(int from, int to, std::vector& vec) +{ + for (int i = from; i < to; i++) { + vec.push_back(i); + } +} + +/** Return index of first invalid character in a Bech32 string. */ +bool CheckCharacters(const std::string& str, std::vector& errors) { + bool lower = false, upper = false; + for (size_t i = 0; i < str.size(); ++i) { + unsigned char c = str[i]; + if (c >= 'a' && c <= 'z') { + if (upper) { + errors.push_back(i); + } else { + lower = true; + } + } else if (c >= 'A' && c <= 'Z') { + if (lower) { + errors.push_back(i); + } else { + upper = true; + } + } else if (c < 33 || c > 126) { + errors.push_back(i); + } + } + return errors.empty(); +} + /** Expand a HRP for use in checksum computation. */ data ExpandHRP(const std::string& hrp) { @@ -196,14 +478,8 @@ std::string Encode(Encoding encoding, const std::string& hrp, const data& values /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str) { - bool lower = false, upper = false; - for (size_t i = 0; i < str.size(); ++i) { - unsigned char c = str[i]; - if (c >= 'a' && c <= 'z') lower = true; - else if (c >= 'A' && c <= 'Z') upper = true; - else if (c < 33 || c > 126) return {}; - } - if (lower && upper) return {}; + std::vector errors; + if (!CheckCharacters(str, errors)) return {}; size_t pos = str.rfind('1'); if (str.size() > 90 || pos == str.npos || pos == 0 || pos + 7 > str.size()) { return {}; @@ -227,4 +503,163 @@ DecodeResult Decode(const std::string& str) { return {result, std::move(hrp), data(values.begin(), values.end() - 6)}; } +/** Find index of an incorrect character in a Bech32 string. */ +std::string LocateErrors(const std::string& str, std::vector& error_locations) { + if (str.size() > 90) { + push_range(90, str.size(), error_locations); + return "Bech32 string too long"; + } + if (!CheckCharacters(str, error_locations)){ + return "Invalid character or mixed case"; + } + size_t pos = str.rfind('1'); + if (pos == str.npos) { + return "Missing separator"; + } + if (pos == 0 || pos + 7 > str.size()) { + error_locations.push_back(pos); + return "Invalid separator position"; + } + std::string hrp; + for (size_t i = 0; i < pos; ++i) { + hrp += LowerCase(str[i]); + } + + size_t length = str.size() - 1 - pos; // length of data part + data values(length); + for (size_t i = pos + 1; i < str.size(); ++i) { + unsigned char c = str[i]; + int8_t rev = CHARSET_REV[c]; + if (rev == -1) { + error_locations.push_back(i); + return "Invalid Base 32 character"; + } + values[i - pos - 1] = rev; + } + + // We attempt error detection with both bech32 and bech32m, and choose the one with the fewest errors + // We can't simply use the segwit version, because that may be one of the errors + for (Encoding encoding : {Encoding::BECH32, Encoding::BECH32M}) { + std::vector possible_errors; + // Recall that (ExpandHRP(hrp) ++ values) is interpreted as a list of coefficients of a polynomial + // over GF(32). PolyMod computes the "remainder" of this polynomial modulo the generator G(x). + uint32_t residue = PolyMod(Cat(ExpandHRP(hrp), values)) ^ EncodingConstant(encoding); + + // All valid codewords should be multiples of G(x), so this remainder (after XORing with the encoding + // constant) should be 0 - hence 0 indicates there are no errors present. + if (residue != 0) { + // If errors are present, our polynomial must be of the form C(x) + E(x) where C is the valid + // codeword (a multiple of G(x)), and E encodes the errors. + uint32_t syn = Syndrome(residue); + + // Unpack the three 10-bit syndrome values + int s0 = syn & 0x3FF; + int s1 = (syn >> 10) & 0x3FF; + int s2 = syn >> 20; + + // Get the discrete logs of these values in GF1024 for more efficient computation + int l_s0 = GF1024_LOG[s0]; + int l_s1 = GF1024_LOG[s1]; + int l_s2 = GF1024_LOG[s2]; + + // First, suppose there is only a single error. Then E(x) = e1*x^p1 for some position p1 + // Then s0 = E((e)^997) = e1*(e)^(997*p1) and s1 = E((e)^998) = e1*(e)^(998*p1) + // Therefore s1/s0 = (e)^p1, and by the same logic, s2/s1 = (e)^p1 too. + // Hence, s1^2 == s0*s2, which is exactly the condition we check first: + if (l_s0 != -1 && l_s1 != -1 && l_s2 != -1 && (2 * l_s1 - l_s2 - l_s0 + 2046) % 1023 == 0) { + // Compute the error position p1 as l_s1 - l_s0 = p1 (mod 1023) + size_t p1 = (l_s1 - l_s0 + 1023) % 1023; // the +1023 ensures it is positive + // Now because s0 = e1*(e)^(997*p1), we get e1 = s0/((e)^(997*p1)). Remember that (e)^1023 = 1, + // so 1/((e)^997) = (e)^(1023-997). + int l_e1 = l_s0 + (1023 - 997) * p1; + // Finally, some sanity checks on the result: + // - The error position should be within the length of the data + // - e1 should be in GF(32), which implies that e1 = (e)^(33k) for some k (the 31 non-zero elements + // of GF(32) form an index 33 subgroup of the 1023 non-zero elements of GF(1024)). + if (p1 < length && !(l_e1 % 33)) { + // Polynomials run from highest power to lowest, so the index p1 is from the right. + // We don't return e1 because it is dangerous to suggest corrections to the user, + // the user should check the address themselves. + possible_errors.push_back(str.size() - p1 - 1); + } + // Otherwise, suppose there are two errors. Then E(x) = e1*x^p1 + e2*x^p2. + } else { + // For all possible first error positions p1 + for (size_t p1 = 0; p1 < length; ++p1) { + // We have guessed p1, and want to solve for p2. Recall that E(x) = e1*x^p1 + e2*x^p2, so + // s0 = E((e)^997) = e1*(e)^(997^p1) + e2*(e)^(997*p2), and similar for s1 and s2. + // + // Consider s2 + s1*(e)^p1 + // = 2e1*(e)^(999^p1) + e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // = e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // (Because we are working in characteristic 2.) + // = e2*(e)^(998*p2) ((e)^p2 + (e)^p1) + // + int s2_s1p1 = s2 ^ (s1 == 0 ? 0 : GF1024_EXP[(l_s1 + p1) % 1023]); + if (s2_s1p1 == 0) continue; + int l_s2_s1p1 = GF1024_LOG[s2_s1p1]; + + // Similarly, s1 + s0*(e)^p1 + // = e2*(e)^(997*p2) ((e)^p2 + (e)^p1) + int s1_s0p1 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP[(l_s0 + p1) % 1023]); + if (s1_s0p1 == 0) continue; + int l_s1_s0p1 = GF1024_LOG[s1_s0p1]; + + // So, putting these together, we can compute the second error position as + // (e)^p2 = (s2 + s1^p1)/(s1 + s0^p1) + // p2 = log((e)^p2) + size_t p2 = (l_s2_s1p1 - l_s1_s0p1 + 1023) % 1023; + + // Sanity checks that p2 is a valid position and not the same as p1 + if (p2 >= length || p1 == p2) continue; + + // Now we want to compute the error values e1 and e2. + // Similar to above, we compute s1 + s0*(e)^p2 + // = e1*(e)^(997*p1) ((e)^p1 + (e)^p2) + int s1_s0p2 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP[(l_s0 + p2) % 1023]); + if (s1_s0p2 == 0) continue; + int l_s1_s0p2 = GF1024_LOG[s1_s0p2]; + + // And compute (the log of) 1/((e)^p1 + (e)^p2)) + int inv_p1_p2 = 1023 - GF1024_LOG[GF1024_EXP[p1] ^ GF1024_EXP[p2]]; + + // Then (s1 + s0*(e)^p1) * (1/((e)^p1 + (e)^p2))) + // = e2*(e)^(997*p2) + // Then recover e2 by dividing by (e)^(997*p2) + int l_e2 = l_s1_s0p1 + inv_p1_p2 + (1023 - 997) * p2; + // Check that e2 is in GF(32) + if (l_e2 % 33) continue; + + // In the same way, (s1 + s0*(e)^p2) * (1/((e)^p1 + (e)^p2))) + // = e1*(e)^(997*p1) + // So recover e1 by dividing by (e)^(997*p1) + int l_e1 = l_s1_s0p2 + inv_p1_p2 + (1023 - 997) * p1; + // Check that e1 is in GF(32) + if (l_e1 % 33) continue; + + // Again, we do not return e1 or e2 for safety. + // Order the error positions from the left of the string and return them + if (p1 > p2) { + possible_errors.push_back(str.size() - p1 - 1); + possible_errors.push_back(str.size() - p2 - 1); + } else { + possible_errors.push_back(str.size() - p2 - 1); + possible_errors.push_back(str.size() - p1 - 1); + } + break; + } + } + } else { + // No errors + error_locations.clear(); + return ""; + } + + if (error_locations.empty() || (!possible_errors.empty() && possible_errors.size() < error_locations.size())) { + error_locations = std::move(possible_errors); + } + } + return "Invalid checksum"; +} + } // namespace bech32 diff --git a/src/bech32.h b/src/bech32.h index e9450ccc2b..7e92d5d23a 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -1,4 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -44,6 +45,9 @@ struct DecodeResult /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str); +/** Return the positions of errors in a Bech32 string. */ +std::string LocateErrors(const std::string& str, std::vector& error_locations); + } // namespace bech32 #endif // BITCOIN_BECH32_H diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index d5275b0b76..dae3a47cd7 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -45,7 +45,7 @@ static void CCoinsCaching(benchmark::Bench& bench) // Benchmark. const CTransaction tx_1(t1); bench.run([&] { - bool success = AreInputsStandard(tx_1, coins, false); + bool success{AreInputsStandard(tx_1, coins)}; assert(success); }); ECC_Stop(); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f6a8c56743..3c24fee60f 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -18,7 +18,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector(MakeTransactionRef(std::move(tx)))); + wtxs.push_back(std::make_unique(MakeTransactionRef(std::move(tx)), TxStateInactive{})); } // Simple benchmark for wallet coin selection. Note that it maybe be necessary diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index f1eeef8885..67c827d0d3 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -12,7 +12,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /* sigOpCost */ 4, lp)); + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } static void RpcMempool(benchmark::Bench& bench) diff --git a/src/chain.cpp b/src/chain.cpp index c09113a866..5d182e1af8 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -5,9 +5,6 @@ #include -/** - * CChain implementation - */ void CChain::SetTip(CBlockIndex *pindex) { if (pindex == nullptr) { vChain.clear(); diff --git a/src/checkqueue.h b/src/checkqueue.h index 7c20e2013c..760dfbddc1 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -167,16 +167,24 @@ class CCheckQueue //! Add a batch of checks to the queue void Add(std::vector& vChecks) { - LOCK(m_mutex); - for (T& check : vChecks) { - queue.push_back(T()); - check.swap(queue.back()); + if (vChecks.empty()) { + return; } - nTodo += vChecks.size(); - if (vChecks.size() == 1) + + { + LOCK(m_mutex); + for (T& check : vChecks) { + queue.emplace_back(); + check.swap(queue.back()); + } + nTodo += vChecks.size(); + } + + if (vChecks.size() == 1) { m_worker_cv.notify_one(); - else if (vChecks.size() > 1) + } else { m_worker_cv.notify_all(); + } } //! Stop all of the worker threads. diff --git a/src/coins.cpp b/src/coins.cpp index ce0b131de6..daead6055e 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } @@ -95,6 +96,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi it->second.coin = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); + TRACE5(utxocache, add, + outpoint.hash.data(), + (uint32_t)outpoint.n, + (uint32_t)coin.nHeight, + (int64_t)coin.out.nValue, + (bool)coin.IsCoinBase()); } void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { @@ -120,6 +127,12 @@ bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { CCoinsMap::iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) return false; cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + TRACE5(utxocache, spent, + outpoint.hash.data(), + (uint32_t)outpoint.n, + (uint32_t)it->second.coin.nHeight, + (int64_t)it->second.coin.out.nValue, + (bool)it->second.coin.IsCoinBase()); if (moveout) { *moveout = std::move(it->second.coin); } @@ -231,6 +244,12 @@ void CCoinsViewCache::Uncache(const COutPoint& hash) CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + TRACE5(utxocache, uncache, + hash.hash.data(), + (uint32_t)hash.n, + (uint32_t)it->second.coin.nHeight, + (int64_t)it->second.coin.out.nValue, + (bool)it->second.coin.IsCoinBase()); cacheCoins.erase(it); } } diff --git a/src/consensus/amount.h b/src/consensus/amount.h index 96566ea13f..59b8e3417a 100644 --- a/src/consensus/amount.h +++ b/src/consensus/amount.h @@ -26,4 +26,4 @@ static constexpr CAmount COIN = 100000000; static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_CONSENSUS_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 2fdc54464a..dbae2c45f2 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -136,6 +136,10 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo TryCreateDirectories(path); LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path)); } + // PathToString() return value is safe to pass to leveldb open function, + // because on POSIX leveldb passes the byte string directly to ::open(), and + // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW + // (see env_posix.cc and env_windows.cc). leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb); dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); diff --git a/src/fs.h b/src/fs.h index 4a0bf39e95..3cf4371fb4 100644 --- a/src/fs.h +++ b/src/fs.h @@ -94,31 +94,34 @@ static inline path operator+(path p1, path p2) /** * Convert path object to byte string. On POSIX, paths natively are byte - * strings so this is trivial. On Windows, paths natively are Unicode, so an - * encoding step is necessary. + * strings, so this is trivial. On Windows, paths natively are Unicode, so an + * encoding step is necessary. The inverse of \ref PathToString is \ref + * PathFromString. The strings returned and parsed by these functions can be + * used to call POSIX APIs, and for roundtrip conversion, logging, and + * debugging. * - * The inverse of \ref PathToString is \ref PathFromString. The strings - * returned and parsed by these functions can be used to call POSIX APIs, and - * for roundtrip conversion, logging, and debugging. But they are not - * guaranteed to be valid UTF-8, and are generally meant to be used internally, - * not externally. When communicating with external programs and libraries that - * require UTF-8, fs::path::u8string() and fs::u8path() methods can be used. - * For other applications, if support for non UTF-8 paths is required, or if - * higher-level JSON or XML or URI or C-style escapes are preferred, it may be - * also be appropriate to use different path encoding functions. - * - * Implementation note: On Windows, the std::filesystem::path(string) - * constructor and std::filesystem::path::string() method are not safe to use - * here, because these methods encode the path using C++'s narrow multibyte - * encoding, which on Windows corresponds to the current "code page", which is - * unpredictable and typically not able to represent all valid paths. So - * std::filesystem::path::u8string() and std::filesystem::u8path() functions - * are used instead on Windows. On POSIX, u8string/u8path functions are not - * safe to use because paths are not always valid UTF-8, so plain string - * methods which do not transform the path there are used. + * Because \ref PathToString and \ref PathFromString functions don't specify an + * encoding, they are meant to be used internally, not externally. They are not + * appropriate to use in applications requiring UTF-8, where + * fs::path::u8string() and fs::u8path() methods should be used instead. Other + * applications could require still different encodings. For example, JSON, XML, + * or URI applications might prefer to use higher level escapes (\uXXXX or + * &XXXX; or %XX) instead of multibyte encoding. Rust, Python, Java applications + * may require encoding paths with their respective UTF-8 derivatives WTF-8, + * PEP-383, and CESU-8 (see https://en.wikipedia.org/wiki/UTF-8#Derivatives). */ static inline std::string PathToString(const path& path) { + // Implementation note: On Windows, the std::filesystem::path(string) + // constructor and std::filesystem::path::string() method are not safe to + // use here, because these methods encode the path using C++'s narrow + // multibyte encoding, which on Windows corresponds to the current "code + // page", which is unpredictable and typically not able to represent all + // valid paths. So std::filesystem::path::u8string() and + // std::filesystem::u8path() functions are used instead on Windows. On + // POSIX, u8string/u8path functions are not safe to use because paths are + // not always valid UTF-8, so plain string methods which do not transform + // the path there are used. #ifdef WIN32 return path.u8string(); #else diff --git a/src/httprpc.h b/src/httprpc.h index 5a3b990646..6daf7d28f5 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -31,4 +31,4 @@ void InterruptREST(); */ void StopREST(); -#endif +#endif // BITCOIN_HTTPRPC_H diff --git a/src/i2p.h b/src/i2p.h index cb2efedba8..86643de637 100644 --- a/src/i2p.h +++ b/src/i2p.h @@ -267,4 +267,4 @@ class Session } // namespace sam } // namespace i2p -#endif /* BITCOIN_I2P_H */ +#endif // BITCOIN_I2P_H diff --git a/src/init.cpp b/src/init.cpp index f659de3a02..d5c3acbaad 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -29,13 +29,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include #include @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -436,7 +437,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxreceivebuffer=", strprintf("Maximum per-connection receive buffer, *1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=", strprintf("Maximum per-connection send buffer, *1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxuploadtarget=", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxuploadtarget=", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2psam=", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2pacceptincoming", "If set and -i2psam is also set then incoming I2P connections are accepted via the SAM proxy. If this is not set but -i2psam is set then only outgoing connections will be made to the I2P network. Ignored if -i2psam is not set. Listening for incoming I2P connections is done through the SAM proxy, not by binding to a local address and port (default: 1)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1109,6 +1110,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { const ArgsManager& args = *Assert(node.args); const CChainParams& chainparams = Params(); + + auto opt_max_upload = ParseByteUnits(args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET), ByteUnit::M); + if (!opt_max_upload) { + return InitError(strprintf(_("Unable to parse -maxuploadtarget: '%s' (possible integer overflow?)"), args.GetArg("-maxuploadtarget", ""))); + } + // ********************************************************* Step 4a: application initialization if (!CreatePidFile(args)) { // Detailed error printed inside CreatePidFile(). @@ -1760,8 +1767,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.nSendBufferMaxSize = 1000 * args.GetIntArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000 * args.GetIntArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = args.GetArgs("-addnode"); - - connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetIntArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); + connOptions.nMaxOutboundLimit = *opt_max_upload; connOptions.m_peer_connect_timeout = peer_connect_timeout; for (const std::string& bind_arg : args.GetArgs("-bind")) { diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index d4ceb517dd..38004ce95d 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -287,9 +287,6 @@ class Chain //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; - - //! Check if Taproot has activated - virtual bool isTaprootActive() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 490563426c..a56ed8802d 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -13,12 +13,13 @@ #include #include +#include #include #include #include -#include #include #include +#include #include #include @@ -34,7 +35,7 @@ struct CRecipient; struct PartiallySignedTransaction; struct WalletContext; struct bilingual_str; -typedef uint8_t isminefilter; +using isminefilter = std::underlying_type::type; namespace interfaces { diff --git a/src/key_io.cpp b/src/key_io.cpp index 615f4c9312..6908c5ea52 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -76,12 +76,16 @@ class DestinationEncoder std::string operator()(const CNoDestination& no) const { return {}; } }; -CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str) +CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str, std::vector* error_locations) { std::vector data; uint160 hash; error_str = ""; - if (DecodeBase58Check(str, data, 21)) { + + // Note this will be false if it is a valid Bech32 address for a different network + bool is_bech32 = (ToLower(str.substr(0, params.Bech32HRP().size())) == params.Bech32HRP()); + + if (!is_bech32 && DecodeBase58Check(str, data, 21)) { // base58-encoded Bitcoin addresses. // Public-key-hash-addresses have version 0 (or 111 testnet). // The data vector contains RIPEMD160(SHA256(pubkey)), where pubkey is the serialized public key. @@ -98,15 +102,27 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return ScriptHash(hash); } - // Set potential error message. - // This message may be changed if the address can also be interpreted as a Bech32 address. - error_str = "Invalid prefix for Base58-encoded address"; + if (!std::equal(script_prefix.begin(), script_prefix.end(), data.begin()) && + !std::equal(pubkey_prefix.begin(), pubkey_prefix.end(), data.begin())) { + error_str = "Invalid prefix for Base58-encoded address"; + } else { + error_str = "Invalid length for Base58 address"; + } + return CNoDestination(); + } else if (!is_bech32) { + // Try Base58 decoding without the checksum, using a much larger max length + if (!DecodeBase58(str, data, 100)) { + error_str = "Invalid HRP or Base58 character in address"; + } else { + error_str = "Invalid checksum or length of Base58 address"; + } + return CNoDestination(); } + data.clear(); const auto dec = bech32::Decode(str); if ((dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) && dec.data.size() > 0) { // Bech32 decoding - error_str = ""; if (dec.hrp != params.Bech32HRP()) { error_str = "Invalid prefix for Bech32 address"; return CNoDestination(); @@ -168,8 +184,13 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } } - // Set error message if address can't be interpreted as Base58 or Bech32. - if (error_str.empty()) error_str = "Invalid address format"; + // Perform Bech32 error location + if (!error_locations) { + std::vector dummy_errors; + error_str = bech32::LocateErrors(str, dummy_errors); + } else { + error_str = bech32::LocateErrors(str, *error_locations); + } return CNoDestination(); } @@ -258,9 +279,9 @@ std::string EncodeDestination(const CTxDestination& dest) return std::visit(DestinationEncoder(Params()), dest); } -CTxDestination DecodeDestination(const std::string& str, std::string& error_msg) +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg, std::vector* error_locations) { - return DecodeDestination(str, Params(), error_msg); + return DecodeDestination(str, Params(), error_msg, error_locations); } CTxDestination DecodeDestination(const std::string& str) @@ -272,7 +293,7 @@ CTxDestination DecodeDestination(const std::string& str) bool IsValidDestinationString(const std::string& str, const CChainParams& params) { std::string error_msg; - return IsValidDestination(DecodeDestination(str, params, error_msg)); + return IsValidDestination(DecodeDestination(str, params, error_msg, nullptr)); } bool IsValidDestinationString(const std::string& str) diff --git a/src/key_io.h b/src/key_io.h index bd81f7847e..2062bb4c44 100644 --- a/src/key_io.h +++ b/src/key_io.h @@ -23,7 +23,7 @@ std::string EncodeExtPubKey(const CExtPubKey& extpubkey); std::string EncodeDestination(const CTxDestination& dest); CTxDestination DecodeDestination(const std::string& str); -CTxDestination DecodeDestination(const std::string& str, std::string& error_msg); +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg, std::vector* error_locations = nullptr); bool IsValidDestinationString(const std::string& str); bool IsValidDestinationString(const std::string& str, const CChainParams& params); diff --git a/src/net.cpp b/src/net.cpp index 82e55d4189..db496c2185 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -326,8 +326,8 @@ bool IsLocal(const CService& addr) CNode* CConnman::FindNode(const CNetAddr& ip) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (static_cast(pnode->addr) == ip) { return pnode; } @@ -337,8 +337,8 @@ CNode* CConnman::FindNode(const CNetAddr& ip) CNode* CConnman::FindNode(const CSubNet& subNet) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (subNet.Match(static_cast(pnode->addr))) { return pnode; } @@ -348,8 +348,8 @@ CNode* CConnman::FindNode(const CSubNet& subNet) CNode* CConnman::FindNode(const std::string& addrName) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (pnode->m_addr_name == addrName) { return pnode; } @@ -359,8 +359,8 @@ CNode* CConnman::FindNode(const std::string& addrName) CNode* CConnman::FindNode(const CService& addr) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (static_cast(pnode->addr) == addr) { return pnode; } @@ -375,8 +375,8 @@ bool CConnman::AlreadyConnectedToAddress(const CAddress& addr) bool CConnman::CheckIncomingNonce(uint64_t nonce) { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (!pnode->fSuccessfullyConnected && !pnode->IsInboundConn() && pnode->GetLocalNonce() == nonce) return false; } @@ -435,7 +435,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo } // It is possible that we already have a connection to the IP/port pszDest resolved to. // In that case, drop the connection that was just created. - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); CNode* pnode = FindNode(static_cast(addrConnect)); if (pnode) { LogPrintf("Failed to open new connection, already connected\n"); @@ -1056,8 +1056,8 @@ bool CConnman::AttemptToEvictConnection() std::vector vEvictionCandidates; { - LOCK(cs_vNodes); - for (const CNode* node : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* node : m_nodes) { if (node->HasPermission(NetPermissionFlags::NoBan)) continue; if (!node->IsInboundConn()) @@ -1084,8 +1084,8 @@ bool CConnman::AttemptToEvictConnection() if (!node_id_to_evict) { return false; } - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (pnode->GetId() == *node_id_to_evict) { LogPrint(BCLog::NET, "selected %s connection for eviction peer=%d; disconnecting\n", pnode->ConnectionTypeAsString(), pnode->GetId()); pnode->fDisconnect = true; @@ -1141,8 +1141,8 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, } { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->IsInboundConn()) nInbound++; } } @@ -1210,8 +1210,8 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); { - LOCK(cs_vNodes); - vNodes.push_back(pnode); + LOCK(m_nodes_mutex); + m_nodes.push_back(pnode); } // We received a new connection, harvest entropy from the time (and our peer count) @@ -1238,8 +1238,8 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ } // no default case, so the compiler can warn about missing cases // Count existing connections - int existing_connections = WITH_LOCK(cs_vNodes, - return std::count_if(vNodes.begin(), vNodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); + int existing_connections = WITH_LOCK(m_nodes_mutex, + return std::count_if(m_nodes.begin(), m_nodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); // Max connections of specified type already exist if (max_connections != std::nullopt && existing_connections >= max_connections) return false; @@ -1255,11 +1255,11 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ void CConnman::DisconnectNodes() { { - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); if (!fNetworkActive) { // Disconnect any connected nodes - for (CNode* pnode : vNodes) { + for (CNode* pnode : m_nodes) { if (!pnode->fDisconnect) { LogPrint(BCLog::NET, "Network not active, dropping peer=%d\n", pnode->GetId()); pnode->fDisconnect = true; @@ -1268,13 +1268,13 @@ void CConnman::DisconnectNodes() } // Disconnect unused nodes - std::vector vNodesCopy = vNodes; - for (CNode* pnode : vNodesCopy) + std::vector nodes_copy = m_nodes; + for (CNode* pnode : nodes_copy) { if (pnode->fDisconnect) { - // remove from vNodes - vNodes.erase(remove(vNodes.begin(), vNodes.end(), pnode), vNodes.end()); + // remove from m_nodes + m_nodes.erase(remove(m_nodes.begin(), m_nodes.end(), pnode), m_nodes.end()); // release outbound grant (if any) pnode->grantOutbound.Release(); @@ -1284,18 +1284,18 @@ void CConnman::DisconnectNodes() // hold in disconnected pool until all refs are released pnode->Release(); - vNodesDisconnected.push_back(pnode); + m_nodes_disconnected.push_back(pnode); } } } { // Delete disconnected nodes - std::list vNodesDisconnectedCopy = vNodesDisconnected; - for (CNode* pnode : vNodesDisconnectedCopy) + std::list nodes_disconnected_copy = m_nodes_disconnected; + for (CNode* pnode : nodes_disconnected_copy) { // Destroy the object only after other threads have stopped using it. if (pnode->GetRefCount() <= 0) { - vNodesDisconnected.remove(pnode); + m_nodes_disconnected.remove(pnode); DeleteNode(pnode); } } @@ -1304,15 +1304,15 @@ void CConnman::DisconnectNodes() void CConnman::NotifyNumConnectionsChanged() { - size_t vNodesSize; + size_t nodes_size; { - LOCK(cs_vNodes); - vNodesSize = vNodes.size(); + LOCK(m_nodes_mutex); + nodes_size = m_nodes.size(); } - if(vNodesSize != nPrevNodeCount) { - nPrevNodeCount = vNodesSize; + if(nodes_size != nPrevNodeCount) { + nPrevNodeCount = nodes_size; if (m_client_interface) { - m_client_interface->NotifyNumConnectionsChanged(vNodesSize); + m_client_interface->NotifyNumConnectionsChanged(nodes_size); } } } @@ -1353,46 +1353,45 @@ bool CConnman::InactivityCheck(const CNode& node) const return false; } -bool CConnman::GenerateSelectSet(std::set &recv_set, std::set &send_set, std::set &error_set) +bool CConnman::GenerateSelectSet(const std::vector& nodes, + std::set& recv_set, + std::set& send_set, + std::set& error_set) { for (const ListenSocket& hListenSocket : vhListenSocket) { recv_set.insert(hListenSocket.socket); } - { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) + for (CNode* pnode : nodes) { + // Implement the following logic: + // * If there is data to send, select() for sending data. As this only + // happens when optimistic write failed, we choose to first drain the + // write buffer in this case before receiving more. This avoids + // needlessly queueing received data, if the remote peer is not themselves + // receiving data. This means properly utilizing TCP flow control signalling. + // * Otherwise, if there is space left in the receive buffer, select() for + // receiving data. + // * Hand off all complete messages to the processor, to be handled without + // blocking here. + + bool select_recv = !pnode->fPauseRecv; + bool select_send; { - // Implement the following logic: - // * If there is data to send, select() for sending data. As this only - // happens when optimistic write failed, we choose to first drain the - // write buffer in this case before receiving more. This avoids - // needlessly queueing received data, if the remote peer is not themselves - // receiving data. This means properly utilizing TCP flow control signalling. - // * Otherwise, if there is space left in the receive buffer, select() for - // receiving data. - // * Hand off all complete messages to the processor, to be handled without - // blocking here. - - bool select_recv = !pnode->fPauseRecv; - bool select_send; - { - LOCK(pnode->cs_vSend); - select_send = !pnode->vSendMsg.empty(); - } + LOCK(pnode->cs_vSend); + select_send = !pnode->vSendMsg.empty(); + } - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) - continue; + LOCK(pnode->cs_hSocket); + if (pnode->hSocket == INVALID_SOCKET) + continue; - error_set.insert(pnode->hSocket); - if (select_send) { - send_set.insert(pnode->hSocket); - continue; - } - if (select_recv) { - recv_set.insert(pnode->hSocket); - } + error_set.insert(pnode->hSocket); + if (select_send) { + send_set.insert(pnode->hSocket); + continue; + } + if (select_recv) { + recv_set.insert(pnode->hSocket); } } @@ -1400,10 +1399,13 @@ bool CConnman::GenerateSelectSet(std::set &recv_set, std::set &s } #ifdef USE_POLL -void CConnman::SocketEvents(std::set &recv_set, std::set &send_set, std::set &error_set) +void CConnman::SocketEvents(const std::vector& nodes, + std::set& recv_set, + std::set& send_set, + std::set& error_set) { std::set recv_select_set, send_select_set, error_select_set; - if (!GenerateSelectSet(recv_select_set, send_select_set, error_select_set)) { + if (!GenerateSelectSet(nodes, recv_select_set, send_select_set, error_select_set)) { interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS)); return; } @@ -1442,10 +1444,13 @@ void CConnman::SocketEvents(std::set &recv_set, std::set &send_s } } #else -void CConnman::SocketEvents(std::set &recv_set, std::set &send_set, std::set &error_set) +void CConnman::SocketEvents(const std::vector& nodes, + std::set& recv_set, + std::set& send_set, + std::set& error_set) { std::set recv_select_set, send_select_set, error_select_set; - if (!GenerateSelectSet(recv_select_set, send_select_set, error_select_set)) { + if (!GenerateSelectSet(nodes, recv_select_set, send_select_set, error_select_set)) { interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS)); return; } @@ -1519,34 +1524,33 @@ void CConnman::SocketEvents(std::set &recv_set, std::set &send_s void CConnman::SocketHandler() { - std::set recv_set, send_set, error_set; - SocketEvents(recv_set, send_set, error_set); - - if (interruptNet) return; + std::set recv_set; + std::set send_set; + std::set error_set; - // - // Accept new connections - // - for (const ListenSocket& hListenSocket : vhListenSocket) { - if (hListenSocket.socket != INVALID_SOCKET && recv_set.count(hListenSocket.socket) > 0) - { - AcceptConnection(hListenSocket); - } - } + const NodesSnapshot snap{*this, /*shuffle=*/false}; - // - // Service each socket - // - std::vector vNodesCopy; - { - LOCK(cs_vNodes); - vNodesCopy = vNodes; - for (CNode* pnode : vNodesCopy) - pnode->AddRef(); + // Check for the readiness of the already connected sockets and the + // listening sockets in one call ("readiness" as in poll(2) or + // select(2)). If none are ready, wait for a short while and return + // empty sets. + SocketEvents(snap.Nodes(), recv_set, send_set, error_set); + + // Service (send/receive) each of the already connected nodes. + SocketHandlerConnected(snap.Nodes(), recv_set, send_set, error_set); } - for (CNode* pnode : vNodesCopy) - { + + // Accept new connections from listening sockets. + SocketHandlerListening(recv_set); +} + +void CConnman::SocketHandlerConnected(const std::vector& nodes, + const std::set& recv_set, + const std::set& send_set, + const std::set& error_set) +{ + for (CNode* pnode : nodes) { if (interruptNet) return; @@ -1628,10 +1632,17 @@ void CConnman::SocketHandler() if (InactivityCheck(*pnode)) pnode->fDisconnect = true; } - { - LOCK(cs_vNodes); - for (CNode* pnode : vNodesCopy) - pnode->Release(); +} + +void CConnman::SocketHandlerListening(const std::set& recv_set) +{ + for (const ListenSocket& listen_socket : vhListenSocket) { + if (interruptNet) { + return; + } + if (listen_socket.socket != INVALID_SOCKET && recv_set.count(listen_socket.socket) > 0) { + AcceptConnection(listen_socket); + } } } @@ -1705,8 +1716,8 @@ void CConnman::ThreadDNSAddressSeed() int nRelevant = 0; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->fSuccessfullyConnected && pnode->IsFullOutboundConn()) ++nRelevant; } } @@ -1814,8 +1825,8 @@ int CConnman::GetExtraFullOutboundCount() const { int full_outbound_peers = 0; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsFullOutboundConn()) { ++full_outbound_peers; } @@ -1828,8 +1839,8 @@ int CConnman::GetExtraBlockRelayCount() const { int block_relay_peers = 0; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsBlockOnlyConn()) { ++block_relay_peers; } @@ -1900,8 +1911,8 @@ void CConnman::ThreadOpenConnections(const std::vector connect) // Checking !dnsseed is cheaper before locking 2 mutexes. if (!add_fixed_seeds_now && !dnsseed) { - LOCK2(m_addr_fetches_mutex, cs_vAddedNodes); - if (m_addr_fetches.empty() && vAddedNodes.empty()) { + LOCK2(m_addr_fetches_mutex, m_added_nodes_mutex); + if (m_addr_fetches.empty() && m_added_nodes.empty()) { add_fixed_seeds_now = true; LogPrintf("Adding fixed seeds as -dnsseed=0, -addnode is not provided and all -seednode(s) attempted\n"); } @@ -1926,8 +1937,8 @@ void CConnman::ThreadOpenConnections(const std::vector connect) std::set > setConnected; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->IsFullOutboundConn()) nOutboundFullRelay++; if (pnode->IsBlockOnlyConn()) nOutboundBlockRelay++; @@ -2115,8 +2126,8 @@ void CConnman::ThreadOpenConnections(const std::vector connect) std::vector CConnman::GetCurrentBlockRelayOnlyConns() const { std::vector ret; - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->IsBlockOnlyConn()) { ret.push_back(pnode->addr); } @@ -2131,9 +2142,9 @@ std::vector CConnman::GetAddedNodeInfo() const std::list lAddresses(0); { - LOCK(cs_vAddedNodes); - ret.reserve(vAddedNodes.size()); - std::copy(vAddedNodes.cbegin(), vAddedNodes.cend(), std::back_inserter(lAddresses)); + LOCK(m_added_nodes_mutex); + ret.reserve(m_added_nodes.size()); + std::copy(m_added_nodes.cbegin(), m_added_nodes.cend(), std::back_inserter(lAddresses)); } @@ -2141,8 +2152,8 @@ std::vector CConnman::GetAddedNodeInfo() const std::map mapConnected; std::map> mapConnectedByName; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->addr.IsValid()) { mapConnected[pnode->addr] = pnode->IsInboundConn(); } @@ -2238,57 +2249,42 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai m_msgproc->InitializeNode(pnode); { - LOCK(cs_vNodes); - vNodes.push_back(pnode); + LOCK(m_nodes_mutex); + m_nodes.push_back(pnode); } } void CConnman::ThreadMessageHandler() { SetSyscallSandboxPolicy(SyscallSandboxPolicy::MESSAGE_HANDLER); - FastRandomContext rng; while (!flagInterruptMsgProc) { - std::vector vNodesCopy; - { - LOCK(cs_vNodes); - vNodesCopy = vNodes; - for (CNode* pnode : vNodesCopy) { - pnode->AddRef(); - } - } - bool fMoreWork = false; - // Randomize the order in which we process messages from/to our peers. - // This prevents attacks in which an attacker exploits having multiple - // consecutive connections in the vNodes list. - Shuffle(vNodesCopy.begin(), vNodesCopy.end(), rng); - - for (CNode* pnode : vNodesCopy) { - if (pnode->fDisconnect) - continue; + // Randomize the order in which we process messages from/to our peers. + // This prevents attacks in which an attacker exploits having multiple + // consecutive connections in the m_nodes list. + const NodesSnapshot snap{*this, /*shuffle=*/true}; - // Receive messages - bool fMoreNodeWork = m_msgproc->ProcessMessages(pnode, flagInterruptMsgProc); - fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); - if (flagInterruptMsgProc) - return; - // Send messages - { - LOCK(pnode->cs_sendProcessing); - m_msgproc->SendMessages(pnode); - } + for (CNode* pnode : snap.Nodes()) { + if (pnode->fDisconnect) + continue; - if (flagInterruptMsgProc) - return; - } + // Receive messages + bool fMoreNodeWork = m_msgproc->ProcessMessages(pnode, flagInterruptMsgProc); + fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); + if (flagInterruptMsgProc) + return; + // Send messages + { + LOCK(pnode->cs_sendProcessing); + m_msgproc->SendMessages(pnode); + } - { - LOCK(cs_vNodes); - for (CNode* pnode : vNodesCopy) - pnode->Release(); + if (flagInterruptMsgProc) + return; + } } WAIT_LOCK(mutexMsgProc, lock); @@ -2698,7 +2694,7 @@ void CConnman::StopNodes() // Delete peer connections. std::vector nodes; - WITH_LOCK(cs_vNodes, nodes.swap(vNodes)); + WITH_LOCK(m_nodes_mutex, nodes.swap(m_nodes)); for (CNode* pnode : nodes) { pnode->CloseSocketDisconnect(); DeleteNode(pnode); @@ -2713,10 +2709,10 @@ void CConnman::StopNodes() } } - for (CNode* pnode : vNodesDisconnected) { + for (CNode* pnode : m_nodes_disconnected) { DeleteNode(pnode); } - vNodesDisconnected.clear(); + m_nodes_disconnected.clear(); vhListenSocket.clear(); semOutbound.reset(); semAddnode.reset(); @@ -2789,21 +2785,21 @@ std::vector CConnman::GetAddresses(CNode& requestor, size_t max_addres bool CConnman::AddNode(const std::string& strNode) { - LOCK(cs_vAddedNodes); - for (const std::string& it : vAddedNodes) { + LOCK(m_added_nodes_mutex); + for (const std::string& it : m_added_nodes) { if (strNode == it) return false; } - vAddedNodes.push_back(strNode); + m_added_nodes.push_back(strNode); return true; } bool CConnman::RemoveAddedNode(const std::string& strNode) { - LOCK(cs_vAddedNodes); - for(std::vector::iterator it = vAddedNodes.begin(); it != vAddedNodes.end(); ++it) { + LOCK(m_added_nodes_mutex); + for(std::vector::iterator it = m_added_nodes.begin(); it != m_added_nodes.end(); ++it) { if (strNode == *it) { - vAddedNodes.erase(it); + m_added_nodes.erase(it); return true; } } @@ -2812,12 +2808,12 @@ bool CConnman::RemoveAddedNode(const std::string& strNode) size_t CConnman::GetNodeCount(ConnectionDirection flags) const { - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); if (flags == ConnectionDirection::Both) // Shortcut if we want total - return vNodes.size(); + return m_nodes.size(); int nNum = 0; - for (const auto& pnode : vNodes) { + for (const auto& pnode : m_nodes) { if (flags & (pnode->IsInboundConn() ? ConnectionDirection::In : ConnectionDirection::Out)) { nNum++; } @@ -2829,9 +2825,9 @@ size_t CConnman::GetNodeCount(ConnectionDirection flags) const void CConnman::GetNodeStats(std::vector& vstats) const { vstats.clear(); - LOCK(cs_vNodes); - vstats.reserve(vNodes.size()); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + vstats.reserve(m_nodes.size()); + for (CNode* pnode : m_nodes) { vstats.emplace_back(); pnode->CopyStats(vstats.back()); vstats.back().m_mapped_as = pnode->addr.GetMappedAS(addrman.GetAsmap()); @@ -2840,7 +2836,7 @@ void CConnman::GetNodeStats(std::vector& vstats) const bool CConnman::DisconnectNode(const std::string& strNode) { - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); if (CNode* pnode = FindNode(strNode)) { LogPrint(BCLog::NET, "disconnect by address%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", strNode) : ""), pnode->GetId()); pnode->fDisconnect = true; @@ -2852,8 +2848,8 @@ bool CConnman::DisconnectNode(const std::string& strNode) bool CConnman::DisconnectNode(const CSubNet& subnet) { bool disconnected = false; - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (subnet.Match(pnode->addr)) { LogPrint(BCLog::NET, "disconnect by subnet%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", subnet.ToString()) : ""), pnode->GetId()); pnode->fDisconnect = true; @@ -2870,8 +2866,8 @@ bool CConnman::DisconnectNode(const CNetAddr& addr) bool CConnman::DisconnectNode(NodeId id) { - LOCK(cs_vNodes); - for(CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for(CNode* pnode : m_nodes) { if (id == pnode->GetId()) { LogPrint(BCLog::NET, "disconnect by id peer=%d; disconnecting\n", pnode->GetId()); pnode->fDisconnect = true; @@ -2883,7 +2879,6 @@ bool CConnman::DisconnectNode(NodeId id) void CConnman::RecordBytesRecv(uint64_t bytes) { - LOCK(cs_totalBytesRecv); nTotalBytesRecv += bytes; } @@ -2960,7 +2955,6 @@ uint64_t CConnman::GetOutboundTargetBytesLeft() const uint64_t CConnman::GetTotalBytesRecv() const { - LOCK(cs_totalBytesRecv); return nTotalBytesRecv; } @@ -3024,7 +3018,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", msg.m_type, nMessageSize, pnode->GetId()); if (gArgs.GetBoolArg("-capturemessages", false)) { - CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); + CaptureMessage(pnode->addr, msg.m_type, msg.data, /*is_incoming=*/false); } TRACE6(net, outbound_message, @@ -3063,8 +3057,8 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) bool CConnman::ForNode(NodeId id, std::function func) { CNode* found = nullptr; - LOCK(cs_vNodes); - for (auto&& pnode : vNodes) { + LOCK(m_nodes_mutex); + for (auto&& pnode : m_nodes) { if(pnode->GetId() == id) { found = pnode; break; diff --git a/src/net.h b/src/net.h index 878f10cd42..dd5cc66a04 100644 --- a/src/net.h +++ b/src/net.h @@ -70,7 +70,7 @@ static const bool DEFAULT_LISTEN = true; /** The maximum number of peer connections to maintain. */ static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 125; /** The default for -maxuploadtarget. 0 = Unlimited */ -static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; +static const std::string DEFAULT_MAX_UPLOAD_TARGET{"0M"}; /** Default for blocks only*/ static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ @@ -791,8 +791,8 @@ class CConnman } vWhitelistedRange = connOptions.vWhitelistedRange; { - LOCK(cs_vAddedNodes); - vAddedNodes = connOptions.m_added_nodes; + LOCK(m_added_nodes_mutex); + m_added_nodes = connOptions.m_added_nodes; } m_onion_binds = connOptions.onion_binds; } @@ -823,8 +823,8 @@ class CConnman using NodeFn = std::function; void ForEachNode(const NodeFn& func) { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { + LOCK(m_nodes_mutex); + for (auto&& node : m_nodes) { if (NodeFullyConnected(node)) func(node); } @@ -832,8 +832,8 @@ class CConnman void ForEachNode(const NodeFn& func) const { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { + LOCK(m_nodes_mutex); + for (auto&& node : m_nodes) { if (NodeFullyConnected(node)) func(node); } @@ -968,7 +968,7 @@ class CConnman /** * Create a `CNode` object from a socket that has just been accepted and add the node to - * the `vNodes` member. + * the `m_nodes` member. * @param[in] hSocket Connected socket to communicate with the peer. * @param[in] permissionFlags The peer's permissions. * @param[in] addr_bind The address and port at our side of the connection. @@ -983,9 +983,57 @@ class CConnman void NotifyNumConnectionsChanged(); /** Return true if the peer is inactive and should be disconnected. */ bool InactivityCheck(const CNode& node) const; - bool GenerateSelectSet(std::set &recv_set, std::set &send_set, std::set &error_set); - void SocketEvents(std::set &recv_set, std::set &send_set, std::set &error_set); + + /** + * Generate a collection of sockets to check for IO readiness. + * @param[in] nodes Select from these nodes' sockets. + * @param[out] recv_set Sockets to check for read readiness. + * @param[out] send_set Sockets to check for write readiness. + * @param[out] error_set Sockets to check for errors. + * @return true if at least one socket is to be checked (the returned set is not empty) + */ + bool GenerateSelectSet(const std::vector& nodes, + std::set& recv_set, + std::set& send_set, + std::set& error_set); + + /** + * Check which sockets are ready for IO. + * @param[in] nodes Select from these nodes' sockets. + * @param[out] recv_set Sockets which are ready for read. + * @param[out] send_set Sockets which are ready for write. + * @param[out] error_set Sockets which have errors. + * This calls `GenerateSelectSet()` to gather a list of sockets to check. + */ + void SocketEvents(const std::vector& nodes, + std::set& recv_set, + std::set& send_set, + std::set& error_set); + + /** + * Check connected and listening sockets for IO readiness and process them accordingly. + */ void SocketHandler(); + + /** + * Do the read/write for connected sockets that are ready for IO. + * @param[in] nodes Nodes to process. The socket of each node is checked against + * `recv_set`, `send_set` and `error_set`. + * @param[in] recv_set Sockets that are ready for read. + * @param[in] send_set Sockets that are ready for send. + * @param[in] error_set Sockets that have an exceptional condition (error). + */ + void SocketHandlerConnected(const std::vector& nodes, + const std::set& recv_set, + const std::set& send_set, + const std::set& error_set); + + /** + * Accept incoming connections, one from each read-ready listening socket. + * @param[in] recv_set Sockets that are ready for read. + */ + void SocketHandlerListening(const std::set& recv_set); + void ThreadSocketHandler(); void ThreadDNSAddressSeed(); @@ -1026,9 +1074,8 @@ class CConnman static bool NodeFullyConnected(const CNode* pnode); // Network usage totals - mutable RecursiveMutex cs_totalBytesRecv; mutable RecursiveMutex cs_totalBytesSent; - uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv) {0}; + std::atomic nTotalBytesRecv{0}; uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; // outbound limit & stats @@ -1051,12 +1098,12 @@ class CConnman bool fAddressesInitialized{false}; AddrMan& addrman; std::deque m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); - RecursiveMutex m_addr_fetches_mutex; - std::vector vAddedNodes GUARDED_BY(cs_vAddedNodes); - mutable RecursiveMutex cs_vAddedNodes; - std::vector vNodes GUARDED_BY(cs_vNodes); - std::list vNodesDisconnected; - mutable RecursiveMutex cs_vNodes; + Mutex m_addr_fetches_mutex; + std::vector m_added_nodes GUARDED_BY(m_added_nodes_mutex); + mutable Mutex m_added_nodes_mutex; + std::vector m_nodes GUARDED_BY(m_nodes_mutex); + std::list m_nodes_disconnected; + mutable RecursiveMutex m_nodes_mutex; std::atomic nLastNodeId{0}; unsigned int nPrevNodeCount{0}; @@ -1177,6 +1224,43 @@ class CConnman */ std::vector m_onion_binds; + /** + * RAII helper to atomically create a copy of `m_nodes` and add a reference + * to each of the nodes. The nodes are released when this object is destroyed. + */ + class NodesSnapshot + { + public: + explicit NodesSnapshot(const CConnman& connman, bool shuffle) + { + { + LOCK(connman.m_nodes_mutex); + m_nodes_copy = connman.m_nodes; + for (auto& node : m_nodes_copy) { + node->AddRef(); + } + } + if (shuffle) { + Shuffle(m_nodes_copy.begin(), m_nodes_copy.end(), FastRandomContext{}); + } + } + + ~NodesSnapshot() + { + for (auto& node : m_nodes_copy) { + node->Release(); + } + } + + const std::vector& Nodes() const + { + return m_nodes_copy; + } + + private: + std::vector m_nodes_copy; + }; + friend struct CConnmanTest; friend struct ConnmanTestMsg; }; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2185ccc700..3f755eb178 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3206,6 +3206,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Stop processing the transaction early if we are still in IBD since we don't + // have enough information to validate it yet. Sending unsolicited transactions + // is not considered a protocol violation, so don't punish the peer. + if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) return; + CTransactionRef ptx; vRecv >> ptx; const CTransaction& tx = *ptx; @@ -4105,7 +4110,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic& interrupt ); if (gArgs.GetBoolArg("-capturemessages", false)) { - CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); + CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /*is_incoming=*/true); } msg.SetVersion(pfrom->GetCommonVersion()); diff --git a/src/netaddress.h b/src/netaddress.h index b0b1c5ca9e..a5b74eb35b 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -385,6 +385,12 @@ class CNetAddr /** * Unserialize from a pre-ADDRv2/BIP155 format from an array. + * + * This function is only called from UnserializeV1Stream() and is a wrapper + * for SetLegacyIPv6(); however, we keep it for symmetry with + * SerializeV1Array() to have pairs of ser/unser functions and to make clear + * that if one is altered, a corresponding reverse modification should be + * applied to the other. */ void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 9c2d23f839..e5f2753123 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -540,8 +540,11 @@ class ChainImpl : public Chain const CBlockIndex* block2 = m_node.chainman->m_blockman.LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving - // output uninitialized. - return FillBlock(ancestor, ancestor_out, lock, active) & FillBlock(block1, block1_out, lock, active) & FillBlock(block2, block2_out, lock, active); + // output uninitialized. Cast bool to int to avoid -Wbitwise-instead-of-logical + // compiler warnings. + return int{FillBlock(ancestor, ancestor_out, lock, active)} & + int{FillBlock(block1, block1_out, lock, active)} & + int{FillBlock(block2, block2_out, lock, active)}; } void findCoins(std::map& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -721,12 +724,6 @@ class ChainImpl : public Chain notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } - bool isTaprootActive() override - { - LOCK(::cs_main); - const CBlockIndex* tip = Assert(m_node.chainman)->ActiveChain().Tip(); - return DeploymentActiveAfter(tip, Params().GetConsensus(), Consensus::DEPLOYMENT_TAPROOT); - } NodeContext& m_node; }; } // namespace diff --git a/src/miner.cpp b/src/node/miner.cpp similarity index 99% rename from src/miner.cpp rename to src/node/miner.cpp index 1ef246cd14..8778a79f8b 100644 --- a/src/miner.cpp +++ b/src/node/miner.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include @@ -21,6 +21,7 @@ #include #include #include +#include #include #include diff --git a/src/miner.h b/src/node/miner.h similarity index 98% rename from src/miner.h rename to src/node/miner.h index 10a80f4392..0e8c02793a 100644 --- a/src/miner.h +++ b/src/node/miner.h @@ -3,20 +3,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_MINER_H -#define BITCOIN_MINER_H +#ifndef BITCOIN_NODE_MINER_H +#define BITCOIN_NODE_MINER_H #include #include -#include #include #include #include -#include #include +#include +class ChainstateManager; class CBlockIndex; class CChainParams; class CScript; @@ -205,4 +205,4 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman); -#endif // BITCOIN_MINER_H +#endif // BITCOIN_NODE_MINER_H diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 8ba896bb01..13c7ec2002 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -67,4 +67,4 @@ class CFeeRate SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; -#endif // BITCOIN_POLICY_FEERATE_H +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index fced397e51..2c30b20d5b 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -161,13 +161,13 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR * * Note that only the non-witness portion of the transaction is checked here. */ -bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, bool taproot_active) +bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) { - if (tx.IsCoinBase()) + if (tx.IsCoinBase()) { return true; // Coinbases don't use vin normally + } - for (unsigned int i = 0; i < tx.vin.size(); i++) - { + for (unsigned int i = 0; i < tx.vin.size(); i++) { const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector > vSolutions; @@ -189,9 +189,6 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, if (subscript.GetSigOpCount(true) > MAX_P2SH_SIGOPS) { return false; } - } else if (whichType == TxoutType::WITNESS_V1_TAPROOT) { - // Don't allow Taproot spends unless Taproot is active. - if (!taproot_active) return false; } } diff --git a/src/policy/policy.h b/src/policy/policy.h index f2a3f35546..f6ac6500f6 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -105,10 +105,9 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR /** * Check for standard transaction types * @param[in] mapInputs Map of previous transactions that have outputs we're spending -* @param[in] taproot_active Whether or taproot consensus rules are active (used to decide whether spends of them are permitted) * @return True if all inputs (scriptSigs) use only standard transaction forms */ -bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, bool taproot_active); +bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs); /** * Check if the transaction is over standard P2WSH resources limit: * 3600bytes witnessScript size, 80bytes per witness stack element, 100 witness stack elements diff --git a/src/psbt.cpp b/src/psbt.cpp index 5445bc8aa1..6585766807 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -223,7 +223,7 @@ void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransactio // Construct a would-be spend of this output, to update sigdata with. // Note that ProduceSignature is used to fill in metadata (not actual signatures), // so provider does not need to provide any private keys (it can be a HidingSigningProvider). - MutableTransactionSignatureCreator creator(&tx, /* index */ 0, out.nValue, SIGHASH_ALL); + MutableTransactionSignatureCreator creator(&tx, /*input_idx=*/0, out.nValue, SIGHASH_ALL); ProduceSignature(provider, creator, out.scriptPubKey, sigdata); // Put redeem_script, witness_script, key paths, into PSBTOutput. @@ -247,7 +247,7 @@ PrecomputedTransactionData PrecomputePSBTData(const PartiallySignedTransaction& return txdata; } -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash, SignatureData* out_sigdata) +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash, SignatureData* out_sigdata, bool finalize) { PSBTInput& input = psbt.inputs.at(index); const CMutableTransaction& tx = *psbt.tx; @@ -295,6 +295,10 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& } // Verify that a witness signature was produced in case one was required. if (require_witness_sig && !sigdata.witness) return false; + + // If we are not finalizing, set sigdata.complete to false to not set the scriptWitness + if (!finalize && sigdata.complete) sigdata.complete = false; + input.FromSignatureData(sigdata); // If we have a witness signature, put a witness UTXO. @@ -324,7 +328,7 @@ bool FinalizePSBT(PartiallySignedTransaction& psbtx) bool complete = true; const PrecomputedTransactionData txdata = PrecomputePSBTData(psbtx); for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - complete &= SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, &txdata, SIGHASH_ALL); + complete &= SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, &txdata, SIGHASH_ALL, nullptr, true); } return complete; diff --git a/src/psbt.h b/src/psbt.h index f6b82b43de..7808a247c0 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -578,7 +578,7 @@ bool PSBTInputSigned(const PSBTInput& input); * txdata should be the output of PrecomputePSBTData (which can be shared across * multiple SignPSBTInput calls). If it is nullptr, a dummy signature will be created. **/ -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr); +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool finalize = true); /** Counts the unsigned inputs of a PSBT. */ size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt); diff --git a/src/pubkey.h b/src/pubkey.h index f174ad8d85..ae6356c246 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -141,7 +141,7 @@ class CPubKey template void Unserialize(Stream& s) { - unsigned int len = ::ReadCompactSize(s); + const unsigned int len(::ReadCompactSize(s)); if (len <= SIZE) { s.read((char*)vch, len); if (len != size()) { @@ -149,9 +149,7 @@ class CPubKey } } else { // invalid pubkey, skip available data - char dummy; - while (len--) - s.read(&dummy, 1); + s.ignore(len); Invalidate(); } } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index b68ce39b53..81a1d88d20 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -870,7 +870,7 @@ void BitcoinGUI::showHelpMessageClicked() #ifdef ENABLE_WALLET void BitcoinGUI::openClicked() { - OpenURIDialog dlg(this); + OpenURIDialog dlg(platformStyle, this); if(dlg.exec()) { Q_EMIT receivedURI(dlg.getURI()); diff --git a/src/qt/forms/openuridialog.ui b/src/qt/forms/openuridialog.ui index 1b7291ab9d..97399e59a2 100644 --- a/src/qt/forms/openuridialog.ui +++ b/src/qt/forms/openuridialog.ui @@ -30,6 +30,27 @@ + + + + Paste address from clipboard + + + + + + + :/icons/editpaste:/icons/editpaste + + + + + 22 + 22 + + + + diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 1c22124616..6b3a4630a3 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -33,7 +33,7 @@ Automatically start %1 after logging in to the system. - Start %1 on system &login + &Start %1 on system login @@ -192,7 +192,7 @@ This allows you or a third party tool to communicate with the node through command-line and JSON-RPC commands. - Enable RPC &server + Enable R&PC server diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 10bf82d532..a68eee718e 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -6,15 +6,20 @@ #include #include +#include #include +#include +#include #include -OpenURIDialog::OpenURIDialog(QWidget *parent) : - QDialog(parent, GUIUtil::dialog_flags), - ui(new Ui::OpenURIDialog) +OpenURIDialog::OpenURIDialog(const PlatformStyle* platformStyle, QWidget* parent) : QDialog(parent, GUIUtil::dialog_flags), + ui(new Ui::OpenURIDialog), + m_platform_style(platformStyle) { ui->setupUi(this); + ui->pasteButton->setIcon(m_platform_style->SingleColorIcon(":/icons/editpaste")); + QObject::connect(ui->pasteButton, &QAbstractButton::clicked, ui->uriEdit, &QLineEdit::paste); GUIUtil::handleCloseWindowShortcut(this); } @@ -32,11 +37,19 @@ QString OpenURIDialog::getURI() void OpenURIDialog::accept() { SendCoinsRecipient rcp; - if(GUIUtil::parseBitcoinURI(getURI(), &rcp)) - { + if (GUIUtil::parseBitcoinURI(getURI(), &rcp)) { /* Only accept value URIs */ QDialog::accept(); } else { ui->uriEdit->setValid(false); } } + +void OpenURIDialog::changeEvent(QEvent* e) +{ + if (e->type() == QEvent::PaletteChange) { + ui->pasteButton->setIcon(m_platform_style->SingleColorIcon(":/icons/editpaste")); + } + + QDialog::changeEvent(e); +} diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index efe4b86f37..f3a8b0ba22 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -7,6 +7,8 @@ #include +class PlatformStyle; + namespace Ui { class OpenURIDialog; } @@ -16,16 +18,19 @@ class OpenURIDialog : public QDialog Q_OBJECT public: - explicit OpenURIDialog(QWidget *parent); + explicit OpenURIDialog(const PlatformStyle* platformStyle, QWidget* parent); ~OpenURIDialog(); QString getURI(); protected Q_SLOTS: void accept() override; + void changeEvent(QEvent* e) override; private: - Ui::OpenURIDialog *ui; + Ui::OpenURIDialog* ui; + + const PlatformStyle* m_platform_style; }; #endif // BITCOIN_QT_OPENURIDIALOG_H diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0c3332ab76..6bcdcc4c29 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -866,7 +866,11 @@ void RPCConsole::clear(bool keep_prompt) } // Set default style sheet +#ifdef Q_OS_MAC + QFontInfo fixedFontInfo(GUIUtil::fixedPitchFont(/*use_embedded_font=*/true)); +#else QFontInfo fixedFontInfo(GUIUtil::fixedPitchFont()); +#endif ui->messagesWidget->document()->setDefaultStyleSheet( QString( "table { }" diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index b26cddf4ae..e7a3d724bb 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) #if defined(WIN32) if (getenv("QT_QPA_PLATFORM") == nullptr) _putenv_s("QT_QPA_PLATFORM", "minimal"); #else - setenv("QT_QPA_PLATFORM", "minimal", /* overwrite */ 0); + setenv("QT_QPA_PLATFORM", "minimal", 0 /* overwrite */); #endif // Don't remove this, it's needed to access diff --git a/src/randomenv.h b/src/randomenv.h index 46cea6f6f2..746516b79b 100644 --- a/src/randomenv.h +++ b/src/randomenv.h @@ -14,4 +14,4 @@ void RandAddDynamicEnv(CSHA512& hasher); /** Gather non-cryptographic environment data that does not change over time. */ void RandAddStaticEnv(CSHA512& hasher); -#endif +#endif // BITCOIN_RANDOMENV_H diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index d9c6761f47..65f6b1429e 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -67,4 +67,4 @@ CBlockPolicyEstimator& EnsureAnyFeeEstimator(const std::any& context); */ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile); -#endif +#endif // BITCOIN_RPC_BLOCKCHAIN_H diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 93e49cb9a8..90fbd823a4 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -115,6 +115,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "walletcreatefundedpsbt", 4, "bip32derivs" }, { "walletprocesspsbt", 1, "sign" }, { "walletprocesspsbt", 3, "bip32derivs" }, + { "walletprocesspsbt", 4, "finalize" }, { "createpsbt", 0, "inputs" }, { "createpsbt", 1, "outputs" }, { "createpsbt", 2, "locktime" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 518c41d12a..89abdd057c 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -13,9 +13,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -1010,7 +1010,7 @@ static RPCHelpMan submitblock() bool new_block; auto sc = std::make_shared(block.GetHash()); RegisterSharedValidationInterface(sc); - bool accepted = chainman.ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); + bool accepted = chainman.ProcessNewBlock(Params(), blockptr, /*force_processing=*/true, /*new_block=*/&new_block); UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { return "duplicate"; diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 39bd9c6091..2bd8a6b050 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -52,6 +52,10 @@ static RPCHelpMan validateaddress() {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, + {RPCResult::Type::ARR, "error_locations", "Indices of likely error locations in address, if known (e.g. Bech32 errors)", + { + {RPCResult::Type::NUM, "index", "index of a potential error"}, + }}, } }, RPCExamples{ @@ -61,7 +65,8 @@ static RPCHelpMan validateaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::string error_msg; - CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + std::vector error_locations; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg, &error_locations); const bool isValid = IsValidDestination(dest); CHECK_NONFATAL(isValid == error_msg.empty()); @@ -77,6 +82,9 @@ static RPCHelpMan validateaddress() UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); } else { + UniValue error_indices(UniValue::VARR); + for (int i : error_locations) error_indices.push_back(i); + ret.pushKV("error_locations", error_indices); ret.pushKV("error", error_msg); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 89f2309cb7..2dd121c6f6 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1619,7 +1619,7 @@ static RPCHelpMan utxoupdatepsbt() } } // We don't actually need private keys further on; hide them as a precaution. - HidingSigningProvider public_provider(&provider, /* nosign */ true, /* nobip32derivs */ false); + HidingSigningProvider public_provider(&provider, /*hide_secret=*/true, /*hide_origin=*/false); // Fetch previous transactions (inputs): CCoinsView viewDummy; @@ -1658,7 +1658,7 @@ static RPCHelpMan utxoupdatepsbt() // Update script/keypath information using descriptor data. // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures // we don't actually care about those here, in fact. - SignPSBTInput(public_provider, psbtx, i, &txdata, /* sighash_type */ 1); + SignPSBTInput(public_provider, psbtx, i, &txdata, /*sighash=*/1); } // Update script/keypath information using descriptor data. diff --git a/src/scheduler.h b/src/scheduler.h index 9eec8c0fa0..be878661db 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -146,4 +146,4 @@ class SingleThreadedSchedulerClient size_t CallbacksPending(); }; -#endif +#endif // BITCOIN_SCHEDULER_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 621a1b9fd6..c3b4d1ddaa 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -851,6 +851,7 @@ class TRDescriptor final : public DescriptorImpl builder.Finalize(xpk); WitnessV1Taproot output = builder.GetOutput(); out.tr_spenddata[output].Merge(builder.GetSpendData()); + out.pubkeys.emplace(keys[0].GetID(), keys[0]); return Vector(GetScriptForDestination(output)); } bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, const StringType type, const DescriptorCache* cache = nullptr) const override diff --git a/src/script/sign.cpp b/src/script/sign.cpp index b282f39e6d..8e044b1e00 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -17,16 +17,16 @@ typedef std::vector valtype; -MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn) - : txTo(txToIn), nIn(nInIn), nHashType(nHashTypeIn), amount(amountIn), checker(txTo, nIn, amountIn, MissingDataBehavior::FAIL), +MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, int hash_type) + : txTo{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, checker{txTo, nIn, amount, MissingDataBehavior::FAIL}, m_txdata(nullptr) { } -MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData* txdata, int nHashTypeIn) - : txTo(txToIn), nIn(nInIn), nHashType(nHashTypeIn), amount(amountIn), - checker(txdata ? MutableTransactionSignatureChecker(txTo, nIn, amount, *txdata, MissingDataBehavior::FAIL) : - MutableTransactionSignatureChecker(txTo, nIn, amount, MissingDataBehavior::FAIL)), +MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type) + : txTo{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, + checker{txdata ? MutableTransactionSignatureChecker{txTo, nIn, amount, *txdata, MissingDataBehavior::FAIL} : + MutableTransactionSignatureChecker{txTo, nIn, amount, MissingDataBehavior::FAIL}}, m_txdata(txdata) { } diff --git a/src/script/sign.h b/src/script/sign.h index 6d3479c143..62335b644a 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -45,8 +45,8 @@ class MutableTransactionSignatureCreator : public BaseSignatureCreator { const PrecomputedTransactionData* m_txdata; public: - MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn); - MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData* txdata, int nHashTypeIn); + MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, int hash_type); + MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type); const BaseSignatureChecker& Checker() const override { return checker; } bool CreateSig(const SigningProvider& provider, std::vector& vchSig, const CKeyID& keyid, const CScript& scriptCode, SigVersion sigversion) const override; bool CreateSchnorrSig(const SigningProvider& provider, std::vector& sig, const XOnlyPubKey& pubkey, const uint256* leaf_hash, const uint256* merkle_root, SigVersion sigversion) const override; diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index b80fbe22ce..17f97fa30c 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -190,8 +190,8 @@ bool FillableSigningProvider::GetCScript(const CScriptID &hash, CScript& redeemS CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& dest) { - // Only supports destinations which map to single public keys, i.e. P2PKH, - // P2WPKH, and P2SH-P2WPKH. + // Only supports destinations which map to single public keys: + // P2PKH, P2WPKH, P2SH-P2WPKH, P2TR if (auto id = std::get_if(&dest)) { return ToKeyID(*id); } @@ -208,5 +208,15 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& } } } + if (auto output_key = std::get_if(&dest)) { + TaprootSpendData spenddata; + CPubKey pub; + if (store.GetTaprootSpendData(*output_key, spenddata) + && !spenddata.internal_key.IsNull() + && spenddata.merkle_root.IsNull() + && store.GetPubKeyByXOnly(spenddata.internal_key, pub)) { + return pub.GetID(); + } + } return CKeyID(); } diff --git a/src/shutdown.h b/src/shutdown.h index ff56c6bd87..9df601d478 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -32,4 +32,4 @@ bool ShutdownRequested(); */ void WaitForShutdown(); -#endif +#endif // BITCOIN_SHUTDOWN_H diff --git a/src/span.h b/src/span.h index 830164514b..746e41f2f4 100644 --- a/src/span.h +++ b/src/span.h @@ -30,7 +30,11 @@ /** A Span is an object that can refer to a contiguous sequence of objects. * - * It implements a subset of C++20's std::span. + * This file implements a subset of C++20's std::span. It can be considered + * temporary compatibility code until C++20 and is designed to be a + * self-contained abstraction without depending on other project files. For this + * reason, Clang lifetimebound is defined here instead of including + * , which also defines it. * * Things to be aware of when writing code that deals with Spans: * @@ -60,7 +64,7 @@ * types that expose a data() and size() member function), functions that * accept a Span as input parameter can be called with any compatible * range-like object. For example, this works: -* + * * void Foo(Span arg); * * Foo(std::vector{1, 2, 3}); // Works @@ -180,6 +184,7 @@ class Span return m_data[m_size - 1]; } constexpr std::size_t size() const noexcept { return m_size; } + constexpr std::size_t size_bytes() const noexcept { return sizeof(C) * m_size; } constexpr bool empty() const noexcept { return size() == 0; } CONSTEXPR_IF_NOT_DEBUG C& operator[](std::size_t pos) const noexcept { @@ -236,11 +241,35 @@ T& SpanPopBack(Span& span) return back; } +// From C++20 as_bytes and as_writeable_bytes +template +Span AsBytes(Span s) noexcept +{ + return {reinterpret_cast(s.data()), s.size_bytes()}; +} +template +Span AsWritableBytes(Span s) noexcept +{ + return {reinterpret_cast(s.data()), s.size_bytes()}; +} + +template +Span MakeByteSpan(V&& v) noexcept +{ + return AsBytes(MakeSpan(std::forward(v))); +} +template +Span MakeWritableByteSpan(V&& v) noexcept +{ + return AsWritableBytes(MakeSpan(std::forward(v))); +} + // Helper functions to safely cast to unsigned char pointers. inline unsigned char* UCharCast(char* c) { return (unsigned char*)c; } inline unsigned char* UCharCast(unsigned char* c) { return c; } inline const unsigned char* UCharCast(const char* c) { return (unsigned char*)c; } inline const unsigned char* UCharCast(const unsigned char* c) { return c; } +inline const unsigned char* UCharCast(const std::byte* c) { return reinterpret_cast(c); } // Helper function to safely convert a Span to a Span<[const] unsigned char>. template constexpr auto UCharSpanCast(Span s) -> Span::type> { return {UCharCast(s.data()), s.size()}; } @@ -248,4 +277,4 @@ template constexpr auto UCharSpanCast(Span s) -> Span constexpr auto MakeUCharSpan(V&& v) -> decltype(UCharSpanCast(MakeSpan(std::forward(v)))) { return UCharSpanCast(MakeSpan(std::forward(v))); } -#endif +#endif // BITCOIN_SPAN_H diff --git a/src/streams.h b/src/streams.h index 31407287ae..9e8f379cd2 100644 --- a/src/streams.h +++ b/src/streams.h @@ -226,7 +226,7 @@ class CDataStream : nType{nTypeIn}, nVersion{nVersionIn} {} - explicit CDataStream(Span sp, int nTypeIn, int nVersionIn) + explicit CDataStream(Span sp, int nTypeIn, int nVersionIn) : vch(sp.data(), sp.data() + sp.size()), nType{nTypeIn}, nVersion{nVersionIn} {} @@ -254,17 +254,17 @@ class CDataStream iterator end() { return vch.end(); } size_type size() const { return vch.size() - nReadPos; } bool empty() const { return vch.size() == nReadPos; } - void resize(size_type n, value_type c=0) { vch.resize(n + nReadPos, c); } + void resize(size_type n, value_type c = value_type{}) { vch.resize(n + nReadPos, c); } void reserve(size_type n) { vch.reserve(n + nReadPos); } const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const uint8_t x) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const uint8_t x) { vch.insert(it, n, x); } + iterator insert(iterator it, const value_type x) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const value_type x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } - void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) + void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) { if (last == first) return; assert(last - first > 0); @@ -278,7 +278,7 @@ class CDataStream vch.insert(it, first, last); } - void insert(iterator it, const char* first, const char* last) + void insert(iterator it, const value_type* first, const value_type* last) { if (last == first) return; assert(last - first > 0); diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 9d1dfd46f1..c5fce7bec0 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -23,6 +23,16 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) BOOST_CHECK_EQUAL(strDec, vstrIn[i]); } + { + const std::vector in_u{0xff, 0x01, 0xff}; + const std::vector in_b{std::byte{0xff}, std::byte{0x01}, std::byte{0xff}}; + const std::string in_s{"\xff\x01\xff"}; + const std::string out_exp{"/wH/"}; + BOOST_CHECK_EQUAL(EncodeBase64(in_u), out_exp); + BOOST_CHECK_EQUAL(EncodeBase64(in_b), out_exp); + BOOST_CHECK_EQUAL(EncodeBase64(in_s), out_exp); + } + // Decoding strings with embedded NUL characters should fail bool failure; (void)DecodeBase64("invalid\0"s, &failure); diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index c0344b3cbb..8eed959228 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2017 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -68,10 +69,36 @@ BOOST_AUTO_TEST_CASE(bech32_testvectors_invalid) "1qzzfhee", "a12UEL5L", "A12uEL5L", + "abcdef1qpzrz9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", }; + static const std::pair ERRORS[] = { + {"Invalid character or mixed case", 0}, + {"Invalid character or mixed case", 0}, + {"Invalid character or mixed case", 0}, + {"Bech32 string too long", 90}, + {"Missing separator", -1}, + {"Invalid separator position", 0}, + {"Invalid Base 32 character", 2}, + {"Invalid separator position", 2}, + {"Invalid character or mixed case", 8}, + {"Invalid checksum", -1}, // The checksum is calculated using the uppercase form so the entire string is invalid, not just a few characters + {"Invalid separator position", 0}, + {"Invalid separator position", 0}, + {"Invalid character or mixed case", 3}, + {"Invalid character or mixed case", 3}, + {"Invalid checksum", 11} + }; + int i = 0; for (const std::string& str : CASES) { + const auto& err = ERRORS[i]; const auto dec = bech32::Decode(str); BOOST_CHECK(dec.encoding == bech32::Encoding::INVALID); + std::vector error_locations; + std::string error = bech32::LocateErrors(str, error_locations); + BOOST_CHECK_EQUAL(err.first, error); + if (err.second == -1) BOOST_CHECK(error_locations.empty()); + else BOOST_CHECK_EQUAL(err.second, error_locations[0]); + i++; } } @@ -91,11 +118,37 @@ BOOST_AUTO_TEST_CASE(bech32m_testvectors_invalid) "au1s5cgom", "M1VUXWEZ", "16plkw9", - "1p2gdwpf" + "1p2gdwpf", + "abcdef1l7aum6echk45nj2s0wdvt2fg8x9yrzpqzd3ryx", + }; + static const std::pair ERRORS[] = { + {"Invalid character or mixed case", 0}, + {"Invalid character or mixed case", 0}, + {"Invalid character or mixed case", 0}, + {"Bech32 string too long", 90}, + {"Missing separator", -1}, + {"Invalid separator position", 0}, + {"Invalid Base 32 character", 2}, + {"Invalid Base 32 character", 3}, + {"Invalid separator position", 2}, + {"Invalid Base 32 character", 8}, + {"Invalid Base 32 character", 7}, + {"Invalid checksum", -1}, + {"Invalid separator position", 0}, + {"Invalid separator position", 0}, + {"Invalid checksum", 21}, }; + int i = 0; for (const std::string& str : CASES) { + const auto& err = ERRORS[i]; const auto dec = bech32::Decode(str); BOOST_CHECK(dec.encoding == bech32::Encoding::INVALID); + std::vector error_locations; + std::string error = bech32::LocateErrors(str, error_locations); + BOOST_CHECK_EQUAL(err.first, error); + if (err.second == -1) BOOST_CHECK(error_locations.empty()); + else BOOST_CHECK_EQUAL(err.second, error_locations[0]); + i++; } } diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 2eb653e9ec..7d50def509 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include