diff --git a/CMakeLists.txt b/CMakeLists.txt index 7155f1b729623..2dca15e626e37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -431,6 +431,7 @@ set(UTIL_SOURCES ./src/fs.cpp ./src/logging.cpp ./src/random.cpp + ./src/randomenv.cpp ./src/rpc/protocol.cpp ./src/sync.cpp ./src/threadinterrupt.cpp diff --git a/build-aux/m4/l_socket.m4 b/build-aux/m4/l_socket.m4 new file mode 100644 index 0000000000000..38923a98fc966 --- /dev/null +++ b/build-aux/m4/l_socket.m4 @@ -0,0 +1,36 @@ +# Illumos/SmartOS requires linking with -lsocket if +# using getifaddrs & freeifaddrs + +m4_define([_CHECK_SOCKET_testbody], [[ + #include <sys/types.h> + #include <ifaddrs.h> + + int main() { + struct ifaddrs *ifaddr; + getifaddrs(&ifaddr); + freeifaddrs(ifaddr); + } +]]) + +AC_DEFUN([CHECK_SOCKET], [ + + AC_LANG_PUSH(C++) + + AC_MSG_CHECKING([whether ifaddrs funcs can be used without link library]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + LIBS="$LIBS -lsocket" + AC_MSG_CHECKING([whether getifaddrs needs -lsocket]) + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_FAILURE([cannot figure out how to use getifaddrs]) + ]) + ]) + + AC_LANG_POP +]) diff --git a/configure.ac b/configure.ac index f540693063088..24d322fbb089c 100644 --- a/configure.ac +++ b/configure.ac @@ -749,8 +749,12 @@ if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) +AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, + [#include <sys/types.h> + #include <ifaddrs.h>] +) AC_CHECK_DECLS([strnlen]) # Check for daemon(3), unrelated to --with-daemon (although used by it) @@ -888,6 +892,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> [ AC_MSG_RESULT(no)] ) +AC_MSG_CHECKING(for sysctl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> + #include <sys/sysctl.h>]], + [[ static const int name[2] = {CTL_KERN, KERN_VERSION}; + #ifdef __linux__ + #error "Don't use sysctl on Linux, it's deprecated even when it works" + #endif + sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #include <sys/sysctl.h>]], diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh new file mode 100755 index 0000000000000..3695b62af0fd3 --- /dev/null +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Test for deterministic coverage across unit test runs. + +export LC_ALL=C + +# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. +# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. +GCOV_EXECUTABLE="gcov" + +# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. +NON_DETERMINISTIC_TESTS=( + "dummy_tests" # We currently don't have any unit tests that have determinism issues +) + +TEST_BITCOIN_BINARY="src/test/test_pivx" + +print_usage() { + echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" +} + +N_TEST_RUNS=2 +BOOST_TEST_RUN_FILTERS="" +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ [a-z] ]]; then + BOOST_TEST_RUN_FILTERS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi +if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then + BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" +else + echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" + echo +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +if ! command -v gcovr > /dev/null; then + echo "Error: gcovr not installed. Exiting." + exit 1 +fi + +if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + exit 1 +fi + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + exit 1 +fi + +get_covr_filename() { + echo "gcovr.run-$1.txt" +} + +TEST_RUN_ID=0 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + find src/ -type f -name "*.gcda" -exec rm {} \; + if [[ $(get_file_suffix_count gcda) != 0 ]]; then + echo "Error: Stale *.gcda files found. Exiting." + exit 1 + fi + TEST_OUTPUT_TEMPFILE=$(mktemp) + if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then + cat "${TEST_OUTPUT_TEMPFILE}" + rm "${TEST_OUTPUT_TEMPFILE}" + exit 1 + fi + rm "${TEST_OUTPUT_TEMPFILE}" + if [[ $(get_file_suffix_count gcda) == 0 ]]; then + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + exit 1 + fi + GCOVR_TEMPFILE=$(mktemp) + if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then + echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." + exit 1 + fi + GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) + mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" + if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then + echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." + exit 1 + fi + if [[ ${TEST_RUN_ID} != 1 ]]; then + COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "${COVERAGE_DIFF}" + exit 1 + fi + rm "${GCOVR_FILENAME}" + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit diff --git a/src/Makefile.am b/src/Makefile.am index 94dd80a921093..ec0037e22ded5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -165,6 +165,7 @@ BITCOIN_CORE_H = \ coins.h \ compat.h \ compat/byteswap.h \ + compat/cpuid.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ @@ -239,6 +240,7 @@ BITCOIN_CORE_H = \ protocol.h \ pubkey.h \ random.h \ + randomenv.h \ reverselock.h \ reverse_iterate.h \ rpc/client.h \ @@ -538,6 +540,7 @@ libbitcoin_util_a_SOURCES = \ interfaces/handler.cpp \ logging.cpp \ random.cpp \ + randomenv.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ support/lockedpool.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 4c450080319e7..59bd3d3b0a9dc 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -15,6 +15,7 @@ bench_bench_pivx_SOURCES = \ bench/base58.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ + bench/chacha20.cpp \ bench/crypto_hash.cpp \ bench/lockedpool.cpp \ bench/perf.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 3f9644e457de8..a63f71c90e9b4 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -219,7 +219,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // find a bucket it is in now - int nRnd = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); + int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucket = -1; for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; @@ -295,7 +295,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP int nFactor = 1; for (int n = 0; n < pinfo->nRefCount; n++) nFactor *= 2; - if (nFactor > 1 && (RandomInt(nFactor) != 0)) + if (nFactor > 1 && (insecure_rand.randrange(nFactor) != 0)) return false; } else { pinfo = Create(addr, source, &nId); @@ -359,12 +359,12 @@ CAddrInfo CAddrMan::Select_(bool newOnly) return CAddrInfo(); // Use a 50% chance for choosing between tried and new table entries. - if (!newOnly && (nTried > 0 && (nNew == 0 || RandomInt(2) == 0))) { + if (!newOnly && (nTried > 0 && (nNew == 0 || insecure_rand.randbool() == 0))) { // use a tried node double fChanceFactor = 1.0; while (1) { - int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); - int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); + int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -372,7 +372,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -380,8 +380,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // use a new node double fChanceFactor = 1.0; while (1) { - int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); + int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -389,7 +389,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -485,7 +485,7 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr) if (vAddr.size() >= nNodes) break; - int nRndPos = RandomInt(vRandom.size() - n) + n; + int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); assert(mapInfo.count(vRandom[n]) == 1); @@ -533,10 +533,6 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) info.nServices = nServices; } -int CAddrMan::RandomInt(int nMax){ - return GetRandInt(nMax); -} - void CAddrMan::ResolveCollisions_() { for (std::set<int>::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) { @@ -603,7 +599,7 @@ CAddrInfo CAddrMan::SelectTriedCollision_() std::set<int>::iterator it = m_tried_collisions.begin(); // Selects a random element from m_tried_collisions - std::advance(it, GetRandInt(m_tried_collisions.size())); + std::advance(it, insecure_rand.randrange(m_tried_collisions.size())); int id_new = *it; // If id_new not found in mapInfo remove it from m_tried_collisions diff --git a/src/addrman.h b/src/addrman.h index 3b1fc4e73aea7..8729166865477 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -269,9 +269,6 @@ class CAddrMan //! Return a random to-be-evicted tried table address. CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - //! Wraps GetRandInt to allow tests to override RandomInt and make it determinismistic. - virtual int RandomInt(int nMax); - #ifdef DEBUG_ADDRMAN //! Perform consistency check. Returns an error code or zero. int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -469,7 +466,7 @@ class CAddrMan { LOCK(cs); std::vector<int>().swap(vRandom); - nKey = GetRandHash(); + nKey = insecure_rand.rand256(); for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { vvNew[bucket][entry] = -1; diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp new file mode 100644 index 0000000000000..7c25adf9cbaf2 --- /dev/null +++ b/src/bench/chacha20.cpp @@ -0,0 +1,46 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <iostream> + +#include <bench/bench.h> +#include <hash.h> +#include <crypto/chacha20.h> + +/* Number of bytes to process per iteration */ +static const uint64_t BUFFER_SIZE_TINY = 64; +static const uint64_t BUFFER_SIZE_SMALL = 256; +static const uint64_t BUFFER_SIZE_LARGE = 1024*1024; + +static void CHACHA20(benchmark::State& state, size_t buffersize) +{ + std::vector<uint8_t> key(32,0); + ChaCha20 ctx(key.data(), key.size()); + ctx.SetIV(0); + ctx.Seek(0); + std::vector<uint8_t> in(buffersize,0); + std::vector<uint8_t> out(buffersize,0); + while (state.KeepRunning()) { + ctx.Crypt(in.data(), out.data(), in.size()); + } +} + +static void CHACHA20_64BYTES(benchmark::State& state) +{ + CHACHA20(state, BUFFER_SIZE_TINY); +} + +static void CHACHA20_256BYTES(benchmark::State& state) +{ + CHACHA20(state, BUFFER_SIZE_SMALL); +} + +static void CHACHA20_1MB(benchmark::State& state) +{ + CHACHA20(state, BUFFER_SIZE_LARGE); +} + +BENCHMARK(CHACHA20_64BYTES); +BENCHMARK(CHACHA20_256BYTES); +BENCHMARK(CHACHA20_1MB); diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h new file mode 100644 index 0000000000000..34ef9bb27f553 --- /dev/null +++ b/src/compat/cpuid.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_CPUID_H +#define BITCOIN_COMPAT_CPUID_H + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#define HAVE_GETCPUID + +#include <cpuid.h> + +// We can't use cpuid.h's __get_cpuid as it does not support subleafs. +void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + +#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#endif // BITCOIN_COMPAT_CPUID_H diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 70c472b74b9d8..b147c7e2322b5 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -71,7 +71,7 @@ void ChaCha20::Seek(uint64_t pos) input[13] = pos >> 32; } -void ChaCha20::Output(unsigned char* c, size_t bytes) +void ChaCha20::Keystream(unsigned char* c, size_t bytes) { uint32_t x[16]; uint32_t j[16]; @@ -126,3 +126,133 @@ void ChaCha20::Output(unsigned char* c, size_t bytes) c += 64; } } + +void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = nullptr; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + // if m has fewer than 64 bytes available, copy m to tmp and + // read from tmp instead + for (i = 0;i < bytes;++i) tmp[i] = m[i]; + m = tmp; + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + x0 ^= ReadLE32(m + 0); + x1 ^= ReadLE32(m + 4); + x2 ^= ReadLE32(m + 8); + x3 ^= ReadLE32(m + 12); + x4 ^= ReadLE32(m + 16); + x5 ^= ReadLE32(m + 20); + x6 ^= ReadLE32(m + 24); + x7 ^= ReadLE32(m + 28); + x8 ^= ReadLE32(m + 32); + x9 ^= ReadLE32(m + 36); + x10 ^= ReadLE32(m + 40); + x11 ^= ReadLE32(m + 44); + x12 ^= ReadLE32(m + 48); + x13 ^= ReadLE32(m + 52); + x14 ^= ReadLE32(m + 56); + x15 ^= ReadLE32(m + 60); + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + m += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index a305977bcd5f4..5a4674f4a8afe 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -8,7 +8,8 @@ #include <stdint.h> #include <stdlib.h> -/** A PRNG class for ChaCha20. */ +/** A class for ChaCha20 256-bit stream cipher developed by Daniel J. Bernstein + https://cr.yp.to/chacha/chacha-20080128.pdf */ class ChaCha20 { private: @@ -17,10 +18,17 @@ class ChaCha20 public: ChaCha20(); ChaCha20(const unsigned char* key, size_t keylen); - void SetKey(const unsigned char* key, size_t keylen); - void SetIV(uint64_t iv); - void Seek(uint64_t pos); - void Output(unsigned char* output, size_t bytes); + void SetKey(const unsigned char* key, size_t keylen); //!< set key with flexible keylength; 256bit recommended */ + void SetIV(uint64_t iv); // set the 64bit nonce + void Seek(uint64_t pos); // set the 64bit block counter + + /** outputs the keystream of size <bytes> into <c> */ + void Keystream(unsigned char* c, size_t bytes); + + /** enciphers the message <input> of length <bytes> and write the enciphered representation into <output> + * Used for encryption and decryption (XOR) + */ + void Crypt(const unsigned char* input, unsigned char* output, size_t bytes); }; #endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index 5566d5db3ef09..404f8e6632cb8 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -17,7 +17,7 @@ class CSHA512 size_t bytes; public: - static const size_t OUTPUT_SIZE = 64; + static constexpr size_t OUTPUT_SIZE = 64; CSHA512(); CSHA512& Write(const unsigned char* data, size_t len); diff --git a/src/init.cpp b/src/init.cpp index 9c4b89cc64bd1..caab0cdf5b333 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1254,6 +1254,11 @@ bool AppInitMain() CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + // Gather some entropy once per minute. + scheduler.scheduleEvery([]{ + RandAddPeriodic(); + }, 60000); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); GetMainSignals().RegisterWithMempoolSignals(mempool); diff --git a/src/libzerocoin/bignum_gmp.cpp b/src/libzerocoin/bignum_gmp.cpp index 6282dd9c3a28a..dee024cfc2d15 100644 --- a/src/libzerocoin/bignum_gmp.cpp +++ b/src/libzerocoin/bignum_gmp.cpp @@ -63,8 +63,7 @@ CBigNum CBigNum::randBignum(const CBigNum& range) size_t size = (mpz_sizeinbase (range.bn, 2) + CHAR_BIT-1) / CHAR_BIT; std::vector<unsigned char> buf(size); - RandAddSeed(); - GetRandBytes(buf.data(), size); + GetStrongRandBytes(buf.data(), size); CBigNum ret(buf); if (ret < 0) @@ -80,8 +79,7 @@ CBigNum CBigNum::randKBitBignum(const uint32_t k) { std::vector<unsigned char> buf((k+7)/8); - RandAddSeed(); - GetRandBytes(buf.data(), (k+7)/8); + GetStrongRandBytes(buf.data(), (k+7)/8); CBigNum ret(buf); if (ret < 0) diff --git a/src/net.cpp b/src/net.cpp index ba077a41bfe5d..e865b241d91e2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -21,6 +21,7 @@ #include "netbase.h" #include "netmessagemaker.h" #include "primitives/transaction.h" +#include "random.h" #include "scheduler.h" #include "validation.h" @@ -30,6 +31,10 @@ #include <fcntl.h> #endif +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS +#include <ifaddrs.h> +#endif + #ifdef USE_UPNP #include <miniupnpc/miniupnpc.h> #include <miniupnpc/upnpcommands.h> @@ -125,11 +130,12 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6>& vSeedsIn const int64_t nOneWeek = 7 * 24 * 60 * 60; std::vector<CAddress> vSeedsOut; vSeedsOut.reserve(vSeedsIn.size()); + FastRandomContext rng; for (std::vector<SeedSpec6>::const_iterator i(vSeedsIn.begin()); i != vSeedsIn.end(); ++i) { struct in6_addr ip; memcpy(&ip, i->addr, sizeof(ip)); CAddress addr(CService(ip, i->port), NODE_NETWORK); - addr.nTime = GetTime() - GetRand(nOneWeek) - nOneWeek; + addr.nTime = GetTime() - rng.randrange(nOneWeek) - nOneWeek; vSeedsOut.push_back(addr); } return vSeedsOut; @@ -174,14 +180,14 @@ void AdvertiseLocal(CNode* pnode) // If discovery is enabled, sometimes give our peer the address it // tells us that it sees us as in case it has a better idea of our // address than we do. + FastRandomContext rng; if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || - GetRand((GetnScore(addrLocal) > LOCAL_MANUAL) ? 8 : 2) == 0)) { + rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) { addrLocal.SetIP(pnode->GetAddrLocal()); } if (addrLocal.IsRoutable()) { LogPrintf("%s: advertising address %s\n", __func__, addrLocal.ToString()); - FastRandomContext insecure_rand; - pnode->PushAddress(addrLocal, insecure_rand); + pnode->PushAddress(addrLocal, rng); } } } @@ -378,6 +384,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char* pszDest, bool fCo pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); + // We're making a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); + return pnode; } else if (!proxyConnectionFailed) { // If connecting to the node failed, and failure is not caused by a problem connecting to @@ -1066,6 +1075,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { LOCK(cs_vNodes); vNodes.push_back(pnode); } + + // We received a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); } void CConnman::ThreadSocketHandler() @@ -1999,7 +2011,7 @@ void Discover() } } } -#else +#elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) // Get local host ip struct ifaddrs* myaddrs; if (getifaddrs(&myaddrs) == 0) { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 49c898edf2b61..33941f3681004 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -553,9 +553,10 @@ void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { unsigned int nEvicted = 0; + FastRandomContext rng; while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = GetRandHash(); + uint256 randomhash = rng.rand256(); std::map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 082f5587bf86d..bdad740f7d2c4 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -7,14 +7,11 @@ #if defined(Q_OS_WIN) #include "init.h" -#include "util.h" #include <windows.h> #include <QDebug> -#include <openssl/rand.h> - // If we don't want a message to be processed by Qt, return true and set result to // the value that the window procedure should return. Otherwise return false. bool WinShutdownMonitor::nativeEventFilter(const QByteArray& eventType, void* pMessage, long* pnResult) @@ -23,16 +20,6 @@ bool WinShutdownMonitor::nativeEventFilter(const QByteArray& eventType, void* pM MSG* pMsg = static_cast<MSG*>(pMessage); - // Seed OpenSSL PRNG with Windows event data (e.g. mouse movements and other user interactions) - if (RAND_event(pMsg->message, pMsg->wParam, pMsg->lParam) == 0) { - // Warn only once as this is performance-critical - static bool warned = false; - if (!warned) { - LogPrintf("%s: OpenSSL RAND_event() failed to seed OpenSSL PRNG with enough data.\n", __func__); - warned = true; - } - } - switch (pMsg->message) { case WM_QUERYENDSESSION: { // Initiate a client shutdown after receiving a WM_QUERYENDSESSION and block diff --git a/src/random.cpp b/src/random.cpp index d7ed086110ce0..c033403df2723 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -6,21 +6,28 @@ #include "random.h" +#include "compat/cpuid.h" +#include "crypto/sha256.h" #include "crypto/sha512.h" #include "support/cleanse.h" +#include "support/allocators/secure.h" #ifdef WIN32 #include "compat.h" // for Windows API #include <wincrypt.h> #endif -#include "util.h" // for LogPrint() -#include "utilstrencodings.h" // for GetTime() +#include "logging.h" // for LogPrint() +#include "sync.h" // for Mutex +#include "utiltime.h" // for GetTime() #include <stdlib.h> #include <limits> #include <chrono> #include <thread> +#include "randomenv.h" + #ifndef WIN32 +#include <fcntl.h> #include <sys/time.h> #endif @@ -35,18 +42,15 @@ #include <sys/random.h> #endif #ifdef HAVE_SYSCTL_ARND +#include "utilstrencodings.h" #include <sys/sysctl.h> #endif #include <mutex> -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -#include <atomic> -#include <cpuid.h> -#endif - #include <openssl/err.h> #include <openssl/rand.h> +#include <openssl/conf.h> [[noreturn]] static void RandFailure() { @@ -54,7 +58,7 @@ std::abort(); } -static inline int64_t GetPerformanceCounter() +static inline int64_t GetPerformanceCounter() noexcept { // Read the hardware time stamp counter when available. // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. @@ -74,109 +78,185 @@ static inline int64_t GetPerformanceCounter() #endif } - -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -static std::atomic<bool> hwrand_initialized{false}; -static bool rdrand_supported = false; +#ifdef HAVE_GETCPUID +static bool g_rdrand_supported = false; +static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; -static void RDRandInit() +static constexpr uint32_t CPUID_F7_EBX_RDSEED = 0x00040000; +#ifdef bit_RDRND +static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND"); +#endif +#ifdef bit_RDSEED +static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); +#endif + +static void InitHardwareRand() { uint32_t eax, ebx, ecx, edx; - if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { + GetCPUID(1, 0, eax, ebx, ecx, edx); + if (ecx & CPUID_F1_ECX_RDRAND) { + g_rdrand_supported = true; + } + GetCPUID(7, 0, eax, ebx, ecx, edx); + if (ebx & CPUID_F7_EBX_RDSEED) { + g_rdseed_supported = true; + } +} + +static void ReportHardwareRand() +{ + // This must be done in a separate function, as InitHardwareRand() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rdseed_supported) { + LogPrintf("Using RdSeed as additional entropy source\n"); + } + if (g_rdrand_supported) { LogPrintf("Using RdRand as an additional entropy source\n"); - rdrand_supported = true; } - hwrand_initialized.store(true); } -#else -static void RDRandInit() {} -#endif -static bool GetHWRand(unsigned char* ent32) { -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) - assert(hwrand_initialized.load(std::memory_order_relaxed)); - if (rdrand_supported) { - uint8_t ok; - // Not all assemblers support the rdrand instruction, write it in hex. +/** Read 64 bits of entropy using rdrand. + * + * Must only be called when RdRand is supported. + */ +static uint64_t GetRdRand() noexcept +{ + // RdRand may very rarely fail. Invoke it up to 10 times in a loop to reduce this risk. #ifdef __i386__ - for (int iter = 0; iter < 4; ++iter) { - uint32_t r1, r2; - __asm__ volatile (".byte 0x0f, 0xc7, 0xf0;" // rdrand %eax - ".byte 0x0f, 0xc7, 0xf2;" // rdrand %edx - "setc %2" : - "=a"(r1), "=d"(r2), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE32(ent32 + 8 * iter, r1); - WriteLE32(ent32 + 8 * iter + 4, r2); - } -#else - uint64_t r1, r2, r3, r4; - __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0, " // rdrand %rax - "0x48, 0x0f, 0xc7, 0xf3, " // rdrand %rbx - "0x48, 0x0f, 0xc7, 0xf1, " // rdrand %rcx - "0x48, 0x0f, 0xc7, 0xf2; " // rdrand %rdx - "setc %4" : - "=a"(r1), "=b"(r2), "=c"(r3), "=d"(r4), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE64(ent32, r1); - WriteLE64(ent32 + 8, r2); - WriteLE64(ent32 + 16, r3); - WriteLE64(ent32 + 24, r4); -#endif - return true; + uint8_t ok; + uint32_t r1, r2; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %rax + if (ok) break; } + return r1; +#else +#error "RdRand is only supported on x86 and x86_64" #endif - return false; } -void RandAddSeed() +/** Read 64 bits of entropy using rdseed. + * + * Must only be called when RdSeed is supported. + */ +static uint64_t GetRdSeed() noexcept { - // Seed with CPU performance counter - int64_t nCounter = GetPerformanceCounter(); - RAND_add(&nCounter, sizeof(nCounter), 1.5); - memory_cleanse((void*)&nCounter, sizeof(nCounter)); + // RdSeed may fail when the HW RNG is overloaded. Loop indefinitely until enough entropy is gathered, + // but pause after every failure. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + do { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %rax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return r1; +#else +#error "RdSeed is only supported on x86 and x86_64" +#endif } -static void RandAddSeedPerfmon() -{ - RandAddSeed(); +#else +/* Access to other hardware random number generators could be added here later, + * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). + * Slower sources should probably be invoked separately, and/or only from + * RandAddPeriodic (which is called once a minute). + */ +static void InitHardwareRand() {} +static void ReportHardwareRand() {} +#endif -#ifdef WIN32 - // Don't need this on Linux, OpenSSL automatically uses /dev/urandom - // Seed with the entire set of perfmon data +/** Add 64 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareFast(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + if (g_rdrand_supported) { + uint64_t out = GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } +#endif +} - // This can take up to 2 seconds, so only do it every 10 minutes - static int64_t nLastPerfmon; - if (GetTime() < nLastPerfmon + 10 * 60) +/** Add 256 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareSlow(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + // When we want 256 bits of entropy, prefer RdSeed over RdRand, as it's + // guaranteed to produce independent randomness on every call. + if (g_rdseed_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRdSeed(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } return; - nLastPerfmon = GetTime(); - - std::vector<unsigned char> vData(250000, 0); - long ret = 0; - unsigned long nSize = 0; - const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data - while (true) { - nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", NULL, NULL, vData.data(), &nSize); - if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) - break; - vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially } - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) { - RAND_add(vData.data(), nSize, nSize / 100.0); - memory_cleanse(vData.data(), nSize); - LogPrint(BCLog::RAND, "%s: %lu bytes\n", __func__, nSize); - } else { - static bool warned = false; // Warn only once - if (!warned) { - LogPrintf("%s: Warning: RegQueryValueExA(HKEY_PERFORMANCE_DATA) failed with code %i\n", __func__, ret); - warned = true; + // When falling back to RdRand, XOR the result of 1024 results. + // This guarantees a reseeding occurs between each. + if (g_rdrand_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = 0; + for (int j = 0; j < 1024; ++j) out ^= GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); } + return; } #endif } +/** Use repeated SHA512 to strengthen the randomness in seed32, and feed into hasher. */ +static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA512& hasher) noexcept +{ + CSHA512 inner_hasher; + inner_hasher.Write(seed, sizeof(seed)); + + // Hash loop + unsigned char buffer[64]; + int64_t stop = GetTimeMicros() + microseconds; + do { + for (int i = 0; i < 1000; ++i) { + inner_hasher.Finalize(buffer); + inner_hasher.Reset(); + inner_hasher.Write(buffer, sizeof(buffer)); + } + // Benchmark operation and feed it into outer hasher. + int64_t perf = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perf, sizeof(perf)); + } while (GetTimeMicros() < stop); + + // Produce output from inner state and feed it to outer hasher. + inner_hasher.Finalize(buffer); + hasher.Write(buffer, sizeof(buffer)); + // Try to clean up. + inner_hasher.Reset(); + memory_cleanse(buffer, sizeof(buffer)); +} + #ifndef WIN32 /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. @@ -272,106 +352,297 @@ void GetOSRand(unsigned char *ent32) #endif } -void GetRandBytes(unsigned char* buf, int num) +void LockingCallbackOpenSSL(int mode, int i, const char* file, int line); + +namespace { + +class RNGState { + Mutex m_mutex; + /* The RNG state consists of 256 bits of entropy, taken from the output of + * one operation's SHA512 output, and fed as input to the next one. + * Carrying 256 bits of entropy should be sufficient to guarantee + * unpredictability as long as any entropy source was ever unpredictable + * to an attacker. To protect against situations where an attacker might + * observe the RNG's state, fresh entropy is always mixed when + * GetStrongRandBytes is called. + */ + unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; + uint64_t m_counter GUARDED_BY(m_mutex) = 0; + bool m_strongly_seeded GUARDED_BY(m_mutex) = false; + std::unique_ptr<Mutex[]> m_mutex_openssl; + + Mutex m_events_mutex; + CSHA256 m_events_hasher GUARDED_BY(m_events_mutex); + +public: + RNGState() noexcept + { + InitHardwareRand(); + + // Init OpenSSL library multithreading support + m_mutex_openssl.reset(new Mutex[CRYPTO_num_locks()]); + CRYPTO_set_locking_callback(LockingCallbackOpenSSL); + + // OpenSSL can optionally load a config file which lists optional loadable modules and engines. + // We don't use them so we don't require the config. However some of our libs may call functions + // which attempt to load the config file, possibly resulting in an exit() or crash if it is missing + // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be + // that the config appears to have been loaded and there are no modules/engines available. + OPENSSL_no_config(); + } + + ~RNGState() + { + // Securely erase the memory used by the OpenSSL PRNG + RAND_cleanup(); + // Shutdown OpenSSL library multithreading support + CRYPTO_set_locking_callback(nullptr); + } + + void AddEvent(uint32_t event_info) noexcept + { + LOCK(m_events_mutex); + + m_events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); + // Get the low four bytes of the performance counter. This translates to roughly the + // subsecond part. + uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); + m_events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); + } + + /** + * Feed (the hash of) all events added through AddEvent() to hasher. + */ + void SeedEvents(CSHA512& hasher) noexcept + { + // We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, + // since we want it to be fast as network peers may be able to trigger it repeatedly. + LOCK(m_events_mutex); + + unsigned char events_hash[32]; + m_events_hasher.Finalize(events_hash); + hasher.Write(events_hash, 32); + + // Re-initialize the hasher with the finalized state to use later. + m_events_hasher.Reset(); + m_events_hasher.Write(events_hash, 32); + } + + /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. + * + * If this function has never been called with strong_seed = true, false is returned. + */ + bool MixExtract(unsigned char* out, size_t num, CSHA512&& hasher, bool strong_seed) noexcept + { + assert(num <= 32); + unsigned char buf[64]; + static_assert(sizeof(buf) == CSHA512::OUTPUT_SIZE, "Buffer needs to have hasher's output size"); + bool ret; + { + LOCK(m_mutex); + ret = (m_strongly_seeded |= strong_seed); + // Write the current state of the RNG into the hasher + hasher.Write(m_state, 32); + // Write a new counter number into the state + hasher.Write((const unsigned char*)&m_counter, sizeof(m_counter)); + ++m_counter; + // Finalize the hasher + hasher.Finalize(buf); + // Store the last 32 bytes of the hash output as new RNG state. + memcpy(m_state, buf + 32, 32); + } + // If desired, copy (up to) the first 32 bytes of the hash output as output. + if (num) { + assert(out != nullptr); + memcpy(out, buf, num); + } + // Best effort cleanup of internal state + hasher.Reset(); + memory_cleanse(buf, 64); + return ret; + } + + Mutex& GetOpenSSLMutex(int i) { return m_mutex_openssl[i]; } +}; + +RNGState& GetRNGState() noexcept +{ + // This C++11 idiom relies on the guarantee that static variable are initialized + // on first call, even when multiple parallel calls are permitted. + static std::vector<RNGState, secure_allocator<RNGState>> g_rng(1); + return g_rng[0]; +} +} + +void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS { - if (RAND_bytes(buf, num) != 1) { - RandFailure(); + RNGState& rng = GetRNGState(); + + if (mode & CRYPTO_LOCK) { + rng.GetOpenSSLMutex(i).lock(); + } else { + rng.GetOpenSSLMutex(i).unlock(); } } -static void AddDataToRng(void* data, size_t len); +/* A note on the use of noexcept in the seeding functions below: + * + * None of the RNG code should ever throw any exception. + */ -void RandAddSeedSleep() +static void SeedTimestamp(CSHA512& hasher) noexcept { - int64_t nPerfCounter1 = GetPerformanceCounter(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - int64_t nPerfCounter2 = GetPerformanceCounter(); + int64_t perfcounter = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); +} - // Combine with and update state - AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1)); - AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2)); +static void SeedFast(CSHA512& hasher) noexcept +{ + unsigned char buffer[32]; + + // Stack pointer to indirectly commit to thread/callstack + const unsigned char* ptr = buffer; + hasher.Write((const unsigned char*)&ptr, sizeof(ptr)); - memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); - memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); + // Hardware randomness is very fast when available; use it always. + SeedHardwareFast(hasher); + + // High-precision timestamp + SeedTimestamp(hasher); } +static void SeedSlow(CSHA512& hasher, RNGState& rng) noexcept +{ + unsigned char buffer[32]; -static std::mutex cs_rng_state; -static unsigned char rng_state[32] = {0}; -static uint64_t rng_counter = 0; + // Everything that the 'fast' seeder includes + SeedFast(hasher); -static void AddDataToRng(void* data, size_t len) { - CSHA512 hasher; - hasher.Write((const unsigned char*)&len, sizeof(len)); - hasher.Write((const unsigned char*)data, len); - unsigned char buf[64]; - { - std::unique_lock<std::mutex> lock(cs_rng_state); - hasher.Write(rng_state, sizeof(rng_state)); - hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); - ++rng_counter; - hasher.Finalize(buf); - memcpy(rng_state, buf + 32, 32); - } - memory_cleanse(buf, 64); + // OS randomness + GetOSRand(buffer); + hasher.Write(buffer, sizeof(buffer)); + + // OpenSSL RNG (for now) + RAND_bytes(buffer, sizeof(buffer)); + hasher.Write(buffer, sizeof(buffer)); + + // Add the events hasher into the mix + rng.SeedEvents(hasher); + + // High-precision timestamp. + // + // Note that we also commit to a timestamp in the Fast seeder, so we indirectly commit to a + // benchmark of all the entropy gathering sources in this function). + SeedTimestamp(hasher); } -void GetStrongRandBytes(unsigned char* out, int num) +/** Extract entropy from rng, strengthen it, and feed it into hasher. */ +static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noexcept { - assert(num <= 32); - CSHA512 hasher; - unsigned char buf[64]; + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen the seed, and feed it into hasher. + Strengthen(strengthen_seed, microseconds, hasher); +} + +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept +{ + // Everything that the 'fast' seeder includes + SeedFast(hasher); + + // High-precision timestamp + SeedTimestamp(hasher); + + // Add the events hasher into the mix + rng.SeedEvents(hasher); + + // Dynamic environment data (performance monitoring, ...) + RandAddDynamicEnv(hasher); + + // Strengthen for 10 ms + SeedStrengthen(hasher, rng, 10000); +} + +static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept +{ + // Gather 256 bits of hardware randomness, if available + SeedHardwareSlow(hasher); - // First source: OpenSSL's RNG - RandAddSeedPerfmon(); - GetRandBytes(buf, 32); - hasher.Write(buf, 32); + // Everything that the 'slow' seeder includes. + SeedSlow(hasher, rng); - // Second source: OS RNG - GetOSRand(buf); - hasher.Write(buf, 32); + // Dynamic environment data (performance monitoring, ...) + RandAddDynamicEnv(hasher); - // Third source: HW RNG, if available. - if (GetHWRand(buf)) { - hasher.Write(buf, 32); + // Static environment data + RandAddStaticEnv(hasher); + + // Strengthen for 100 ms + SeedStrengthen(hasher, rng, 100000); +} + +enum class RNGLevel { + FAST, //!< Automatically called by GetRandBytes + SLOW, //!< Automatically called by GetStrongRandBytes + PERIODIC, //!< Called by RandAddPeriodic() +}; + +static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept +{ + // Make sure the RNG is initialized first (as all Seed* function possibly need hwrand to be available). + RNGState& rng = GetRNGState(); + + assert(num <= 32); + + CSHA512 hasher; + switch (level) { + case RNGLevel::FAST: + SeedFast(hasher); + break; + case RNGLevel::SLOW: + SeedSlow(hasher, rng); + break; + case RNGLevel::PERIODIC: + SeedPeriodic(hasher, rng); + break; } // Combine with and update state - { - std::unique_lock<std::mutex> lock(cs_rng_state); - hasher.Write(rng_state, sizeof(rng_state)); - hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); - ++rng_counter; - hasher.Finalize(buf); - memcpy(rng_state, buf + 32, 32); + if (!rng.MixExtract(out, num, std::move(hasher), false)) { + // On the first invocation, also seed with SeedStartup(). + CSHA512 startup_hasher; + SeedStartup(startup_hasher, rng); + rng.MixExtract(out, num, std::move(startup_hasher), true); } - // Produce output - memcpy(out, buf, num); - memory_cleanse(buf, 64); + // For anything but the 'fast' level, feed the resulting RNG output (after an additional hashing step) back into OpenSSL. + if (level != RNGLevel::FAST) { + unsigned char buf[64]; + CSHA512().Write(out, num).Finalize(buf); + RAND_add(buf, sizeof(buf), num); + memory_cleanse(buf, 64); + } } -uint64_t GetRand(uint64_t nMax) -{ - if (nMax == 0) - return 0; +void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } +void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } +void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } - // The range of the random source must be a multiple of the modulus - // to give every possible output value an equal possibility - uint64_t nRange = (std::numeric_limits<uint64_t>::max() / nMax) * nMax; - uint64_t nRand = 0; - do { - GetRandBytes((unsigned char*)&nRand, sizeof(nRand)); - } while (nRand >= nRange); - return (nRand % nMax); +bool g_mock_deterministic_tests{false}; + +uint64_t GetRand(uint64_t nMax) noexcept +{ + return FastRandomContext(g_mock_deterministic_tests).randrange(nMax); } -int GetRandInt(int nMax) +int GetRandInt(int nMax) noexcept { return GetRand(nMax); } -uint256 GetRandHash() +uint256 GetRandHash() noexcept { uint256 hash; GetRandBytes((unsigned char*)&hash, sizeof(hash)); @@ -385,7 +656,7 @@ void FastRandomContext::RandomSeed() requires_seed = false; } -uint256 FastRandomContext::rand256() +uint256 FastRandomContext::rand256() noexcept { if (bytebuf_size < 32) { FillByteBuffer(); @@ -401,12 +672,12 @@ std::vector<unsigned char> FastRandomContext::randbytes(size_t len) if (requires_seed) RandomSeed(); std::vector<unsigned char> ret(len); if (len > 0) { - rng.Output(&ret[0], len); + rng.Keystream(&ret[0], len); } return ret; } -FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +FastRandomContext::FastRandomContext(const uint256& seed) noexcept : requires_seed(false), bytebuf_size(0), bitbuf_size(0) { rng.SetKey(seed.begin(), 32); } @@ -416,7 +687,7 @@ bool Random_SanityCheck() uint64_t start = GetPerformanceCounter(); /* This does not measure the quality of randomness, but it does test that - * OSRandom() overwrites all 32 bytes of the output given a maximum + * GetOSRand() overwrites all 32 bytes of the output given a maximum * number of tries. */ static const ssize_t MAX_TRIES = 1024; @@ -449,13 +720,15 @@ bool Random_SanityCheck() if (stop == start) return false; // We called GetPerformanceCounter. Use it as entropy. - RAND_add((const unsigned char*)&start, sizeof(start), 1); - RAND_add((const unsigned char*)&stop, sizeof(stop), 1); + CSHA512 to_add; + to_add.Write((const unsigned char*)&start, sizeof(start)); + to_add.Write((const unsigned char*)&stop, sizeof(stop)); + GetRNGState().MixExtract(nullptr, 0, std::move(to_add), false); return true; } -FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +FastRandomContext::FastRandomContext(bool fDeterministic) noexcept : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) { if (!fDeterministic) { return; @@ -480,5 +753,8 @@ FastRandomContext& FastRandomContext::operator=(FastRandomContext&& from) noexce void RandomInit() { - RDRandInit(); + // Invoke RNG code to trigger initialization (if not already performed) + ProcRand(nullptr, 0, RNGLevel::FAST); + + ReportHardwareRand(); } diff --git a/src/random.h b/src/random.h index e953e72198337..632221bdae681 100644 --- a/src/random.h +++ b/src/random.h @@ -12,34 +12,95 @@ #include "uint256.h" #include <stdint.h> +#include <limits> -/* Seed OpenSSL PRNG with additional entropy data */ -void RandAddSeed(); +/** + * Overall design of the RNG and entropy sources. + * + * We maintain a single global 256-bit RNG state for all high-quality randomness. + * The following (classes of) functions interact with that state by mixing in new + * entropy, and optionally extracting random output from it: + * + * - The GetRand*() class of functions, as well as construction of FastRandomContext objects, + * perform 'fast' seeding, consisting of mixing in: + * - A stack pointer (indirectly committing to calling thread and call stack) + * - A high-precision timestamp (rdtsc when available, c++ high_resolution_clock otherwise) + * - 64 bits from the hardware RNG (rdrand) when available. + * These entropy sources are very fast, and only designed to protect against situations + * where a VM state restore/copy results in multiple systems with the same randomness. + * FastRandomContext on the other hand does not protect against this once created, but + * is even faster (and acceptable to use inside tight loops). + * + * - The GetStrongRand*() class of function perform 'slow' seeding, including everything + * that fast seeding includes, but additionally: + * - OS entropy (/dev/urandom, getrandom(), ...). The application will terminate if + * this entropy source fails. + * - Bytes from OpenSSL's RNG (which itself may be seeded from various sources) + * - Another high-precision timestamp (indirectly committing to a benchmark of all the + * previous sources). + * These entropy sources are slower, but designed to make sure the RNG state contains + * fresh data that is unpredictable to attackers. + * + * - RandAddPeriodic() seeds everything that fast seeding includes, but additionally: + * - A high-precision timestamp + * - Dynamic environment data (performance monitoring, ...) + * - Strengthen the entropy for 10 ms using repeated SHA512. + * This is run once every minute. + * + * On first use of the RNG (regardless of what function is called first), all entropy + * sources used in the 'slow' seeder are included, but also: + * - 256 bits from the hardware RNG (rdseed or rdrand) when available. + * - Dynamic environment data (performance monitoring, ...) + * - Static environment data + * - Strengthen the entropy for 100 ms using repeated SHA512. + * + * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and + * (up to) the first 32 bytes of H are produced as output, while the last 32 bytes + * become the new RNG state. +*/ + +/** + * Generate random data via the internal PRNG. + * + * These functions are designed to be fast (sub microsecond), but do not necessarily + * meaningfully add entropy to the PRNG state. + * + * Thread-safe. + */ +void GetRandBytes(unsigned char* buf, int num) noexcept; +uint64_t GetRand(uint64_t nMax) noexcept; +int GetRandInt(int nMax) noexcept; +uint256 GetRandHash() noexcept; /** - * Functions to gather random data via the OpenSSL PRNG + * Gather entropy from various sources, feed it into the internal PRNG, and + * generate random data using it. + * + * This function will cause failure whenever the OS RNG fails. + * + * Thread-safe. */ -void GetRandBytes(unsigned char* buf, int num); -uint64_t GetRand(uint64_t nMax); -int GetRandInt(int nMax); -uint256 GetRandHash(); +void GetStrongRandBytes(unsigned char* buf, int num) noexcept; /** - * Add a little bit of randomness to the output of GetStrongRangBytes. - * This sleeps for a millisecond, so should only be called when there is - * no other work to be done. + * Gather entropy from various expensive sources, and feed them to the PRNG state. + * + * Thread-safe. */ -void RandAddSeedSleep(); +void RandAddPeriodic() noexcept; /** - * Function to gather random data from multiple sources, failing whenever any - * of those source fail to provide a result. + * Gathers entropy from the low bits of the time at which events occur. Should + * be called with a uint32_t describing the event at the time an event occurs. + * + * Thread-safe. */ -void GetStrongRandBytes(unsigned char* buf, int num); +void RandAddEvent(const uint32_t event_info) noexcept; /** * Fast randomness source. This is seeded once with secure random data, but - * is completely deterministic and insecure after that. + * is completely deterministic and does not gather more entropy after that. + * * This class is not thread-safe. */ class FastRandomContext { @@ -60,7 +121,7 @@ class FastRandomContext { if (requires_seed) { RandomSeed(); } - rng.Output(bytebuf, sizeof(bytebuf)); + rng.Keystream(bytebuf, sizeof(bytebuf)); bytebuf_size = sizeof(bytebuf); } @@ -71,10 +132,10 @@ class FastRandomContext { } public: - explicit FastRandomContext(bool fDeterministic = false); + explicit FastRandomContext(bool fDeterministic = false) noexcept; /** Initialize with explicit seed (only for testing) */ - explicit FastRandomContext(const uint256& seed); + explicit FastRandomContext(const uint256& seed) noexcept; // Do not permit copying a FastRandomContext (move it, or create a new one to get reseeded). FastRandomContext(const FastRandomContext&) = delete; @@ -85,7 +146,7 @@ class FastRandomContext { FastRandomContext& operator=(FastRandomContext&& from) noexcept; /** Generate a random 64-bit integer. */ - uint64_t rand64() + uint64_t rand64() noexcept { if (bytebuf_size < 8) FillByteBuffer(); uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); @@ -94,7 +155,7 @@ class FastRandomContext { } /** Generate a random (bits)-bit integer. */ - uint64_t randbits(int bits) { + uint64_t randbits(int bits) noexcept { if (bits == 0) { return 0; } else if (bits > 32) { @@ -109,7 +170,7 @@ class FastRandomContext { } /** Generate a random integer in the range [0..range). */ - uint64_t randrange(uint64_t range) + uint64_t randrange(uint64_t range) noexcept { --range; int bits = CountBits(range); @@ -123,15 +184,44 @@ class FastRandomContext { std::vector<unsigned char> randbytes(size_t len); /** Generate a random 32-bit integer. */ - uint32_t rand32() { return randbits(32); } + uint32_t rand32() noexcept { return randbits(32); } /** generate a random uint256. */ - uint256 rand256(); + uint256 rand256() noexcept; /** Generate a random boolean. */ - bool randbool() { return randbits(1); } + bool randbool() noexcept { return randbits(1); } + + // Compatibility with the C++11 UniformRandomBitGenerator concept + typedef uint64_t result_type; + static constexpr uint64_t min() { return 0; } + static constexpr uint64_t max() { return std::numeric_limits<uint64_t>::max(); } + inline uint64_t operator()() noexcept { return rand64(); } }; +/** More efficient than using std::shuffle on a FastRandomContext. + * + * This is more efficient as std::shuffle will consume entropy in groups of + * 64 bits at the time and throw away most. + * + * This also works around a bug in libstdc++ std::shuffle that may cause + * type::operator=(type&&) to be invoked on itself, which the library's + * debug mode detects and panics on. This is a known issue, see + * https://stackoverflow.com/questions/22915325/avoiding-self-assignment-in-stdshuffle + */ +template<typename I, typename R> +void Shuffle(I first, I last, R&& rng) +{ + while (first != last) { + size_t j = rng.randrange(last - first); + if (j) { + using std::swap; + swap(*first, *(first + j)); + } + ++first; + } +} + /* Number of random bytes returned by GetOSRand. * When changing this constant make sure to change all call sites, and make * sure that the underlying OS APIs for all platforms support the number. @@ -149,7 +239,12 @@ void GetOSRand(unsigned char *ent32); */ bool Random_SanityCheck(); -/** Initialize the RNG. */ +/** + * Initialize global RNG state and log any CPU features that are used. + * + * Calling this function is optional. RNG state will be initialized when first + * needed if it is not called. + */ void RandomInit(); #endif // BITCOIN_RANDOM_H diff --git a/src/randomenv.cpp b/src/randomenv.cpp new file mode 100644 index 0000000000000..eed4db22c0195 --- /dev/null +++ b/src/randomenv.cpp @@ -0,0 +1,517 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include "config/pivx-config.h" +#endif + +#include "randomenv.h" + +#include "clientversion.h" +#include "compat/cpuid.h" +#include "crypto/sha512.h" +#include "support/cleanse.h" +#include "utiltime.h" // for GetTime() +#ifdef WIN32 +#include "compat.h" // for Windows API +#endif + +#include <algorithm> +#include <chrono> +#include <climits> +#include <thread> +#include <vector> + +#include <stdint.h> +#include <string.h> +#ifndef WIN32 +#include <sys/types.h> // must go before a number of other headers +#include <fcntl.h> +#include <netinet/in.h> +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/utsname.h> +#include <unistd.h> +#endif +#ifdef __MACH__ +#include <mach/clock.h> +#include <mach/mach.h> +#include <mach/mach_time.h> +#endif +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS +#include <ifaddrs.h> +#endif +#if HAVE_SYSCTL +#include <sys/sysctl.h> +#if HAVE_VM_VM_PARAM_H +#include <vm/vm_param.h> +#endif +#if HAVE_SYS_RESOURCES_H +#include <sys/resources.h> +#endif +#if HAVE_SYS_VMMETER_H +#include <sys/vmmeter.h> +#endif +#endif +#ifdef __linux__ +#include <sys/auxv.h> +#endif + +//! Necessary on some platforms +extern char** environ; + +namespace { + +void RandAddSeedPerfmon(CSHA512& hasher) +{ +#ifdef WIN32 + // Don't need this on Linux, OpenSSL automatically uses /dev/urandom + // Seed with the entire set of perfmon data + + // This can take up to 2 seconds, so only do it every 10 minutes + static int64_t nLastPerfmon; + if (GetTime() < nLastPerfmon + 10 * 60) + return; + nLastPerfmon = GetTime(); + + std::vector<unsigned char> vData(250000, 0); + long ret = 0; + unsigned long nSize = 0; + const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data + while (true) { + nSize = vData.size(); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); + if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) + break; + vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially + } + RegCloseKey(HKEY_PERFORMANCE_DATA); + if (ret == ERROR_SUCCESS) { + hasher.Write(vData.data(), nSize); + memory_cleanse(vData.data(), nSize); + } else { + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. + } +#endif +} + +/** Helper to easily feed data into a CSHA512. + * + * Note that this does not serialize the passed object (like stream.h's << operators do). + * Its raw memory representation is used directly. + */ +template<typename T> +CSHA512& operator<<(CSHA512& hasher, const T& data) { + static_assert(!std::is_same<typename std::decay<T>::type, char*>::value, "Calling operator<<(CSHA512, char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, unsigned char*>::value, "Calling operator<<(CSHA512, unsigned char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const char*>::value, "Calling operator<<(CSHA512, const char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const unsigned char*>::value, "Calling operator<<(CSHA512, const unsigned char*) is probably not what you want"); + hasher.Write((const unsigned char*)&data, sizeof(data)); + return hasher; +} + +#ifndef WIN32 +void AddSockaddr(CSHA512& hasher, const struct sockaddr *addr) +{ + if (addr == nullptr) return; + switch (addr->sa_family) { + case AF_INET: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in)); + break; + case AF_INET6: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in6)); + break; + default: + hasher.Write((const unsigned char*)&addr->sa_family, sizeof(addr->sa_family)); + } +} + +void AddFile(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + int f = open(path, O_RDONLY); + size_t total = 0; + if (f != -1) { + unsigned char fbuf[4096]; + int n; + hasher.Write((const unsigned char*)&f, sizeof(f)); + if (fstat(f, &sb) == 0) hasher << sb; + do { + n = read(f, fbuf, sizeof(fbuf)); + if (n > 0) hasher.Write(fbuf, n); + total += n; + /* not bothering with EINTR handling. */ + } while (n == sizeof(fbuf) && total < 1048576); // Read only the first 1 Mbyte + close(f); + } +} + +void AddPath(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + if (stat(path, &sb) == 0) { + hasher.Write((const unsigned char*)path, strlen(path) + 1); + hasher << sb; + } +} +#endif + +#if HAVE_SYSCTL +template<int... S> +void AddSysctl(CSHA512& hasher) +{ + int CTL[sizeof...(S)] = {S...}; + unsigned char buffer[65536]; + size_t siz = 65536; + int ret = sysctl(CTL, sizeof...(S), buffer, &siz, nullptr, 0); + if (ret == 0 || (ret == -1 && errno == ENOMEM)) { + hasher << sizeof(CTL); + hasher.Write((const unsigned char*)CTL, sizeof(CTL)); + if (siz > sizeof(buffer)) siz = sizeof(buffer); + hasher << siz; + hasher.Write(buffer, siz); + } +} +#endif + +#ifdef HAVE_GETCPUID +void inline AddCPUID(CSHA512& hasher, uint32_t leaf, uint32_t subleaf, uint32_t& ax, uint32_t& bx, uint32_t& cx, uint32_t& dx) +{ + GetCPUID(leaf, subleaf, ax, bx, cx, dx); + hasher << leaf << subleaf << ax << bx << cx << dx; +} + +void AddAllCPUID(CSHA512& hasher) +{ + uint32_t ax, bx, cx, dx; + // Iterate over all standard leaves + AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax + uint32_t max = ax; + for (uint32_t leaf = 1; leaf <= max && leaf <= 0xFF; ++leaf) { + uint32_t maxsub = 0; + for (uint32_t subleaf = 0; subleaf <= 0xFF; ++subleaf) { + AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); + // Iterate subleafs for leaf values 4, 7, 11, 13 + if (leaf == 4) { + if ((ax & 0x1f) == 0) break; + } else if (leaf == 7) { + if (subleaf == 0) maxsub = ax; + if (subleaf == maxsub) break; + } else if (leaf == 11) { + if ((cx & 0xff00) == 0) break; + } else if (leaf == 13) { + if (ax == 0 && bx == 0 && cx == 0 && dx == 0) break; + } else { + // For any other leaf, stop after subleaf 0. + break; + } + } + } + // Iterate over all extended leaves + AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax + uint32_t ext_max = ax; + for (uint32_t leaf = 0x80000001; leaf <= ext_max && leaf <= 0x800000FF; ++leaf) { + AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); + } +} +#endif +} // namespace + +void RandAddDynamicEnv(CSHA512& hasher) +{ + RandAddSeedPerfmon(hasher); + + // Various clocks +#ifdef WIN32 + FILETIME ftime; + GetSystemTimeAsFileTime(&ftime); + hasher << ftime; +#else +# ifndef __MACH__ + // On non-MacOS systems, use various clock_gettime() calls. + struct timespec ts = {}; +# ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &ts); + hasher << ts; +# endif +# ifdef CLOCK_REALTIME + clock_gettime(CLOCK_REALTIME, &ts); + hasher << ts; +# endif +# ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &ts); + hasher << ts; +# endif +# else + // On MacOS use mach_absolute_time (number of CPU ticks since boot) as a replacement for CLOCK_MONOTONIC, + // and clock_get_time for CALENDAR_CLOCK as a replacement for CLOCK_REALTIME. + hasher << mach_absolute_time(); + // From https://gist.github.com/jbenet/1087739 + clock_serv_t cclock; + mach_timespec_t mts = {}; + if (host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock) == KERN_SUCCESS && clock_get_time(cclock, &mts) == KERN_SUCCESS) { + hasher << mts; + mach_port_deallocate(mach_task_self(), cclock); + } +# endif + // gettimeofday is available on all UNIX systems, but only has microsecond precision. + struct timeval tv = {}; + gettimeofday(&tv, nullptr); + hasher << tv; +#endif + // Probably redundant, but also use all the clocks C++11 provides: + hasher << std::chrono::system_clock::now().time_since_epoch().count(); + hasher << std::chrono::steady_clock::now().time_since_epoch().count(); + hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); + +#ifndef WIN32 + // Current resource usage. + struct rusage usage = {}; + if (getrusage(RUSAGE_SELF, &usage) == 0) hasher << usage; +#endif + +#ifdef __linux__ + AddFile(hasher, "/proc/diskstats"); + AddFile(hasher, "/proc/vmstat"); + AddFile(hasher, "/proc/schedstat"); + AddFile(hasher, "/proc/zoneinfo"); + AddFile(hasher, "/proc/meminfo"); + AddFile(hasher, "/proc/softirqs"); + AddFile(hasher, "/proc/stat"); + AddFile(hasher, "/proc/self/schedstat"); + AddFile(hasher, "/proc/self/status"); +#endif + +#if HAVE_SYSCTL +# ifdef CTL_KERN +# if defined(KERN_PROC) && defined(KERN_PROC_ALL) + AddSysctl<CTL_KERN, KERN_PROC, KERN_PROC_ALL>(hasher); +# endif +# endif +# ifdef CTL_HW +# ifdef HW_DISKSTATS + AddSysctl<CTL_HW, HW_DISKSTATS>(hasher); +# endif +# endif +# ifdef CTL_VM +# ifdef VM_LOADAVG + AddSysctl<CTL_VM, VM_LOADAVG>(hasher); +# endif +# ifdef VM_TOTAL + AddSysctl<CTL_VM, VM_TOTAL>(hasher); +# endif +# ifdef VM_METER + AddSysctl<CTL_VM, VM_METER>(hasher); +# endif +# endif +#endif + + // Stack and heap location + void* addr = malloc(4097); + hasher << &addr << addr; + free(addr); +} + +void RandAddStaticEnv(CSHA512& hasher) +{ + // Some compile-time static properties + hasher << (CHAR_MIN < 0) << sizeof(void*) << sizeof(long) << sizeof(int); +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + hasher << __GNUC__ << __GNUC_MINOR__ << __GNUC_PATCHLEVEL__; +#endif +#ifdef _MSC_VER + hasher << _MSC_VER; +#endif + hasher << __cplusplus; +#ifdef _XOPEN_VERSION + hasher << _XOPEN_VERSION; +#endif +#ifdef __VERSION__ + const char* COMPILER_VERSION = __VERSION__; + hasher.Write((const unsigned char*)COMPILER_VERSION, strlen(COMPILER_VERSION) + 1); +#endif + + // Bitcoin client version + hasher << CLIENT_VERSION; + +#ifdef __linux__ + // Information available through getauxval() +# ifdef AT_HWCAP + hasher << getauxval(AT_HWCAP); +# endif +# ifdef AT_HWCAP2 + hasher << getauxval(AT_HWCAP2); +# endif +# ifdef AT_RANDOM + const unsigned char* random_aux = (const unsigned char*)getauxval(AT_RANDOM); + if (random_aux) hasher.Write(random_aux, 16); +# endif +# ifdef AT_PLATFORM + const char* platform_str = (const char*)getauxval(AT_PLATFORM); + if (platform_str) hasher.Write((const unsigned char*)platform_str, strlen(platform_str) + 1); +# endif +# ifdef AT_EXECFN + const char* exec_str = (const char*)getauxval(AT_EXECFN); + if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); +# endif +#endif // __linux__ + +#ifdef HAVE_GETCPUID + AddAllCPUID(hasher); +#endif + + // Memory locations + hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; + + // Hostname + char hname[256]; + if (gethostname(hname, 256) == 0) { + hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); + } + +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS + // Network interfaces + struct ifaddrs *ifad = NULL; + getifaddrs(&ifad); + struct ifaddrs *ifit = ifad; + while (ifit != NULL) { + hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); + hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); + hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); + AddSockaddr(hasher, ifit->ifa_addr); + AddSockaddr(hasher, ifit->ifa_netmask); + AddSockaddr(hasher, ifit->ifa_dstaddr); + ifit = ifit->ifa_next; + } + freeifaddrs(ifad); +#endif + +#ifndef WIN32 + // UNIX kernel information + struct utsname name; + if (uname(&name) != -1) { + hasher.Write((const unsigned char*)&name.sysname, strlen(name.sysname) + 1); + hasher.Write((const unsigned char*)&name.nodename, strlen(name.nodename) + 1); + hasher.Write((const unsigned char*)&name.release, strlen(name.release) + 1); + hasher.Write((const unsigned char*)&name.version, strlen(name.version) + 1); + hasher.Write((const unsigned char*)&name.machine, strlen(name.machine) + 1); + } + + /* Path and filesystem provided data */ + AddPath(hasher, "/"); + AddPath(hasher, "."); + AddPath(hasher, "/tmp"); + AddPath(hasher, "/home"); + AddPath(hasher, "/proc"); +#ifdef __linux__ + AddFile(hasher, "/proc/cmdline"); + AddFile(hasher, "/proc/cpuinfo"); + AddFile(hasher, "/proc/version"); +#endif + AddFile(hasher, "/etc/passwd"); + AddFile(hasher, "/etc/group"); + AddFile(hasher, "/etc/hosts"); + AddFile(hasher, "/etc/resolv.conf"); + AddFile(hasher, "/etc/timezone"); + AddFile(hasher, "/etc/localtime"); +#endif + + // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these + // will exist on every system. +#if HAVE_SYSCTL +# ifdef CTL_HW +# ifdef HW_MACHINE + AddSysctl<CTL_HW, HW_MACHINE>(hasher); +# endif +# ifdef HW_MODEL + AddSysctl<CTL_HW, HW_MODEL>(hasher); +# endif +# ifdef HW_NCPU + AddSysctl<CTL_HW, HW_NCPU>(hasher); +# endif +# ifdef HW_PHYSMEM + AddSysctl<CTL_HW, HW_PHYSMEM>(hasher); +# endif +# ifdef HW_USERMEM + AddSysctl<CTL_HW, HW_USERMEM>(hasher); +# endif +# ifdef HW_MACHINE_ARCH + AddSysctl<CTL_HW, HW_MACHINE_ARCH>(hasher); +# endif +# ifdef HW_REALMEM + AddSysctl<CTL_HW, HW_REALMEM>(hasher); +# endif +# ifdef HW_CPU_FREQ + AddSysctl<CTL_HW, HW_CPU_FREQ>(hasher); +# endif +# ifdef HW_BUS_FREQ + AddSysctl<CTL_HW, HW_BUS_FREQ>(hasher); +# endif +# ifdef HW_CACHELINE + AddSysctl<CTL_HW, HW_CACHELINE>(hasher); +# endif +# endif +# ifdef CTL_KERN +# ifdef KERN_BOOTFILE + AddSysctl<CTL_KERN, KERN_BOOTFILE>(hasher); +# endif +# ifdef KERN_BOOTTIME + AddSysctl<CTL_KERN, KERN_BOOTTIME>(hasher); +# endif +# ifdef KERN_CLOCKRATE + AddSysctl<CTL_KERN, KERN_CLOCKRATE>(hasher); +# endif +# ifdef KERN_HOSTID + AddSysctl<CTL_KERN, KERN_HOSTID>(hasher); +# endif +# ifdef KERN_HOSTUUID + AddSysctl<CTL_KERN, KERN_HOSTUUID>(hasher); +# endif +# ifdef KERN_HOSTNAME + AddSysctl<CTL_KERN, KERN_HOSTNAME>(hasher); +# endif +# ifdef KERN_OSRELDATE + AddSysctl<CTL_KERN, KERN_OSRELDATE>(hasher); +# endif +# ifdef KERN_OSRELEASE + AddSysctl<CTL_KERN, KERN_OSRELEASE>(hasher); +# endif +# ifdef KERN_OSREV + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_OSTYPE + AddSysctl<CTL_KERN, KERN_OSTYPE>(hasher); +# endif +# ifdef KERN_POSIX1 + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_VERSION + AddSysctl<CTL_KERN, KERN_VERSION>(hasher); +# endif +# endif +#endif + + // Env variables + if (environ) { + for (size_t i = 0; environ[i]; ++i) { + hasher.Write((const unsigned char*)environ[i], strlen(environ[i])); + } + } + + // Process, thread, user, session, group, ... ids. +#ifdef WIN32 + hasher << GetCurrentProcessId() << GetCurrentThreadId(); +#else + hasher << getpid() << getppid() << getsid(0) << getpgid(0) << getuid() << geteuid() << getgid() << getegid(); +#endif + hasher << std::this_thread::get_id(); +} diff --git a/src/randomenv.h b/src/randomenv.h new file mode 100644 index 0000000000000..6c49d3deede07 --- /dev/null +++ b/src/randomenv.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOMENV_H +#define BITCOIN_RANDOMENV_H + +#include "crypto/sha512.h" + +/** Gather non-cryptographic environment data that changes over time. */ +void RandAddDynamicEnv(CSHA512& hasher); + +/** Gather non-cryptographic environment data that does not change over time. */ +void RandAddStaticEnv(CSHA512& hasher); + +#endif diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 86c08510ce455..ac52c43588ac6 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -32,8 +32,6 @@ void CScheduler::serviceQueue() try { if (!shouldStop() && taskQueue.empty()) { reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); - // Use this chance to get a tiny bit more entropy - RandAddSeedSleep(); } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. diff --git a/src/sync.cpp b/src/sync.cpp index 01403b72d4f3f..bef4212a53405 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -5,15 +5,16 @@ #include "sync.h" -#include <memory> -#include <set> - -#include "util.h" +#include "logging.h" #include "utilstrencodings.h" #include "util/threadnames.h" #include <stdio.h> +#include <map> +#include <memory> +#include <set> + #ifdef DEBUG_LOCKCONTENTION #if !defined(HAVE_THREAD_LOCAL) static_assert(false, "thread_local is not supported"); diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index ac21d069ba70e..e3d0f01076ddf 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -9,9 +9,11 @@ #include "test/test_pivx.h" +#include "arith_uint256.h" #include "keystore.h" #include "net_processing.h" #include "net.h" +#include "pubkey.h" #include "pow.h" #include "script/sign.h" #include "serialize.h" @@ -139,10 +141,26 @@ CTransactionRef RandomOrphan() return it->second.tx; } +static void MakeNewKeyWithFastRandomContext(CKey& key) +{ + std::vector<unsigned char> keydata; + keydata = insecure_rand_ctx.randbytes(32); + key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn*/ true); + assert(key.IsValid()); +} + BOOST_AUTO_TEST_CASE(DoS_mapOrphans) { + // This test had non-deterministic coverage due to + // randomly selected seeds. + // This seed is chosen so that all branches of the function + // ecdsa_signature_parse_der_lax are executed during this test. + // Specifically branches that run only when an ECDSA + // signature's R and S values have leading zeros. + insecure_rand_ctx = FastRandomContext(ArithToUint256(arith_uint256(33))); + CKey key; - key.MakeNewKey(true); + MakeNewKeyWithFastRandomContext(key); CBasicKeyStore keystore; keystore.AddKey(key); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index bfe2f7d3c5152..e57f50927ecd3 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -29,12 +29,6 @@ class CAddrManTest : public CAddrMan insecure_rand = FastRandomContext(true); } - int RandomInt(int nMax) - { - state = ReadLE64((CHashWriter(SER_GETHASH, 0) << state).GetHash().begin()); - return (unsigned int)(state % nMax); - } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = NULL) { LOCK(cs); diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index d8646534f4c91..9b99d10fca7c4 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -463,6 +463,9 @@ static std::vector<unsigned char> RandomData() BOOST_AUTO_TEST_CASE(rolling_bloom) { + SeedInsecureRand(/* deterministic */ true); + g_mock_deterministic_tests = true; + // last-100-entry, 1% false positive: CRollingBloomFilter rb1(100, 0.01); @@ -487,12 +490,8 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) if (rb1.contains(RandomData())) ++nHits; } - // Run test_bitcoin with --log_level=message to see BOOST_TEST_MESSAGEs: - BOOST_TEST_MESSAGE("RollingBloomFilter got " << nHits << " false positives (~100 expected)"); - - // Insanely unlikely to get a fp count outside this range: - BOOST_CHECK(nHits > 25); - BOOST_CHECK(nHits < 175); + // Expect about 100 hits + BOOST_CHECK_EQUAL(nHits, 75); BOOST_CHECK(rb1.contains(data[DATASIZE - 1])); rb1.reset(); @@ -516,10 +515,8 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) if (rb1.contains(data[i])) ++nHits; } - // Expect about 5 false positives, more than 100 means - // something is definitely broken. - BOOST_TEST_MESSAGE("RollingBloomFilter got " << nHits << " false positives (~5 expected)"); - BOOST_CHECK(nHits < 100); + // Expect about 5 false positives + BOOST_CHECK_EQUAL(nHits, 6); // last-1000-entry, 0.01% false positive: CRollingBloomFilter rb2(1000, 0.001); @@ -530,6 +527,7 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) for (int i = 0; i < DATASIZE; i++) { BOOST_CHECK(rb2.contains(data[i])); } + g_mock_deterministic_tests = false; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index b34c1c03763aa..181e3ec4518c1 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -66,17 +66,36 @@ void TestHMACSHA512(const std::string &hexkey, const std::string &hexin, const s TestVector(CHMAC_SHA512(&key[0], key.size()), ParseHex(hexin), ParseHex(hexout)); } -void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) +void TestChaCha20(const std::string &hex_message, const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) { std::vector<unsigned char> key = ParseHex(hexkey); + std::vector<unsigned char> m = ParseHex(hex_message); ChaCha20 rng(key.data(), key.size()); rng.SetIV(nonce); rng.Seek(seek); std::vector<unsigned char> out = ParseHex(hexout); std::vector<unsigned char> outres; outres.resize(out.size()); - rng.Output(outres.data(), outres.size()); + assert(hex_message.empty() || m.size() == out.size()); + + // perform the ChaCha20 round(s), if message is provided it will output the encrypted ciphertext otherwise the keystream + if (!hex_message.empty()) { + rng.Crypt(m.data(), outres.data(), outres.size()); + } else { + rng.Keystream(outres.data(), outres.size()); + } BOOST_CHECK(out == outres); + if (!hex_message.empty()) { + // Manually XOR with the keystream and compare the output + rng.SetIV(nonce); + rng.Seek(seek); + std::vector<unsigned char> only_keystream(outres.size()); + rng.Keystream(only_keystream.data(), only_keystream.size()); + for (size_t i = 0; i != m.size(); i++) { + outres[i] = m[i] ^ only_keystream[i]; + } + BOOST_CHECK(out == outres); + } } void TestAES128(const std::string &hexkey, const std::string &hexin, const std::string &hexout) @@ -421,25 +440,37 @@ BOOST_AUTO_TEST_CASE(rfc6979_hmac_sha256) BOOST_AUTO_TEST_CASE(chacha20_testvector) { // Test vector from RFC 7539 - TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + + // test encryption + TestChaCha20("4c616469657320616e642047656e746c656d656e206f662074686520636c617373206f66202739393a204966204920636f756" + "c64206f6666657220796f75206f6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73637265656e" + "20776f756c642062652069742e", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + "6e2e359a2568f98041ba0728dd0d6981e97e7aec1d4360c20a27afccfd9fae0bf91b65c5524733ab8f593dabcd62b3571639d" + "624e65152ab8f530c359f0861d807ca0dbf500d6a6156a38e088a22b65e52bc514d16ccf806818ce91ab77937365af90bbf74" + "a35be6b40b8eedf2785e42874d" + ); + + // test keystream output + TestChaCha20("", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b9334794cb" "a40c63e34cdea212c4cf07d41b769a6749f3f630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53ac40c5945398b6eda1a" "832c89c167eacd901d7e2bf363"); // Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000000", 0, 0, "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b" "8f41518a11cc387b669b2ee6586"); - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000001", 0, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000001", 0, 0, "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d79" "2b1c43fea817e9ad275ae546963"); - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b52770" "62eb7a0433e445f41e3"); - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 1, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000000", 1, 0, "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc4" "97a0b466e7d6bbdb0041b2f586b"); - TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, + TestChaCha20("", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3b" "e59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc1" "18be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5" diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 278066b94f76a..ce532c96293d6 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -24,40 +24,25 @@ * using BOOST_CHECK_CLOSE to fail. * */ -FastRandomContext insecure_rand(true); BOOST_AUTO_TEST_SUITE(cuckoocache_tests); - -/** insecure_GetRandHash fills in a uint256 from insecure_rand - */ -void insecure_GetRandHash(uint256& t) -{ - uint32_t* ptr = (uint32_t*)t.begin(); - for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); -} - - - /* Test that no values not inserted into the cache are read out of it. * * There are no repeats in the first 200000 insecure_GetRandHash calls */ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); uint256 v; for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - cc.insert(v); + cc.insert(InsecureRand256()); } for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - BOOST_CHECK(!cc.contains(v, false)); + BOOST_CHECK(!cc.contains(InsecureRand256(), false)); } }; @@ -67,7 +52,7 @@ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) template <typename Cache> double test_cache(size_t megabytes, double load) { - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -77,7 +62,7 @@ double test_cache(size_t megabytes, double load) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -138,7 +123,7 @@ template <typename Cache> void test_cache_erase(size_t megabytes) { double load = 1; - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -148,7 +133,7 @@ void test_cache_erase(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -201,7 +186,7 @@ template <typename Cache> void test_cache_erase_parallel(size_t megabytes) { double load = 1; - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -211,7 +196,7 @@ void test_cache_erase_parallel(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -303,7 +288,7 @@ void test_cache_generations() // iterations with non-deterministic values, so it isn't "overfit" to the // specific entropy in FastRandomContext(true) and implementation of the // cache. - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); // block_activity models a chunk of network activity. n_insert elements are // adde to the cache. The first and last n/4 are stored for removal later @@ -320,7 +305,7 @@ void test_cache_generations() for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)inserts[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } for (uint32_t i = 0; i < n_insert / 4; ++i) reads.push_back(inserts[i]); diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 1627c08a9bbb4..fe0ea4b7ad476 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -8,6 +8,9 @@ #include <boost/test/unit_test.hpp> +#include <random> +#include <algorithm> + BOOST_FIXTURE_TEST_SUITE(random_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(osrandom_tests) @@ -18,9 +21,14 @@ BOOST_AUTO_TEST_CASE(osrandom_tests) BOOST_AUTO_TEST_CASE(fastrandom_tests) { // Check that deterministic FastRandomContexts are deterministic + g_mock_deterministic_tests = true; FastRandomContext ctx1(true); FastRandomContext ctx2(true); + for (int i = 10; i > 0; --i) { + BOOST_CHECK_EQUAL(GetRand(std::numeric_limits<uint64_t>::max()), uint64_t{10393729187455219830U}); + BOOST_CHECK_EQUAL(GetRandInt(std::numeric_limits<int>::max()), int{769702006}); + } BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); @@ -35,6 +43,11 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) BOOST_CHECK(ctx1.randbytes(50) == ctx2.randbytes(50)); // Check that a nondeterministic ones are not + g_mock_deterministic_tests = false; + for (int i = 10; i > 0; --i) { + BOOST_CHECK(GetRand(std::numeric_limits<uint64_t>::max()) != uint64_t{10393729187455219830U}); + BOOST_CHECK(GetRandInt(std::numeric_limits<int>::max()) != int{769702006}); + } { FastRandomContext ctx3, ctx4; BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal @@ -64,4 +77,57 @@ BOOST_AUTO_TEST_CASE(fastrandom_randbits) } } +/** Does-it-compile test for compatibility with standard C++11 RNG interface. */ +BOOST_AUTO_TEST_CASE(stdrandom_test) +{ + FastRandomContext ctx; + std::uniform_int_distribution<int> distribution(3, 9); + for (int i = 0; i < 100; ++i) { + int x = distribution(ctx); + BOOST_CHECK(x >= 3); + BOOST_CHECK(x <= 9); + + std::vector<int> test{1,2,3,4,5,6,7,8,9,10}; + std::shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } + Shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } + } + +} + +/** Test that Shuffle reaches every permutation with equal probability. */ +BOOST_AUTO_TEST_CASE(shuffle_stat_test) +{ + FastRandomContext ctx(true); + uint32_t counts[5 * 5 * 5 * 5 * 5] = {0}; + for (int i = 0; i < 12000; ++i) { + int data[5] = {0, 1, 2, 3, 4}; + Shuffle(std::begin(data), std::end(data), ctx); + int pos = data[0] + data[1] * 5 + data[2] * 25 + data[3] * 125 + data[4] * 625; + ++counts[pos]; + } + unsigned int sum = 0; + double chi_score = 0.0; + for (int i = 0; i < 5 * 5 * 5 * 5 * 5; ++i) { + int i1 = i % 5, i2 = (i / 5) % 5, i3 = (i / 25) % 5, i4 = (i / 125) % 5, i5 = i / 625; + uint32_t count = counts[i]; + if (i1 == i2 || i1 == i3 || i1 == i4 || i1 == i5 || i2 == i3 || i2 == i4 || i2 == i5 || i3 == i4 || i3 == i5 || i4 == i5) { + BOOST_CHECK(count == 0); + } else { + chi_score += ((count - 100.0) * (count - 100.0)) / 100.0; + BOOST_CHECK(count > 50); + BOOST_CHECK(count < 150); + sum += count; + } + } + BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval + BOOST_CHECK(chi_score < 210.275); + BOOST_CHECK_EQUAL(sum, 12000); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_pivx.cpp b/src/test/test_pivx.cpp index 05ca418990388..d5e0812c31e6d 100644 --- a/src/test/test_pivx.cpp +++ b/src/test/test_pivx.cpp @@ -12,7 +12,6 @@ #include "evo/evodb.h" #include "miner.h" #include "net_processing.h" -#include "random.h" #include "rpc/server.h" #include "rpc/register.h" #include "script/sigcache.h" @@ -27,8 +26,7 @@ std::unique_ptr<CConnman> g_connman; CClientUIInterface uiInterface; // Declared but not defined in guiinterface.h -uint256 insecure_rand_seed = GetRandHash(); -FastRandomContext insecure_rand_ctx(insecure_rand_seed); +FastRandomContext insecure_rand_ctx; extern bool fPrintToConsole; extern void noui_connect(); @@ -41,7 +39,6 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) BasicTestingSetup::BasicTestingSetup() { - RandomInit(); ECC_Start(); SetupEnvironment(); InitSignatureCache(); diff --git a/src/test/test_pivx.h b/src/test/test_pivx.h index 251c9ffb929df..42f1b499cac88 100644 --- a/src/test/test_pivx.h +++ b/src/test/test_pivx.h @@ -11,17 +11,16 @@ #include <boost/thread.hpp> -extern uint256 insecure_rand_seed; extern FastRandomContext insecure_rand_ctx; -static inline void SeedInsecureRand(bool fDeterministic = false) +/** + * Flag to make GetRand in random.h return the same number + */ +extern bool g_mock_deterministic_tests; + +static inline void SeedInsecureRand(bool deterministic = false) { - if (fDeterministic) { - insecure_rand_seed = uint256(); - } else { - insecure_rand_seed = GetRandHash(); - } - insecure_rand_ctx = FastRandomContext(insecure_rand_seed); + insecure_rand_ctx = FastRandomContext(deterministic); } static inline uint32_t InsecureRand32() { return insecure_rand_ctx.rand32(); } diff --git a/src/util.cpp b/src/util.cpp index c25216fc147fc..424897daeb421 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -79,9 +79,6 @@ #include <boost/program_options/detail/config_file.hpp> #include <boost/program_options/parsers.hpp> -#include <openssl/conf.h> -#include <openssl/crypto.h> -#include <openssl/rand.h> const char * const PIVX_CONF_FILENAME = "pivx.conf"; const char * const PIVX_PID_FILENAME = "pivx.pid"; @@ -101,57 +98,6 @@ ArgsManager gArgs; bool fDaemon = false; CTranslationInterface translationInterface; -/** Init OpenSSL library multithreading support */ -static RecursiveMutex** ppmutexOpenSSL; -void locking_callback(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS -{ - if (mode & CRYPTO_LOCK) { - ENTER_CRITICAL_SECTION(*ppmutexOpenSSL[i]); - } else { - LEAVE_CRITICAL_SECTION(*ppmutexOpenSSL[i]); - } -} - -// Init -class CInit -{ -public: - CInit() - { - // Init OpenSSL library multithreading support - ppmutexOpenSSL = (RecursiveMutex**)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(RecursiveMutex*)); - for (int i = 0; i < CRYPTO_num_locks(); i++) - ppmutexOpenSSL[i] = new RecursiveMutex(); - CRYPTO_set_locking_callback(locking_callback); - - // OpenSSL can optionally load a config file which lists optional loadable modules and engines. - // We don't use them so we don't require the config. However some of our libs may call functions - // which attempt to load the config file, possibly resulting in an exit() or crash if it is missing - // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be - // that the config appears to have been loaded and there are no modules/engines available. - OPENSSL_no_config(); - -#ifdef WIN32 - // Seed OpenSSL PRNG with current contents of the screen - RAND_screen(); -#endif - - // Seed OpenSSL PRNG with performance counter - RandAddSeed(); - } - ~CInit() - { - // Securely erase the memory used by the PRNG - RAND_cleanup(); - // Shutdown OpenSSL library multithreading support - CRYPTO_set_locking_callback(NULL); - for (int i = 0; i < CRYPTO_num_locks(); i++) - delete ppmutexOpenSSL[i]; - OPENSSL_free(ppmutexOpenSSL); - } -} instance_of_cinit; - - /** * Interpret a string argument as a boolean. * diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index beee18dc5e26d..2dc2ead0f8564 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2731,7 +2731,7 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int std::vector<std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> > > vValue; CAmount nTotalLower = 0; - random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); for (const COutput& output : vCoins) { if (!output.fSpendable)