From 94b2ead6baa6762afa376ed8ac20c6c48b2a1fad Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 20 Mar 2018 19:10:39 -0700 Subject: [PATCH 01/55] Make FastRandomContext support standard C++11 RNG interface This makes it possible to plug it into the various standard C++11 random distribution algorithms and other functions like std::shuffle. --- src/random.h | 7 +++++++ src/test/random_tests.cpp | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/random.h b/src/random.h index e953e72198337..4cc2582ce470b 100644 --- a/src/random.h +++ b/src/random.h @@ -12,6 +12,7 @@ #include "uint256.h" #include +#include /* Seed OpenSSL PRNG with additional entropy data */ void RandAddSeed(); @@ -130,6 +131,12 @@ class FastRandomContext { /** Generate a random boolean. */ bool randbool() { return randbits(1); } + + // Compatibility with the C++11 UniformRandomBitGenerator concept + typedef uint64_t result_type; + static constexpr uint64_t min() { return 0; } + static constexpr uint64_t max() { return std::numeric_limits::max(); } + inline uint64_t operator()() { return rand64(); } }; /* Number of random bytes returned by GetOSRand. diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 1627c08a9bbb4..d8f2489829fd1 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -8,6 +8,9 @@ #include +#include +#include + BOOST_FIXTURE_TEST_SUITE(random_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(osrandom_tests) @@ -64,4 +67,23 @@ BOOST_AUTO_TEST_CASE(fastrandom_randbits) } } +/** Does-it-compile test for compatibility with standard C++11 RNG interface. */ +BOOST_AUTO_TEST_CASE(stdrandom_test) +{ + FastRandomContext ctx; + std::uniform_int_distribution distribution(3, 9); + for (int i = 0; i < 100; ++i) { + int x = distribution(ctx); + BOOST_CHECK(x >= 3); + BOOST_CHECK(x <= 9); + + std::vector test{1,2,3,4,5,6,7,8,9,10}; + std::shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } + } + +} + BOOST_AUTO_TEST_SUITE_END() From e862564b8b642c1f60e7b18eceda3e2703000cb0 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sun, 28 Mar 2021 01:03:52 -0700 Subject: [PATCH 02/55] Make addrman use its local RNG exclusively --- src/addrman.cpp | 26 +++++++++++--------------- src/addrman.h | 5 +---- src/test/addrman_tests.cpp | 6 ------ 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/addrman.cpp b/src/addrman.cpp index 3f9644e457de8..a63f71c90e9b4 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -219,7 +219,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // find a bucket it is in now - int nRnd = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); + int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucket = -1; for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; @@ -295,7 +295,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP int nFactor = 1; for (int n = 0; n < pinfo->nRefCount; n++) nFactor *= 2; - if (nFactor > 1 && (RandomInt(nFactor) != 0)) + if (nFactor > 1 && (insecure_rand.randrange(nFactor) != 0)) return false; } else { pinfo = Create(addr, source, &nId); @@ -359,12 +359,12 @@ CAddrInfo CAddrMan::Select_(bool newOnly) return CAddrInfo(); // Use a 50% chance for choosing between tried and new table entries. - if (!newOnly && (nTried > 0 && (nNew == 0 || RandomInt(2) == 0))) { + if (!newOnly && (nTried > 0 && (nNew == 0 || insecure_rand.randbool() == 0))) { // use a tried node double fChanceFactor = 1.0; while (1) { - int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); - int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); + int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -372,7 +372,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -380,8 +380,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // use a new node double fChanceFactor = 1.0; while (1) { - int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); + int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -389,7 +389,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -485,7 +485,7 @@ void CAddrMan::GetAddr_(std::vector& vAddr) if (vAddr.size() >= nNodes) break; - int nRndPos = RandomInt(vRandom.size() - n) + n; + int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); assert(mapInfo.count(vRandom[n]) == 1); @@ -533,10 +533,6 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) info.nServices = nServices; } -int CAddrMan::RandomInt(int nMax){ - return GetRandInt(nMax); -} - void CAddrMan::ResolveCollisions_() { for (std::set::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) { @@ -603,7 +599,7 @@ CAddrInfo CAddrMan::SelectTriedCollision_() std::set::iterator it = m_tried_collisions.begin(); // Selects a random element from m_tried_collisions - std::advance(it, GetRandInt(m_tried_collisions.size())); + std::advance(it, insecure_rand.randrange(m_tried_collisions.size())); int id_new = *it; // If id_new not found in mapInfo remove it from m_tried_collisions diff --git a/src/addrman.h b/src/addrman.h index 3b1fc4e73aea7..8729166865477 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -269,9 +269,6 @@ class CAddrMan //! Return a random to-be-evicted tried table address. CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - //! Wraps GetRandInt to allow tests to override RandomInt and make it determinismistic. - virtual int RandomInt(int nMax); - #ifdef DEBUG_ADDRMAN //! Perform consistency check. Returns an error code or zero. int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -469,7 +466,7 @@ class CAddrMan { LOCK(cs); std::vector().swap(vRandom); - nKey = GetRandHash(); + nKey = insecure_rand.rand256(); for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { vvNew[bucket][entry] = -1; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index bfe2f7d3c5152..e57f50927ecd3 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -29,12 +29,6 @@ class CAddrManTest : public CAddrMan insecure_rand = FastRandomContext(true); } - int RandomInt(int nMax) - { - state = ReadLE64((CHashWriter(SER_GETHASH, 0) << state).GetHash().begin()); - return (unsigned int)(state % nMax); - } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = NULL) { LOCK(cs); From 1cdf1243f36f4abf7b14071e39efca7c2927e005 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sun, 28 Mar 2021 01:14:24 -0700 Subject: [PATCH 03/55] Use a local FastRandomContext in a few more places in net --- src/net.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index ba077a41bfe5d..c7bd86b4e0756 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -125,11 +125,12 @@ static std::vector convertSeed6(const std::vector& vSeedsIn const int64_t nOneWeek = 7 * 24 * 60 * 60; std::vector vSeedsOut; vSeedsOut.reserve(vSeedsIn.size()); + FastRandomContext rng; for (std::vector::const_iterator i(vSeedsIn.begin()); i != vSeedsIn.end(); ++i) { struct in6_addr ip; memcpy(&ip, i->addr, sizeof(ip)); CAddress addr(CService(ip, i->port), NODE_NETWORK); - addr.nTime = GetTime() - GetRand(nOneWeek) - nOneWeek; + addr.nTime = GetTime() - rng.randrange(nOneWeek) - nOneWeek; vSeedsOut.push_back(addr); } return vSeedsOut; @@ -174,14 +175,14 @@ void AdvertiseLocal(CNode* pnode) // If discovery is enabled, sometimes give our peer the address it // tells us that it sees us as in case it has a better idea of our // address than we do. + FastRandomContext rng; if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || - GetRand((GetnScore(addrLocal) > LOCAL_MANUAL) ? 8 : 2) == 0)) { + rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) { addrLocal.SetIP(pnode->GetAddrLocal()); } if (addrLocal.IsRoutable()) { LogPrintf("%s: advertising address %s\n", __func__, addrLocal.ToString()); - FastRandomContext insecure_rand; - pnode->PushAddress(addrLocal, insecure_rand); + pnode->PushAddress(addrLocal, rng); } } } From 746d466949cb772555558090e880994180df1d41 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sun, 28 Mar 2021 01:26:38 -0700 Subject: [PATCH 04/55] Introduce a Shuffle for FastRandomContext and use it in wallet --- src/random.h | 23 +++++++++++++++++++++++ src/test/random_tests.cpp | 34 ++++++++++++++++++++++++++++++++++ src/wallet/wallet.cpp | 2 +- 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/random.h b/src/random.h index 4cc2582ce470b..a1babfad424bc 100644 --- a/src/random.h +++ b/src/random.h @@ -139,6 +139,29 @@ class FastRandomContext { inline uint64_t operator()() { return rand64(); } }; +/** More efficient than using std::shuffle on a FastRandomContext. + * + * This is more efficient as std::shuffle will consume entropy in groups of + * 64 bits at the time and throw away most. + * + * This also works around a bug in libstdc++ std::shuffle that may cause + * type::operator=(type&&) to be invoked on itself, which the library's + * debug mode detects and panics on. This is a known issue, see + * https://stackoverflow.com/questions/22915325/avoiding-self-assignment-in-stdshuffle + */ +template +void Shuffle(I first, I last, R&& rng) +{ + while (first != last) { + size_t j = rng.randrange(last - first); + if (j) { + using std::swap; + swap(*first, *(first + j)); + } + ++first; + } +} + /* Number of random bytes returned by GetOSRand. * When changing this constant make sure to change all call sites, and make * sure that the underlying OS APIs for all platforms support the number. diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index d8f2489829fd1..424bffdeb9120 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -82,8 +82,42 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) for (int j = 1; j <= 10; ++j) { BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); } + Shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } } } +/** Test that Shuffle reaches every permutation with equal probability. */ +BOOST_AUTO_TEST_CASE(shuffle_stat_test) +{ + FastRandomContext ctx(true); + uint32_t counts[5 * 5 * 5 * 5 * 5] = {0}; + for (int i = 0; i < 12000; ++i) { + int data[5] = {0, 1, 2, 3, 4}; + Shuffle(std::begin(data), std::end(data), ctx); + int pos = data[0] + data[1] * 5 + data[2] * 25 + data[3] * 125 + data[4] * 625; + ++counts[pos]; + } + unsigned int sum = 0; + double chi_score = 0.0; + for (int i = 0; i < 5 * 5 * 5 * 5 * 5; ++i) { + int i1 = i % 5, i2 = (i / 5) % 5, i3 = (i / 25) % 5, i4 = (i / 125) % 5, i5 = i / 625; + uint32_t count = counts[i]; + if (i1 == i2 || i1 == i3 || i1 == i4 || i1 == i5 || i2 == i3 || i2 == i4 || i2 == i5 || i3 == i4 || i3 == i5 || i4 == i5) { + BOOST_CHECK(count == 0); + } else { + chi_score += ((count - 100.0) * (count - 100.0)) / 100.0; + BOOST_CHECK(count > 50); + BOOST_CHECK(count < 150); + sum += count; + } + } + BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval + BOOST_CHECK(chi_score < 210.275); + BOOST_CHECK_EQUAL(sum, 12000); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index beee18dc5e26d..2dc2ead0f8564 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2731,7 +2731,7 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int std::vector > > vValue; CAmount nTotalLower = 0; - random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); for (const COutput& output : vCoins) { if (!output.fSpendable) From af52bf5f7fedeadc532742e105c5fbe1018988e4 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sun, 28 Mar 2021 02:01:14 -0700 Subject: [PATCH 05/55] Use a FastRandomContext in LimitOrphanTxSize --- src/net_processing.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 49c898edf2b61..33941f3681004 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -553,9 +553,10 @@ void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { unsigned int nEvicted = 0; + FastRandomContext rng; while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = GetRandHash(); + uint256 randomhash = rng.rand256(); std::map::iterator it = mapOrphanTransactions.lower_bound(randomhash); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); From 972effa490f09daa4b2aa7a22cab60636eca1542 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sun, 28 Mar 2021 02:43:51 -0700 Subject: [PATCH 06/55] Make unit tests use the insecure_rand_ctx exclusively --- src/test/cuckoocache_tests.cpp | 37 ++++++++++------------------------ 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 278066b94f76a..ce532c96293d6 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -24,40 +24,25 @@ * using BOOST_CHECK_CLOSE to fail. * */ -FastRandomContext insecure_rand(true); BOOST_AUTO_TEST_SUITE(cuckoocache_tests); - -/** insecure_GetRandHash fills in a uint256 from insecure_rand - */ -void insecure_GetRandHash(uint256& t) -{ - uint32_t* ptr = (uint32_t*)t.begin(); - for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); -} - - - /* Test that no values not inserted into the cache are read out of it. * * There are no repeats in the first 200000 insecure_GetRandHash calls */ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); CuckooCache::cache cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); uint256 v; for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - cc.insert(v); + cc.insert(InsecureRand256()); } for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - BOOST_CHECK(!cc.contains(v, false)); + BOOST_CHECK(!cc.contains(InsecureRand256(), false)); } }; @@ -67,7 +52,7 @@ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) template double test_cache(size_t megabytes, double load) { - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); std::vector hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -77,7 +62,7 @@ double test_cache(size_t megabytes, double load) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -138,7 +123,7 @@ template void test_cache_erase(size_t megabytes) { double load = 1; - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); std::vector hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -148,7 +133,7 @@ void test_cache_erase(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -201,7 +186,7 @@ template void test_cache_erase_parallel(size_t megabytes) { double load = 1; - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); std::vector hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -211,7 +196,7 @@ void test_cache_erase_parallel(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -303,7 +288,7 @@ void test_cache_generations() // iterations with non-deterministic values, so it isn't "overfit" to the // specific entropy in FastRandomContext(true) and implementation of the // cache. - insecure_rand = FastRandomContext(true); + SeedInsecureRand(true); // block_activity models a chunk of network activity. n_insert elements are // adde to the cache. The first and last n/4 are stored for removal later @@ -320,7 +305,7 @@ void test_cache_generations() for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)inserts[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = insecure_rand.rand32(); + *(ptr++) = InsecureRand32(); } for (uint32_t i = 0; i < n_insert / 4; ++i) reads.push_back(inserts[i]); From 32e6c42f3fbe56fc28795fd8e6c3f02edd3d99fd Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sun, 28 Mar 2021 02:54:18 -0700 Subject: [PATCH 07/55] Simplify testing RNG code --- src/test/test_pivx.cpp | 3 +-- src/test/test_pivx.h | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/test/test_pivx.cpp b/src/test/test_pivx.cpp index 05ca418990388..7331a354fb11c 100644 --- a/src/test/test_pivx.cpp +++ b/src/test/test_pivx.cpp @@ -27,8 +27,7 @@ std::unique_ptr g_connman; CClientUIInterface uiInterface; // Declared but not defined in guiinterface.h -uint256 insecure_rand_seed = GetRandHash(); -FastRandomContext insecure_rand_ctx(insecure_rand_seed); +FastRandomContext insecure_rand_ctx; extern bool fPrintToConsole; extern void noui_connect(); diff --git a/src/test/test_pivx.h b/src/test/test_pivx.h index 251c9ffb929df..54bfa9accf0e2 100644 --- a/src/test/test_pivx.h +++ b/src/test/test_pivx.h @@ -11,17 +11,11 @@ #include -extern uint256 insecure_rand_seed; extern FastRandomContext insecure_rand_ctx; -static inline void SeedInsecureRand(bool fDeterministic = false) +static inline void SeedInsecureRand(bool deterministic = false) { - if (fDeterministic) { - insecure_rand_seed = uint256(); - } else { - insecure_rand_seed = GetRandHash(); - } - insecure_rand_ctx = FastRandomContext(insecure_rand_seed); + insecure_rand_ctx = FastRandomContext(deterministic); } static inline uint32_t InsecureRand32() { return insecure_rand_ctx.rand32(); } From 1a5dbc501f90b0e19d6bf8ea4b5e523b85ce4abe Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 19 Dec 2018 01:50:36 -0800 Subject: [PATCH 08/55] Don't log RandAddSeedPerfmon details These are hard to deal with, as in a follow-up this function can get called before the logging infrastructure is initialized. --- src/random.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index d7ed086110ce0..1039f53162f72 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -166,13 +166,13 @@ static void RandAddSeedPerfmon() if (ret == ERROR_SUCCESS) { RAND_add(vData.data(), nSize, nSize / 100.0); memory_cleanse(vData.data(), nSize); - LogPrint(BCLog::RAND, "%s: %lu bytes\n", __func__, nSize); } else { - static bool warned = false; // Warn only once - if (!warned) { - LogPrintf("%s: Warning: RegQueryValueExA(HKEY_PERFORMANCE_DATA) failed with code %i\n", __func__, ret); - warned = true; - } + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it is isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. } #endif } From d76ee8334b78d604564780ed250ba437ca9b0b2c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Dec 2018 16:48:21 -0800 Subject: [PATCH 09/55] Automatically initialize RNG on first use. --- src/random.cpp | 68 +++++++++++++++++++++++++++++++----------- src/random.h | 7 ++++- src/test/test_pivx.cpp | 2 -- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 1039f53162f72..a37bd1be4715c 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -74,7 +74,6 @@ static inline int64_t GetPerformanceCounter() #endif } - #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) static std::atomic hwrand_initialized{false}; static bool rdrand_supported = false; @@ -83,13 +82,24 @@ static void RDRandInit() { uint32_t eax, ebx, ecx, edx; if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { - LogPrintf("Using RdRand as an additional entropy source\n"); rdrand_supported = true; } hwrand_initialized.store(true); } + +static void RDRandReport() +{ + assert(hwrand_initialized.load(std::memory_order_relaxed)); + if (rdrand_supported) { + // This must be done in a separate function, as HWRandInit() may be indirectly called + // from global constructors, before logging is initialized. + LogPrintf("Using RdRand as an additional entropy source\n"); + } +} + #else static void RDRandInit() {} +static void RDRandReport() {} #endif static bool GetHWRand(unsigned char* ent32) { @@ -279,6 +289,26 @@ void GetRandBytes(unsigned char* buf, int num) } } +namespace { +struct RNGState { + Mutex m_mutex; + unsigned char m_state[32] = {0}; + uint64_t m_counter = 0; + + explicit RNGState() { + RDRandInit(); + } +}; + +RNGState& GetRNGState() +{ + // This C++11 idiom relies on the guarantee that static variable are initialized + // on first call, even when multiple parallel calls are permitted. + static std::unique_ptr g_rng{new RNGState()}; + return *g_rng; +} +} + static void AddDataToRng(void* data, size_t len); void RandAddSeedSleep() @@ -295,29 +325,28 @@ void RandAddSeedSleep() memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); } - -static std::mutex cs_rng_state; -static unsigned char rng_state[32] = {0}; -static uint64_t rng_counter = 0; - static void AddDataToRng(void* data, size_t len) { + RNGState& rng = GetRNGState(); + CSHA512 hasher; hasher.Write((const unsigned char*)&len, sizeof(len)); hasher.Write((const unsigned char*)data, len); unsigned char buf[64]; { - std::unique_lock lock(cs_rng_state); - hasher.Write(rng_state, sizeof(rng_state)); - hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); - ++rng_counter; + WAIT_LOCK(rng.m_mutex, lock); + hasher.Write(rng.m_state, sizeof(rng.m_state)); + hasher.Write((const unsigned char*)&rng.m_counter, sizeof(rng.m_counter)); + ++rng.m_counter; hasher.Finalize(buf); - memcpy(rng_state, buf + 32, 32); + memcpy(rng.m_state, buf + 32, 32); } memory_cleanse(buf, 64); } void GetStrongRandBytes(unsigned char* out, int num) { + RNGState& rng = GetRNGState(); + assert(num <= 32); CSHA512 hasher; unsigned char buf[64]; @@ -338,12 +367,12 @@ void GetStrongRandBytes(unsigned char* out, int num) // Combine with and update state { - std::unique_lock lock(cs_rng_state); - hasher.Write(rng_state, sizeof(rng_state)); - hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); - ++rng_counter; + WAIT_LOCK(rng.m_mutex, lock); + hasher.Write(rng.m_state, sizeof(rng.m_state)); + hasher.Write((const unsigned char*)&rng.m_counter, sizeof(rng.m_counter)); + ++rng.m_counter; hasher.Finalize(buf); - memcpy(rng_state, buf + 32, 32); + memcpy(rng.m_state, buf + 32, 32); } // Produce output @@ -480,5 +509,8 @@ FastRandomContext& FastRandomContext::operator=(FastRandomContext&& from) noexce void RandomInit() { - RDRandInit(); + // Invoke RNG code to trigger initialization (if not already performed) + GetRNGState(); + + RDRandReport(); } diff --git a/src/random.h b/src/random.h index a1babfad424bc..02eae32a20962 100644 --- a/src/random.h +++ b/src/random.h @@ -179,7 +179,12 @@ void GetOSRand(unsigned char *ent32); */ bool Random_SanityCheck(); -/** Initialize the RNG. */ +/** + * Initialize global RNG state and log any CPU features that are used. + * + * Calling this function is optional. RNG state will be initialized when first + * needed if it is not called. + */ void RandomInit(); #endif // BITCOIN_RANDOM_H diff --git a/src/test/test_pivx.cpp b/src/test/test_pivx.cpp index 7331a354fb11c..d5e0812c31e6d 100644 --- a/src/test/test_pivx.cpp +++ b/src/test/test_pivx.cpp @@ -12,7 +12,6 @@ #include "evo/evodb.h" #include "miner.h" #include "net_processing.h" -#include "random.h" #include "rpc/server.h" #include "rpc/register.h" #include "script/sigcache.h" @@ -40,7 +39,6 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) BasicTestingSetup::BasicTestingSetup() { - RandomInit(); ECC_Start(); SetupEnvironment(); InitSignatureCache(); From 2326535f52513d95b47a66ab9e2f0896ea7601b1 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 16 Jan 2019 15:15:21 -0800 Subject: [PATCH 10/55] Rename some hardware RNG related functions --- src/random.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index a37bd1be4715c..538f4f87889f1 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -78,7 +78,7 @@ static inline int64_t GetPerformanceCounter() static std::atomic hwrand_initialized{false}; static bool rdrand_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; -static void RDRandInit() +static void InitHardwareRand() { uint32_t eax, ebx, ecx, edx; if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { @@ -87,7 +87,7 @@ static void RDRandInit() hwrand_initialized.store(true); } -static void RDRandReport() +static void ReportHardwareRand() { assert(hwrand_initialized.load(std::memory_order_relaxed)); if (rdrand_supported) { @@ -98,11 +98,16 @@ static void RDRandReport() } #else -static void RDRandInit() {} -static void RDRandReport() {} +/* Access to other hardware random number generators could be added here later, + * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). + * Slower sources should probably be invoked separately, and/or only from + * RandAddSeedSleep (which is called during idle background operation). + */ +static void InitHardwareRand() {} +static void ReportHardwareRand() {} #endif -static bool GetHWRand(unsigned char* ent32) { +static bool GetHardwareRand(unsigned char* ent32) { #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) assert(hwrand_initialized.load(std::memory_order_relaxed)); if (rdrand_supported) { @@ -296,7 +301,7 @@ struct RNGState { uint64_t m_counter = 0; explicit RNGState() { - RDRandInit(); + InitHardwareRand(); } }; @@ -361,7 +366,7 @@ void GetStrongRandBytes(unsigned char* out, int num) hasher.Write(buf, 32); // Third source: HW RNG, if available. - if (GetHWRand(buf)) { + if (GetHardwareRand(buf)) { hasher.Write(buf, 32); } @@ -512,5 +517,5 @@ void RandomInit() // Invoke RNG code to trigger initialization (if not already performed) GetRNGState(); - RDRandReport(); + ReportHardwareRand(); } From 298f97c5ef4963928d5b196fde2f125598b35e5f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Dec 2018 17:03:30 -0800 Subject: [PATCH 11/55] Add thread safety annotations to RNG state --- src/random.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 538f4f87889f1..a047af873eeb8 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -297,10 +297,11 @@ void GetRandBytes(unsigned char* buf, int num) namespace { struct RNGState { Mutex m_mutex; - unsigned char m_state[32] = {0}; - uint64_t m_counter = 0; + unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; + uint64_t m_counter GUARDED_BY(m_mutex) = 0; - explicit RNGState() { + RNGState() + { InitHardwareRand(); } }; From 5f20e6252e8322e4dcea3772b3584772d4e5c283 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Dec 2018 16:04:35 -0800 Subject: [PATCH 12/55] Abstract out seeding/extracting entropy into RNGState::MixExtract --- src/crypto/sha512.h | 2 +- src/random.cpp | 60 +++++++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index 5566d5db3ef09..404f8e6632cb8 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -17,7 +17,7 @@ class CSHA512 size_t bytes; public: - static const size_t OUTPUT_SIZE = 64; + static constexpr size_t OUTPUT_SIZE = 64; CSHA512(); CSHA512& Write(const unsigned char* data, size_t len); diff --git a/src/random.cpp b/src/random.cpp index a047af873eeb8..0fb07721f8805 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -304,6 +304,34 @@ struct RNGState { { InitHardwareRand(); } + + /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. */ + void MixExtract(unsigned char* out, size_t num, CSHA512&& hasher) + { + assert(num <= 32); + unsigned char buf[64]; + static_assert(sizeof(buf) == CSHA512::OUTPUT_SIZE, "Buffer needs to have hasher's output size"); + { + LOCK(m_mutex); + // Write the current state of the RNG into the hasher + hasher.Write(m_state, 32); + // Write a new counter number into the state + hasher.Write((const unsigned char*)&m_counter, sizeof(m_counter)); + ++m_counter; + // Finalize the hasher + hasher.Finalize(buf); + // Store the last 32 bytes of the hash output as new RNG state. + memcpy(m_state, buf + 32, 32); + } + // If desired, copy (up to) the first 32 bytes of the hash output as output. + if (num) { + assert(out != nullptr); + memcpy(out, buf, num); + } + // Best effort cleanup of internal state + hasher.Reset(); + memory_cleanse(buf, 64); + } }; RNGState& GetRNGState() @@ -315,38 +343,29 @@ RNGState& GetRNGState() } } -static void AddDataToRng(void* data, size_t len); +static void AddDataToRng(void* data, size_t len, RNGState& rng); void RandAddSeedSleep() { + RNGState& rng = GetRNGState(); + int64_t nPerfCounter1 = GetPerformanceCounter(); std::this_thread::sleep_for(std::chrono::milliseconds(1)); int64_t nPerfCounter2 = GetPerformanceCounter(); // Combine with and update state - AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1)); - AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2)); + AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1), rng); + AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2), rng); memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); } -static void AddDataToRng(void* data, size_t len) { - RNGState& rng = GetRNGState(); - +static void AddDataToRng(void* data, size_t len, RNGState& rng) { CSHA512 hasher; hasher.Write((const unsigned char*)&len, sizeof(len)); hasher.Write((const unsigned char*)data, len); - unsigned char buf[64]; - { - WAIT_LOCK(rng.m_mutex, lock); - hasher.Write(rng.m_state, sizeof(rng.m_state)); - hasher.Write((const unsigned char*)&rng.m_counter, sizeof(rng.m_counter)); - ++rng.m_counter; - hasher.Finalize(buf); - memcpy(rng.m_state, buf + 32, 32); - } - memory_cleanse(buf, 64); + rng.MixExtract(nullptr, 0, std::move(hasher)); } void GetStrongRandBytes(unsigned char* out, int num) @@ -372,14 +391,7 @@ void GetStrongRandBytes(unsigned char* out, int num) } // Combine with and update state - { - WAIT_LOCK(rng.m_mutex, lock); - hasher.Write(rng.m_state, sizeof(rng.m_state)); - hasher.Write((const unsigned char*)&rng.m_counter, sizeof(rng.m_counter)); - ++rng.m_counter; - hasher.Finalize(buf); - memcpy(rng.m_state, buf + 32, 32); - } + rng.MixExtract(out, num, std::move(hasher)); // Produce output memcpy(out, buf, num); From 038a45af3e448814d5a93b46341353d4cfdb3f1f Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Sat, 27 Mar 2021 20:26:08 -0700 Subject: [PATCH 13/55] Integrate util/system's CInit into RNGState This guarantees that OpenSSL is initialized properly whenever randomness is used, even when that randomness is invoked from global constructors. Note that this patch uses Mutex directly, rather than CCriticalSection. This is because the lock-detection code is not necessarily initialized during global constructors. --- src/random.cpp | 43 ++++++++++++++++++++++++++++++++++++++++ src/util.cpp | 54 -------------------------------------------------- 2 files changed, 43 insertions(+), 54 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 0fb07721f8805..145491793529c 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -47,6 +47,7 @@ #include #include +#include [[noreturn]] static void RandFailure() { @@ -294,15 +295,46 @@ void GetRandBytes(unsigned char* buf, int num) } } +void LockingCallbackOpenSSL(int mode, int i, const char* file, int line); + namespace { + struct RNGState { Mutex m_mutex; unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; uint64_t m_counter GUARDED_BY(m_mutex) = 0; + std::unique_ptr m_mutex_openssl; RNGState() { InitHardwareRand(); + + // Init OpenSSL library multithreading support + m_mutex_openssl.reset(new Mutex[CRYPTO_num_locks()]); + CRYPTO_set_locking_callback(LockingCallbackOpenSSL); + + // OpenSSL can optionally load a config file which lists optional loadable modules and engines. + // We don't use them so we don't require the config. However some of our libs may call functions + // which attempt to load the config file, possibly resulting in an exit() or crash if it is missing + // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be + // that the config appears to have been loaded and there are no modules/engines available. + OPENSSL_no_config(); + +#ifdef WIN32 + // Seed OpenSSL PRNG with current contents of the screen + RAND_screen(); +#endif + + // Seed OpenSSL PRNG with performance counter + RandAddSeed(); + } + + ~RNGState() + { + // Securely erase the memory used by the OpenSSL PRNG + RAND_cleanup(); + // Shutdown OpenSSL library multithreading support + CRYPTO_set_locking_callback(nullptr); } /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. */ @@ -343,6 +375,17 @@ RNGState& GetRNGState() } } +void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS +{ + RNGState& rng = GetRNGState(); + + if (mode & CRYPTO_LOCK) { + rng.m_mutex_openssl[i].lock(); + } else { + rng.m_mutex_openssl[i].unlock(); + } +} + static void AddDataToRng(void* data, size_t len, RNGState& rng); void RandAddSeedSleep() diff --git a/src/util.cpp b/src/util.cpp index c25216fc147fc..424897daeb421 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -79,9 +79,6 @@ #include #include -#include -#include -#include const char * const PIVX_CONF_FILENAME = "pivx.conf"; const char * const PIVX_PID_FILENAME = "pivx.pid"; @@ -101,57 +98,6 @@ ArgsManager gArgs; bool fDaemon = false; CTranslationInterface translationInterface; -/** Init OpenSSL library multithreading support */ -static RecursiveMutex** ppmutexOpenSSL; -void locking_callback(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS -{ - if (mode & CRYPTO_LOCK) { - ENTER_CRITICAL_SECTION(*ppmutexOpenSSL[i]); - } else { - LEAVE_CRITICAL_SECTION(*ppmutexOpenSSL[i]); - } -} - -// Init -class CInit -{ -public: - CInit() - { - // Init OpenSSL library multithreading support - ppmutexOpenSSL = (RecursiveMutex**)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(RecursiveMutex*)); - for (int i = 0; i < CRYPTO_num_locks(); i++) - ppmutexOpenSSL[i] = new RecursiveMutex(); - CRYPTO_set_locking_callback(locking_callback); - - // OpenSSL can optionally load a config file which lists optional loadable modules and engines. - // We don't use them so we don't require the config. However some of our libs may call functions - // which attempt to load the config file, possibly resulting in an exit() or crash if it is missing - // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be - // that the config appears to have been loaded and there are no modules/engines available. - OPENSSL_no_config(); - -#ifdef WIN32 - // Seed OpenSSL PRNG with current contents of the screen - RAND_screen(); -#endif - - // Seed OpenSSL PRNG with performance counter - RandAddSeed(); - } - ~CInit() - { - // Securely erase the memory used by the PRNG - RAND_cleanup(); - // Shutdown OpenSSL library multithreading support - CRYPTO_set_locking_callback(NULL); - for (int i = 0; i < CRYPTO_num_locks(); i++) - delete ppmutexOpenSSL[i]; - OPENSSL_free(ppmutexOpenSSL); - } -} instance_of_cinit; - - /** * Interpret a string argument as a boolean. * From 698d133ce9f664e257aa44052fc33723d48359cb Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 13 Dec 2018 18:37:29 -0800 Subject: [PATCH 14/55] Switch all RNG code to the built-in PRNG. It includes the following policy changes: * All GetRand* functions seed the stack pointer and rdrand result (in addition to the performance counter) * The periodic entropy added by the idle scheduler now seeds stack pointer, rdrand and perfmon data (once every 10 minutes) in addition to just a sleep timing. * The entropy added when calling GetStrongRandBytes no longer includes the once-per-10-minutes perfmon data on windows (it is moved to the idle scheduler instead, where latency matters less). Other changes: * OpenSSL is no longer seeded directly anywhere. Instead, any generated randomness through our own RNG is fed back to OpenSSL (after an additional hashing step to prevent leaking our RNG state). * Seeding that was previously done directly in RandAddSeedSleep is now moved to SeedSleep(), which is indirectly invoked through ProcRand from RandAddSeedSleep. * Seeding that was previously done directly in GetStrongRandBytes() is now moved to SeedSlow(), which is indirectly invoked through ProcRand from GetStrongRandBytes(). --- src/libzerocoin/bignum_gmp.cpp | 4 +- src/random.cpp | 185 +++++++++++++++++++++------------ src/random.h | 31 +++--- src/scheduler.cpp | 2 +- 4 files changed, 140 insertions(+), 82 deletions(-) diff --git a/src/libzerocoin/bignum_gmp.cpp b/src/libzerocoin/bignum_gmp.cpp index 6282dd9c3a28a..0de0e403e1f2e 100644 --- a/src/libzerocoin/bignum_gmp.cpp +++ b/src/libzerocoin/bignum_gmp.cpp @@ -63,7 +63,7 @@ CBigNum CBigNum::randBignum(const CBigNum& range) size_t size = (mpz_sizeinbase (range.bn, 2) + CHAR_BIT-1) / CHAR_BIT; std::vector buf(size); - RandAddSeed(); + RandAddSeedSleep(); GetRandBytes(buf.data(), size); CBigNum ret(buf); @@ -80,7 +80,7 @@ CBigNum CBigNum::randKBitBignum(const uint32_t k) { std::vector buf((k+7)/8); - RandAddSeed(); + RandAddSeedSleep(); GetRandBytes(buf.data(), (k+7)/8); CBigNum ret(buf); diff --git a/src/random.cpp b/src/random.cpp index 145491793529c..8998843771f43 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -145,18 +145,8 @@ static bool GetHardwareRand(unsigned char* ent32) { return false; } -void RandAddSeed() +static void RandAddSeedPerfmon(CSHA512& hasher) { - // Seed with CPU performance counter - int64_t nCounter = GetPerformanceCounter(); - RAND_add(&nCounter, sizeof(nCounter), 1.5); - memory_cleanse((void*)&nCounter, sizeof(nCounter)); -} - -static void RandAddSeedPerfmon() -{ - RandAddSeed(); - #ifdef WIN32 // Don't need this on Linux, OpenSSL automatically uses /dev/urandom // Seed with the entire set of perfmon data @@ -180,7 +170,7 @@ static void RandAddSeedPerfmon() } RegCloseKey(HKEY_PERFORMANCE_DATA); if (ret == ERROR_SUCCESS) { - RAND_add(vData.data(), nSize, nSize / 100.0); + hasher.Write(vData.data(), nSize); memory_cleanse(vData.data(), nSize); } else { // Performance data is only a best-effort attempt at improving the @@ -288,13 +278,6 @@ void GetOSRand(unsigned char *ent32) #endif } -void GetRandBytes(unsigned char* buf, int num) -{ - if (RAND_bytes(buf, num) != 1) { - RandFailure(); - } -} - void LockingCallbackOpenSSL(int mode, int i, const char* file, int line); namespace { @@ -303,6 +286,7 @@ struct RNGState { Mutex m_mutex; unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; uint64_t m_counter GUARDED_BY(m_mutex) = 0; + bool m_strongly_seeded GUARDED_BY(m_mutex) = false; std::unique_ptr m_mutex_openssl; RNGState() @@ -319,14 +303,6 @@ struct RNGState { // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be // that the config appears to have been loaded and there are no modules/engines available. OPENSSL_no_config(); - -#ifdef WIN32 - // Seed OpenSSL PRNG with current contents of the screen - RAND_screen(); -#endif - - // Seed OpenSSL PRNG with performance counter - RandAddSeed(); } ~RNGState() @@ -337,14 +313,19 @@ struct RNGState { CRYPTO_set_locking_callback(nullptr); } - /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. */ - void MixExtract(unsigned char* out, size_t num, CSHA512&& hasher) + /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. + * + * If this function has never been called with strong_seed = true, false is returned. + */ + bool MixExtract(unsigned char* out, size_t num, CSHA512&& hasher, bool strong_seed) { assert(num <= 32); unsigned char buf[64]; static_assert(sizeof(buf) == CSHA512::OUTPUT_SIZE, "Buffer needs to have hasher's output size"); + bool ret; { LOCK(m_mutex); + ret = (m_strongly_seeded |= strong_seed); // Write the current state of the RNG into the hasher hasher.Write(m_state, 32); // Write a new counter number into the state @@ -363,6 +344,7 @@ struct RNGState { // Best effort cleanup of internal state hasher.Reset(); memory_cleanse(buf, 64); + return ret; } }; @@ -386,61 +368,128 @@ void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THRE } } -static void AddDataToRng(void* data, size_t len, RNGState& rng); +static void SeedTimestamp(CSHA512& hasher) +{ + int64_t perfcounter = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); +} -void RandAddSeedSleep() +static void SeedFast(CSHA512& hasher) { - RNGState& rng = GetRNGState(); + unsigned char buffer[32]; - int64_t nPerfCounter1 = GetPerformanceCounter(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - int64_t nPerfCounter2 = GetPerformanceCounter(); + // Stack pointer to indirectly commit to thread/callstack + const unsigned char* ptr = buffer; + hasher.Write((const unsigned char*)&ptr, sizeof(ptr)); - // Combine with and update state - AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1), rng); - AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2), rng); + // Hardware randomness is very fast when available; use it always. + bool have_hw_rand = GetHardwareRand(buffer); + if (have_hw_rand) hasher.Write(buffer, sizeof(buffer)); - memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); - memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); + // High-precision timestamp + SeedTimestamp(hasher); } -static void AddDataToRng(void* data, size_t len, RNGState& rng) { - CSHA512 hasher; - hasher.Write((const unsigned char*)&len, sizeof(len)); - hasher.Write((const unsigned char*)data, len); - rng.MixExtract(nullptr, 0, std::move(hasher)); +static void SeedSlow(CSHA512& hasher) +{ + unsigned char buffer[32]; + + // Everything that the 'fast' seeder includes + SeedFast(hasher); + + // OS randomness + GetOSRand(buffer); + hasher.Write(buffer, sizeof(buffer)); + + // OpenSSL RNG (for now) + RAND_bytes(buffer, sizeof(buffer)); + hasher.Write(buffer, sizeof(buffer)); + + // High-precision timestamp. + // + // Note that we also commit to a timestamp in the Fast seeder, so we indirectly commit to a + // benchmark of all the entropy gathering sources in this function). + SeedTimestamp(hasher); } -void GetStrongRandBytes(unsigned char* out, int num) +static void SeedSleep(CSHA512& hasher) { - RNGState& rng = GetRNGState(); + // Everything that the 'fast' seeder includes + SeedFast(hasher); - assert(num <= 32); - CSHA512 hasher; - unsigned char buf[64]; + // High-precision timestamp + SeedTimestamp(hasher); - // First source: OpenSSL's RNG - RandAddSeedPerfmon(); - GetRandBytes(buf, 32); - hasher.Write(buf, 32); + // Sleep for 1ms + MilliSleep(1); - // Second source: OS RNG - GetOSRand(buf); - hasher.Write(buf, 32); + // High-precision timestamp after sleeping (as we commit to both the time before and after, this measures the delay) + SeedTimestamp(hasher); - // Third source: HW RNG, if available. - if (GetHardwareRand(buf)) { - hasher.Write(buf, 32); + // Windows performance monitor data (once every 10 minutes) + RandAddSeedPerfmon(hasher); +} + +static void SeedStartup(CSHA512& hasher) +{ +#ifdef WIN32 + RAND_screen(); +#endif + + // Everything that the 'slow' seeder includes. + SeedSlow(hasher); + + // Windows performance monitor data. + RandAddSeedPerfmon(hasher); +} + +enum class RNGLevel { + FAST, //!< Automatically called by GetRandBytes + SLOW, //!< Automatically called by GetStrongRandBytes + SLEEP, //!< Called by RandAddSeedSleep() +}; + +static void ProcRand(unsigned char* out, int num, RNGLevel level) +{ + // Make sure the RNG is initialized first (as all Seed* function possibly need hwrand to be available). + RNGState& rng = GetRNGState(); + + assert(num <= 32); + + CSHA512 hasher; + switch (level) { + case RNGLevel::FAST: + SeedFast(hasher); + break; + case RNGLevel::SLOW: + SeedSlow(hasher); + break; + case RNGLevel::SLEEP: + SeedSleep(hasher); + break; } // Combine with and update state - rng.MixExtract(out, num, std::move(hasher)); + if (!rng.MixExtract(out, num, std::move(hasher), false)) { + // On the first invocation, also seed with SeedStartup(). + CSHA512 startup_hasher; + SeedStartup(startup_hasher); + rng.MixExtract(out, num, std::move(startup_hasher), true); + } - // Produce output - memcpy(out, buf, num); - memory_cleanse(buf, 64); + // For anything but the 'fast' level, feed the resulting RNG output (after an additional hashing step) back into OpenSSL. + if (level != RNGLevel::FAST) { + unsigned char buf[64]; + CSHA512().Write(out, num).Finalize(buf); + RAND_add(buf, sizeof(buf), num); + memory_cleanse(buf, 64); + } } +void GetRandBytes(unsigned char* buf, int num) { ProcRand(buf, num, RNGLevel::FAST); } +void GetStrongRandBytes(unsigned char* buf, int num) { ProcRand(buf, num, RNGLevel::SLOW); } +void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } + uint64_t GetRand(uint64_t nMax) { if (nMax == 0) @@ -539,8 +588,10 @@ bool Random_SanityCheck() if (stop == start) return false; // We called GetPerformanceCounter. Use it as entropy. - RAND_add((const unsigned char*)&start, sizeof(start), 1); - RAND_add((const unsigned char*)&stop, sizeof(stop), 1); + CSHA512 to_add; + to_add.Write((const unsigned char*)&start, sizeof(start)); + to_add.Write((const unsigned char*)&stop, sizeof(stop)); + GetRNGState().MixExtract(nullptr, 0, std::move(to_add), false); return true; } @@ -571,7 +622,7 @@ FastRandomContext& FastRandomContext::operator=(FastRandomContext&& from) noexce void RandomInit() { // Invoke RNG code to trigger initialization (if not already performed) - GetRNGState(); + ProcRand(nullptr, 0, RNGLevel::FAST); ReportHardwareRand(); } diff --git a/src/random.h b/src/random.h index 02eae32a20962..ffd87815f621c 100644 --- a/src/random.h +++ b/src/random.h @@ -14,11 +14,13 @@ #include #include -/* Seed OpenSSL PRNG with additional entropy data */ -void RandAddSeed(); - /** - * Functions to gather random data via the OpenSSL PRNG + * Generate random data via the internal PRNG. + * + * These functions are designed to be fast (sub microsecond), but do not necessarily + * meaningfully add entropy to the PRNG state. + * + * Thread-safe. */ void GetRandBytes(unsigned char* buf, int num); uint64_t GetRand(uint64_t nMax); @@ -26,21 +28,26 @@ int GetRandInt(int nMax); uint256 GetRandHash(); /** - * Add a little bit of randomness to the output of GetStrongRangBytes. - * This sleeps for a millisecond, so should only be called when there is - * no other work to be done. + * Gather entropy from various sources, feed it into the internal PRNG, and + * generate random data using it. + * + * This function will cause failure whenever the OS RNG fails. + * + * Thread-safe. */ -void RandAddSeedSleep(); +void GetStrongRandBytes(unsigned char* buf, int num); /** - * Function to gather random data from multiple sources, failing whenever any - * of those source fail to provide a result. + * Sleep for 1ms, gather entropy from various sources, and feed them to the PRNG state. + * + * Thread-safe. */ -void GetStrongRandBytes(unsigned char* buf, int num); +void RandAddSeedSleep(); /** * Fast randomness source. This is seeded once with secure random data, but - * is completely deterministic and insecure after that. + * is completely deterministic and does not gather more entropy after that. + * * This class is not thread-safe. */ class FastRandomContext { diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 86c08510ce455..ed50cc177a2e8 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -32,7 +32,7 @@ void CScheduler::serviceQueue() try { if (!shouldStop() && taskQueue.empty()) { reverse_lock > rlock(lock); - // Use this chance to get a tiny bit more entropy + // Use this chance to get more entropy RandAddSeedSleep(); } while (!shouldStop() && taskQueue.empty()) { From 774899f92a30441d193a59d4ec1d7177a5cf3243 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Dec 2018 16:22:22 -0800 Subject: [PATCH 15/55] Remove hwrand_initialized. All access to hwrand is now gated by GetRNGState, which initializes the hwrand code. --- src/random.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 8998843771f43..d49b14b6c9311 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -76,7 +76,6 @@ static inline int64_t GetPerformanceCounter() } #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -static std::atomic hwrand_initialized{false}; static bool rdrand_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; static void InitHardwareRand() @@ -85,12 +84,10 @@ static void InitHardwareRand() if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { rdrand_supported = true; } - hwrand_initialized.store(true); } static void ReportHardwareRand() { - assert(hwrand_initialized.load(std::memory_order_relaxed)); if (rdrand_supported) { // This must be done in a separate function, as HWRandInit() may be indirectly called // from global constructors, before logging is initialized. @@ -110,7 +107,6 @@ static void ReportHardwareRand() {} static bool GetHardwareRand(unsigned char* ent32) { #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) - assert(hwrand_initialized.load(std::memory_order_relaxed)); if (rdrand_supported) { uint8_t ok; // Not all assemblers support the rdrand instruction, write it in hex. From 5bc2583eaef57867aa30c4ce8c42786bd2fb3f12 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Dec 2018 15:11:33 -0800 Subject: [PATCH 16/55] Sprinkle some sweet noexcepts over the RNG code --- src/random.cpp | 49 ++++++++++++++++++++++++++++++++----------------- src/random.h | 28 ++++++++++++++-------------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index d49b14b6c9311..73bfb2cd9d748 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -55,7 +55,7 @@ std::abort(); } -static inline int64_t GetPerformanceCounter() +static inline int64_t GetPerformanceCounter() noexcept { // Read the hardware time stamp counter when available. // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. @@ -105,7 +105,7 @@ static void InitHardwareRand() {} static void ReportHardwareRand() {} #endif -static bool GetHardwareRand(unsigned char* ent32) { +static bool GetHardwareRand(unsigned char* ent32) noexcept { #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) if (rdrand_supported) { uint8_t ok; @@ -285,7 +285,7 @@ struct RNGState { bool m_strongly_seeded GUARDED_BY(m_mutex) = false; std::unique_ptr m_mutex_openssl; - RNGState() + RNGState() noexcept { InitHardwareRand(); @@ -313,7 +313,7 @@ struct RNGState { * * If this function has never been called with strong_seed = true, false is returned. */ - bool MixExtract(unsigned char* out, size_t num, CSHA512&& hasher, bool strong_seed) + bool MixExtract(unsigned char* out, size_t num, CSHA512&& hasher, bool strong_seed) noexcept { assert(num <= 32); unsigned char buf[64]; @@ -344,7 +344,7 @@ struct RNGState { } }; -RNGState& GetRNGState() +RNGState& GetRNGState() noexcept { // This C++11 idiom relies on the guarantee that static variable are initialized // on first call, even when multiple parallel calls are permitted. @@ -364,13 +364,28 @@ void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THRE } } -static void SeedTimestamp(CSHA512& hasher) +/* A note on the use of noexcept in the seeding functions below: + * + * None of the RNG code should ever throw any exception, with the sole exception + * of MilliSleep in SeedSleep, which can (and does) support interruptions which + * cause a boost::thread_interrupted to be thrown. + * + * This means that SeedSleep, and all functions that invoke it are throwing. + * However, we know that GetRandBytes() and GetStrongRandBytes() never trigger + * this sleeping logic, so they are noexcept. The same is true for all the + * GetRand*() functions that use GetRandBytes() indirectly. + * + * TODO: After moving away from interruptible boost-based thread management, + * everything can become noexcept here. + */ + +static void SeedTimestamp(CSHA512& hasher) noexcept { int64_t perfcounter = GetPerformanceCounter(); hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); } -static void SeedFast(CSHA512& hasher) +static void SeedFast(CSHA512& hasher) noexcept { unsigned char buffer[32]; @@ -386,7 +401,7 @@ static void SeedFast(CSHA512& hasher) SeedTimestamp(hasher); } -static void SeedSlow(CSHA512& hasher) +static void SeedSlow(CSHA512& hasher) noexcept { unsigned char buffer[32]; @@ -426,7 +441,7 @@ static void SeedSleep(CSHA512& hasher) RandAddSeedPerfmon(hasher); } -static void SeedStartup(CSHA512& hasher) +static void SeedStartup(CSHA512& hasher) noexcept { #ifdef WIN32 RAND_screen(); @@ -482,11 +497,11 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) } } -void GetRandBytes(unsigned char* buf, int num) { ProcRand(buf, num, RNGLevel::FAST); } -void GetStrongRandBytes(unsigned char* buf, int num) { ProcRand(buf, num, RNGLevel::SLOW); } +void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } +void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } -uint64_t GetRand(uint64_t nMax) +uint64_t GetRand(uint64_t nMax) noexcept { if (nMax == 0) return 0; @@ -501,12 +516,12 @@ uint64_t GetRand(uint64_t nMax) return (nRand % nMax); } -int GetRandInt(int nMax) +int GetRandInt(int nMax) noexcept { return GetRand(nMax); } -uint256 GetRandHash() +uint256 GetRandHash() noexcept { uint256 hash; GetRandBytes((unsigned char*)&hash, sizeof(hash)); @@ -520,7 +535,7 @@ void FastRandomContext::RandomSeed() requires_seed = false; } -uint256 FastRandomContext::rand256() +uint256 FastRandomContext::rand256() noexcept { if (bytebuf_size < 32) { FillByteBuffer(); @@ -541,7 +556,7 @@ std::vector FastRandomContext::randbytes(size_t len) return ret; } -FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +FastRandomContext::FastRandomContext(const uint256& seed) noexcept : requires_seed(false), bytebuf_size(0), bitbuf_size(0) { rng.SetKey(seed.begin(), 32); } @@ -592,7 +607,7 @@ bool Random_SanityCheck() return true; } -FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +FastRandomContext::FastRandomContext(bool fDeterministic) noexcept : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) { if (!fDeterministic) { return; diff --git a/src/random.h b/src/random.h index ffd87815f621c..b799e25132dc7 100644 --- a/src/random.h +++ b/src/random.h @@ -22,10 +22,10 @@ * * Thread-safe. */ -void GetRandBytes(unsigned char* buf, int num); -uint64_t GetRand(uint64_t nMax); -int GetRandInt(int nMax); -uint256 GetRandHash(); +void GetRandBytes(unsigned char* buf, int num) noexcept; +uint64_t GetRand(uint64_t nMax) noexcept; +int GetRandInt(int nMax) noexcept; +uint256 GetRandHash() noexcept; /** * Gather entropy from various sources, feed it into the internal PRNG, and @@ -35,7 +35,7 @@ uint256 GetRandHash(); * * Thread-safe. */ -void GetStrongRandBytes(unsigned char* buf, int num); +void GetStrongRandBytes(unsigned char* buf, int num) noexcept; /** * Sleep for 1ms, gather entropy from various sources, and feed them to the PRNG state. @@ -79,10 +79,10 @@ class FastRandomContext { } public: - explicit FastRandomContext(bool fDeterministic = false); + explicit FastRandomContext(bool fDeterministic = false) noexcept; /** Initialize with explicit seed (only for testing) */ - explicit FastRandomContext(const uint256& seed); + explicit FastRandomContext(const uint256& seed) noexcept; // Do not permit copying a FastRandomContext (move it, or create a new one to get reseeded). FastRandomContext(const FastRandomContext&) = delete; @@ -93,7 +93,7 @@ class FastRandomContext { FastRandomContext& operator=(FastRandomContext&& from) noexcept; /** Generate a random 64-bit integer. */ - uint64_t rand64() + uint64_t rand64() noexcept { if (bytebuf_size < 8) FillByteBuffer(); uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); @@ -102,7 +102,7 @@ class FastRandomContext { } /** Generate a random (bits)-bit integer. */ - uint64_t randbits(int bits) { + uint64_t randbits(int bits) noexcept { if (bits == 0) { return 0; } else if (bits > 32) { @@ -117,7 +117,7 @@ class FastRandomContext { } /** Generate a random integer in the range [0..range). */ - uint64_t randrange(uint64_t range) + uint64_t randrange(uint64_t range) noexcept { --range; int bits = CountBits(range); @@ -131,19 +131,19 @@ class FastRandomContext { std::vector randbytes(size_t len); /** Generate a random 32-bit integer. */ - uint32_t rand32() { return randbits(32); } + uint32_t rand32() noexcept { return randbits(32); } /** generate a random uint256. */ - uint256 rand256(); + uint256 rand256() noexcept; /** Generate a random boolean. */ - bool randbool() { return randbits(1); } + bool randbool() noexcept { return randbits(1); } // Compatibility with the C++11 UniformRandomBitGenerator concept typedef uint64_t result_type; static constexpr uint64_t min() { return 0; } static constexpr uint64_t max() { return std::numeric_limits::max(); } - inline uint64_t operator()() { return rand64(); } + inline uint64_t operator()() noexcept { return rand64(); } }; /** More efficient than using std::shuffle on a FastRandomContext. From 787d72f0fec3cfa9bb0edb9cec4f6e03a4668b98 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 4 Jan 2019 02:00:44 -0800 Subject: [PATCH 17/55] DRY: Implement GetRand using FastRandomContext::randrange --- src/random.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 73bfb2cd9d748..778b12311a83d 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -503,17 +503,7 @@ void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } uint64_t GetRand(uint64_t nMax) noexcept { - if (nMax == 0) - return 0; - - // The range of the random source must be a multiple of the modulus - // to give every possible output value an equal possibility - uint64_t nRange = (std::numeric_limits::max() / nMax) * nMax; - uint64_t nRand = 0; - do { - GetRandBytes((unsigned char*)&nRand, sizeof(nRand)); - } while (nRand >= nRange); - return (nRand % nMax); + return FastRandomContext().randrange(nMax); } int GetRandInt(int nMax) noexcept From 080deb3ad0e2dfd1fbd075f6005af099bcbe4a08 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 10 Jan 2019 18:19:50 -0800 Subject: [PATCH 18/55] Encapsulate RNGState better --- src/random.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 778b12311a83d..2bc0282996880 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -278,13 +278,14 @@ void LockingCallbackOpenSSL(int mode, int i, const char* file, int line); namespace { -struct RNGState { +class RNGState { Mutex m_mutex; unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; uint64_t m_counter GUARDED_BY(m_mutex) = 0; bool m_strongly_seeded GUARDED_BY(m_mutex) = false; std::unique_ptr m_mutex_openssl; +public: RNGState() noexcept { InitHardwareRand(); @@ -342,6 +343,8 @@ struct RNGState { memory_cleanse(buf, 64); return ret; } + + Mutex& GetOpenSSLMutex(int i) { return m_mutex_openssl[i]; } }; RNGState& GetRNGState() noexcept @@ -358,9 +361,9 @@ void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THRE RNGState& rng = GetRNGState(); if (mode & CRYPTO_LOCK) { - rng.m_mutex_openssl[i].lock(); + rng.GetOpenSSLMutex(i).lock(); } else { - rng.m_mutex_openssl[i].unlock(); + rng.GetOpenSSLMutex(i).unlock(); } } From 2b6381e5149be830b4d241c68554cebe5cac4b83 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 10 Jan 2019 18:34:17 -0800 Subject: [PATCH 19/55] Use secure allocator for RNG state --- src/random.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 2bc0282996880..784b2813d9d66 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -20,6 +20,8 @@ #include #include +#include "allocators.h" + #ifndef WIN32 #include #endif @@ -351,8 +353,8 @@ RNGState& GetRNGState() noexcept { // This C++11 idiom relies on the guarantee that static variable are initialized // on first call, even when multiple parallel calls are permitted. - static std::unique_ptr g_rng{new RNGState()}; - return *g_rng; + static std::vector> g_rng(1); + return g_rng[0]; } } From 4ffda1fd45d345d5ba7a5ecda7fc1a141632daf5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 13 Jan 2019 10:51:17 -0800 Subject: [PATCH 20/55] Document RNG design in random.h --- src/random.cpp | 8 ++++++++ src/random.h | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/random.cpp b/src/random.cpp index 784b2813d9d66..6c6636df1e861 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -282,6 +282,14 @@ namespace { class RNGState { Mutex m_mutex; + /* The RNG state consists of 256 bits of entropy, taken from the output of + * one operation's SHA512 output, and fed as input to the next one. + * Carrying 256 bits of entropy should be sufficient to guarantee + * unpredictability as long as any entropy source was ever unpredictable + * to an attacker. To protect against situations where an attacker might + * observe the RNG's state, fresh entropy is always mixed when + * GetStrongRandBytes is called. + */ unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; uint64_t m_counter GUARDED_BY(m_mutex) = 0; bool m_strongly_seeded GUARDED_BY(m_mutex) = false; diff --git a/src/random.h b/src/random.h index b799e25132dc7..d5398ea4b26a8 100644 --- a/src/random.h +++ b/src/random.h @@ -14,6 +14,49 @@ #include #include +/** + * Overall design of the RNG and entropy sources. + * + * We maintain a single global 256-bit RNG state for all high-quality randomness. + * The following (classes of) functions interact with that state by mixing in new + * entropy, and optionally extracting random output from it: + * + * - The GetRand*() class of functions, as well as construction of FastRandomContext objects, + * perform 'fast' seeding, consisting of mixing in: + * - A stack pointer (indirectly committing to calling thread and call stack) + * - A high-precision timestamp (rdtsc when available, c++ high_resolution_clock otherwise) + * - Hardware RNG (rdrand) when available. + * These entropy sources are very fast, and only designed to protect against situations + * where a VM state restore/copy results in multiple systems with the same randomness. + * FastRandomContext on the other hand does not protect against this once created, but + * is even faster (and acceptable to use inside tight loops). + * + * - The GetStrongRand*() class of function perform 'slow' seeding, including everything + * that fast seeding includes, but additionally: + * - OS entropy (/dev/urandom, getrandom(), ...). The application will terminate if + * this entropy source fails. + * - Bytes from OpenSSL's RNG (which itself may be seeded from various sources) + * - Another high-precision timestamp (indirectly committing to a benchmark of all the + * previous sources). + * These entropy sources are slower, but designed to make sure the RNG state contains + * fresh data that is unpredictable to attackers. + * + * - RandAddSeedSleep() seeds everything that fast seeding includes, but additionally: + * - A high-precision timestamp before and after sleeping 1ms. + * - (On Windows) Once every 10 minutes, performance monitoring data from the OS. + * These just exploit the fact the system is idle to improve the quality of the RNG + * slightly. + * + * On first use of the RNG (regardless of what function is called first), all entropy + * sources used in the 'slow' seeder are included, but also: + * - (On Windows) Performance monitoring data from the OS. + * - (On Windows) Through OpenSSL, the screen contents. + * + * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and + * (up to) the first 32 bytes of H are produced as output, while the last 32 bytes + * become the new RNG state. +*/ + /** * Generate random data via the internal PRNG. * From 67e336d3dc75b94a0d3b097327e043a6bbe07267 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 24 Jan 2019 18:40:02 -0800 Subject: [PATCH 21/55] Use RdSeed when available, and reduce RdRand load This introduces support for autodetecting and using the RdSeed instruction. In addition: * In SeedFast, only 64 bits of entropy are generated through RdRand (256 was relatively slow). * In SeedStartup, 256 bits of entropy are generated, using RdSeed (preferably) or RdRand (otherwise). --- src/random.cpp | 172 ++++++++++++++++++++++++++++++++++++++----------- src/random.h | 3 +- 2 files changed, 136 insertions(+), 39 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 6c6636df1e861..b7889e5ca8c3a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -78,25 +78,119 @@ static inline int64_t GetPerformanceCounter() noexcept } #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -static bool rdrand_supported = false; +static bool g_rdrand_supported = false; +static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; +static constexpr uint32_t CPUID_F7_EBX_RDSEED = 0x00040000; +#ifdef bit_RDRND +static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND"); +#endif +#ifdef bit_RDSEED +static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); +#endif +static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ + // We can't use __get_cpuid as it doesn't support subleafs. +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + static void InitHardwareRand() { uint32_t eax, ebx, ecx, edx; - if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { - rdrand_supported = true; + GetCPUID(1, 0, eax, ebx, ecx, edx); + if (ecx & CPUID_F1_ECX_RDRAND) { + g_rdrand_supported = true; + } + GetCPUID(7, 0, eax, ebx, ecx, edx); + if (ebx & CPUID_F7_EBX_RDSEED) { + g_rdseed_supported = true; } } static void ReportHardwareRand() { - if (rdrand_supported) { - // This must be done in a separate function, as HWRandInit() may be indirectly called - // from global constructors, before logging is initialized. + // This must be done in a separate function, as HWRandInit() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rdseed_supported) { + LogPrintf("Using RdSeed as additional entropy source\n"); + } + if (g_rdrand_supported) { LogPrintf("Using RdRand as an additional entropy source\n"); } } +/** Read 64 bits of entropy using rdrand. + * + * Must only be called when RdRand is supported. + */ +static uint64_t GetRdRand() noexcept +{ + // RdRand may very rarely fail. Invoke it up to 10 times in a loop to reduce this risk. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %rax + if (ok) break; + } + return r1; +#else +#error "RdRand is only supported on x86 and x86_64" +#endif +} + +/** Read 64 bits of entropy using rdseed. + * + * Must only be called when RdSeed is supported. + */ +static uint64_t GetRdSeed() noexcept +{ + // RdSeed may fail when the HW RNG is overloaded. Loop indefinitely until enough entropy is gathered, + // but pause after every failure. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + do { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %rax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return r1; +#else +#error "RdSeed is only supported on x86 and x86_64" +#endif +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -107,40 +201,40 @@ static void InitHardwareRand() {} static void ReportHardwareRand() {} #endif -static bool GetHardwareRand(unsigned char* ent32) noexcept { +/** Add 64 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareFast(CSHA512& hasher) noexcept { #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) - if (rdrand_supported) { - uint8_t ok; - // Not all assemblers support the rdrand instruction, write it in hex. -#ifdef __i386__ - for (int iter = 0; iter < 4; ++iter) { - uint32_t r1, r2; - __asm__ volatile (".byte 0x0f, 0xc7, 0xf0;" // rdrand %eax - ".byte 0x0f, 0xc7, 0xf2;" // rdrand %edx - "setc %2" : - "=a"(r1), "=d"(r2), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE32(ent32 + 8 * iter, r1); - WriteLE32(ent32 + 8 * iter + 4, r2); - } -#else - uint64_t r1, r2, r3, r4; - __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0, " // rdrand %rax - "0x48, 0x0f, 0xc7, 0xf3, " // rdrand %rbx - "0x48, 0x0f, 0xc7, 0xf1, " // rdrand %rcx - "0x48, 0x0f, 0xc7, 0xf2; " // rdrand %rdx - "setc %4" : - "=a"(r1), "=b"(r2), "=c"(r3), "=d"(r4), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE64(ent32, r1); - WriteLE64(ent32 + 8, r2); - WriteLE64(ent32 + 16, r3); - WriteLE64(ent32 + 24, r4); + if (g_rdrand_supported) { + uint64_t out = GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif - return true; +} + +/** Add 256 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareSlow(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + // When we want 256 bits of entropy, prefer RdSeed over RdRand, as it's + // guaranteed to produce independent randomness on every call. + if (g_rdseed_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRdSeed(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } + // When falling back to RdRand, XOR the result of 1024 results. + // This guarantees a reseeding occurs between each. + if (g_rdrand_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = 0; + for (int j = 0; j < 1024; ++j) out ^= GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; } #endif - return false; } static void RandAddSeedPerfmon(CSHA512& hasher) @@ -407,8 +501,7 @@ static void SeedFast(CSHA512& hasher) noexcept hasher.Write((const unsigned char*)&ptr, sizeof(ptr)); // Hardware randomness is very fast when available; use it always. - bool have_hw_rand = GetHardwareRand(buffer); - if (have_hw_rand) hasher.Write(buffer, sizeof(buffer)); + SeedHardwareFast(hasher); // High-precision timestamp SeedTimestamp(hasher); @@ -460,6 +553,9 @@ static void SeedStartup(CSHA512& hasher) noexcept RAND_screen(); #endif + // Gather 256 bits of hardware randomness, if available + SeedHardwareSlow(hasher); + // Everything that the 'slow' seeder includes. SeedSlow(hasher); diff --git a/src/random.h b/src/random.h index d5398ea4b26a8..42b1356585c7c 100644 --- a/src/random.h +++ b/src/random.h @@ -25,7 +25,7 @@ * perform 'fast' seeding, consisting of mixing in: * - A stack pointer (indirectly committing to calling thread and call stack) * - A high-precision timestamp (rdtsc when available, c++ high_resolution_clock otherwise) - * - Hardware RNG (rdrand) when available. + * - 64 bits from the hardware RNG (rdrand) when available. * These entropy sources are very fast, and only designed to protect against situations * where a VM state restore/copy results in multiple systems with the same randomness. * FastRandomContext on the other hand does not protect against this once created, but @@ -49,6 +49,7 @@ * * On first use of the RNG (regardless of what function is called first), all entropy * sources used in the 'slow' seeder are included, but also: + * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. * - (On Windows) Through OpenSSL, the screen contents. * From 0190decbefb3136695f2bc56e7a088c27f6d5c06 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Dec 2018 15:50:31 -0800 Subject: [PATCH 22/55] Add hash strengthening to the RNG Once every minute, this will feed the RNG state through repeated SHA512 for 10ms. The timings of that operation are used as entropy source as well. --- src/random.cpp | 58 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index b7889e5ca8c3a..631ccb3b80cbf 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -237,6 +237,34 @@ static void SeedHardwareSlow(CSHA512& hasher) noexcept { #endif } +/** Use repeated SHA512 to strengthen the randomness in seed32, and feed into hasher. */ +static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA512& hasher) noexcept +{ + CSHA512 inner_hasher; + inner_hasher.Write(seed, sizeof(seed)); + + // Hash loop + unsigned char buffer[64]; + int64_t stop = GetTimeMicros() + microseconds; + do { + for (int i = 0; i < 1000; ++i) { + inner_hasher.Finalize(buffer); + inner_hasher.Reset(); + inner_hasher.Write(buffer, sizeof(buffer)); + } + // Benchmark operation and feed it into outer hasher. + int64_t perf = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perf, sizeof(perf)); + } while (GetTimeMicros() < stop); + + // Produce output from inner state and feed it to outer hasher. + inner_hasher.Finalize(buffer); + hasher.Write(buffer, sizeof(buffer)); + // Try to clean up. + inner_hasher.Reset(); + memory_cleanse(buffer, sizeof(buffer)); +} + static void RandAddSeedPerfmon(CSHA512& hasher) { #ifdef WIN32 @@ -529,7 +557,23 @@ static void SeedSlow(CSHA512& hasher) noexcept SeedTimestamp(hasher); } -static void SeedSleep(CSHA512& hasher) +/** Extract entropy from rng, strengthen it, and feed it into hasher. */ +static void SeedStrengthen(CSHA512& hasher, RNGState& rng) noexcept +{ + static std::atomic last_strengthen{0}; + int64_t last_time = last_strengthen.load(); + int64_t current_time = GetTimeMicros(); + if (current_time > last_time + 60000000) { // Only run once a minute + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen it for 10ms (100ms on first run), and feed it into hasher. + Strengthen(strengthen_seed, last_time == 0 ? 100000 : 10000, hasher); + last_strengthen = current_time; + } +} + +static void SeedSleep(CSHA512& hasher, RNGState& rng) { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -545,9 +589,12 @@ static void SeedSleep(CSHA512& hasher) // Windows performance monitor data (once every 10 minutes) RandAddSeedPerfmon(hasher); + + // Strengthen every minute + SeedStrengthen(hasher, rng); } -static void SeedStartup(CSHA512& hasher) noexcept +static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept { #ifdef WIN32 RAND_screen(); @@ -561,6 +608,9 @@ static void SeedStartup(CSHA512& hasher) noexcept // Windows performance monitor data. RandAddSeedPerfmon(hasher); + + // Strengthen + SeedStrengthen(hasher, rng); } enum class RNGLevel { @@ -585,7 +635,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedSlow(hasher); break; case RNGLevel::SLEEP: - SeedSleep(hasher); + SeedSleep(hasher, rng); break; } @@ -593,7 +643,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) if (!rng.MixExtract(out, num, std::move(hasher), false)) { // On the first invocation, also seed with SeedStartup(). CSHA512 startup_hasher; - SeedStartup(startup_hasher); + SeedStartup(startup_hasher, rng); rng.MixExtract(out, num, std::move(startup_hasher), true); } From 7b332235842ea5fdd58f3544ca25d8eb89c8ac94 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 21 Jan 2019 13:25:14 -0800 Subject: [PATCH 23/55] Document strenghtening --- src/random.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/random.h b/src/random.h index 42b1356585c7c..5c9f3d609f4fa 100644 --- a/src/random.h +++ b/src/random.h @@ -44,6 +44,7 @@ * - RandAddSeedSleep() seeds everything that fast seeding includes, but additionally: * - A high-precision timestamp before and after sleeping 1ms. * - (On Windows) Once every 10 minutes, performance monitoring data from the OS. + - - Once every minute, strengthen the entropy for 10 ms using repeated SHA512. * These just exploit the fact the system is idle to improve the quality of the RNG * slightly. * @@ -52,6 +53,7 @@ * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. * - (On Windows) Through OpenSSL, the screen contents. + * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and * (up to) the first 32 bytes of H are produced as output, while the last 32 bytes From c82e359135f0788e3a74df8ed603e89d0f853dbf Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 1 Feb 2019 14:06:32 -0800 Subject: [PATCH 24/55] test: Make bloom tests deterministic --- src/random.cpp | 4 +++- src/test/bloom_tests.cpp | 18 ++++++++---------- src/test/random_tests.cpp | 10 ++++++++++ src/test/test_pivx.h | 5 +++++ 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 631ccb3b80cbf..c4209cbead5c7 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -660,9 +660,11 @@ void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNG void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } +bool g_mock_deterministic_tests{false}; + uint64_t GetRand(uint64_t nMax) noexcept { - return FastRandomContext().randrange(nMax); + return FastRandomContext(g_mock_deterministic_tests).randrange(nMax); } int GetRandInt(int nMax) noexcept diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index d8646534f4c91..9b99d10fca7c4 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -463,6 +463,9 @@ static std::vector RandomData() BOOST_AUTO_TEST_CASE(rolling_bloom) { + SeedInsecureRand(/* deterministic */ true); + g_mock_deterministic_tests = true; + // last-100-entry, 1% false positive: CRollingBloomFilter rb1(100, 0.01); @@ -487,12 +490,8 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) if (rb1.contains(RandomData())) ++nHits; } - // Run test_bitcoin with --log_level=message to see BOOST_TEST_MESSAGEs: - BOOST_TEST_MESSAGE("RollingBloomFilter got " << nHits << " false positives (~100 expected)"); - - // Insanely unlikely to get a fp count outside this range: - BOOST_CHECK(nHits > 25); - BOOST_CHECK(nHits < 175); + // Expect about 100 hits + BOOST_CHECK_EQUAL(nHits, 75); BOOST_CHECK(rb1.contains(data[DATASIZE - 1])); rb1.reset(); @@ -516,10 +515,8 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) if (rb1.contains(data[i])) ++nHits; } - // Expect about 5 false positives, more than 100 means - // something is definitely broken. - BOOST_TEST_MESSAGE("RollingBloomFilter got " << nHits << " false positives (~5 expected)"); - BOOST_CHECK(nHits < 100); + // Expect about 5 false positives + BOOST_CHECK_EQUAL(nHits, 6); // last-1000-entry, 0.01% false positive: CRollingBloomFilter rb2(1000, 0.001); @@ -530,6 +527,7 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) for (int i = 0; i < DATASIZE; i++) { BOOST_CHECK(rb2.contains(data[i])); } + g_mock_deterministic_tests = false; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 424bffdeb9120..fe0ea4b7ad476 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -21,9 +21,14 @@ BOOST_AUTO_TEST_CASE(osrandom_tests) BOOST_AUTO_TEST_CASE(fastrandom_tests) { // Check that deterministic FastRandomContexts are deterministic + g_mock_deterministic_tests = true; FastRandomContext ctx1(true); FastRandomContext ctx2(true); + for (int i = 10; i > 0; --i) { + BOOST_CHECK_EQUAL(GetRand(std::numeric_limits::max()), uint64_t{10393729187455219830U}); + BOOST_CHECK_EQUAL(GetRandInt(std::numeric_limits::max()), int{769702006}); + } BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); @@ -38,6 +43,11 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) BOOST_CHECK(ctx1.randbytes(50) == ctx2.randbytes(50)); // Check that a nondeterministic ones are not + g_mock_deterministic_tests = false; + for (int i = 10; i > 0; --i) { + BOOST_CHECK(GetRand(std::numeric_limits::max()) != uint64_t{10393729187455219830U}); + BOOST_CHECK(GetRandInt(std::numeric_limits::max()) != int{769702006}); + } { FastRandomContext ctx3, ctx4; BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal diff --git a/src/test/test_pivx.h b/src/test/test_pivx.h index 54bfa9accf0e2..42f1b499cac88 100644 --- a/src/test/test_pivx.h +++ b/src/test/test_pivx.h @@ -13,6 +13,11 @@ extern FastRandomContext insecure_rand_ctx; +/** + * Flag to make GetRand in random.h return the same number + */ +extern bool g_mock_deterministic_tests; + static inline void SeedInsecureRand(bool deterministic = false) { insecure_rand_ctx = FastRandomContext(deterministic); From 28c9cdb694013d71d8ff421123d498bc706e5b59 Mon Sep 17 00:00:00 2001 From: practicalswift Date: Wed, 30 Jan 2019 21:53:11 +0100 Subject: [PATCH 25/55] tests: Add script checking for deterministic line coverage --- .../devtools/test_deterministic_coverage.sh | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100755 contrib/devtools/test_deterministic_coverage.sh diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh new file mode 100755 index 0000000000000..a9f02baa377dd --- /dev/null +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Test for deterministic coverage across unit test runs. + +export LC_ALL=C + +# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. +# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. +GCOV_EXECUTABLE="gcov" + +# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. +NON_DETERMINISTIC_TESTS=( + "DoS_tests/DoS_mapOrphans" # RandomOrphan() +) + +TEST_BITCOIN_BINARY="src/test/test_pivx" + +print_usage() { + echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" +} + +N_TEST_RUNS=2 +BOOST_TEST_RUN_FILTERS="" +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ [a-z] ]]; then + BOOST_TEST_RUN_FILTERS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi +if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then + BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" +else + echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" + echo +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +if ! command -v gcovr > /dev/null; then + echo "Error: gcovr not installed. Exiting." + exit 1 +fi + +if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + exit 1 +fi + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + exit 1 +fi + +get_covr_filename() { + echo "gcovr.run-$1.txt" +} + +TEST_RUN_ID=0 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + find src/ -type f -name "*.gcda" -exec rm {} \; + if [[ $(get_file_suffix_count gcda) != 0 ]]; then + echo "Error: Stale *.gcda files found. Exiting." + exit 1 + fi + TEST_OUTPUT_TEMPFILE=$(mktemp) + if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then + cat "${TEST_OUTPUT_TEMPFILE}" + rm "${TEST_OUTPUT_TEMPFILE}" + exit 1 + fi + rm "${TEST_OUTPUT_TEMPFILE}" + if [[ $(get_file_suffix_count gcda) == 0 ]]; then + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + exit 1 + fi + GCOVR_TEMPFILE=$(mktemp) + if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then + echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." + exit 1 + fi + GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) + mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" + if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then + echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." + exit 1 + fi + if [[ ${TEST_RUN_ID} != 1 ]]; then + COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "${COVERAGE_DIFF}" + exit 1 + fi + rm "${GCOVR_FILENAME}" + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit From 6966aa9083e661a4c14a92bac73a98de3eecee00 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Fri, 1 Mar 2019 08:03:58 -0800 Subject: [PATCH 26/55] Add ChaCha20 encryption option (XOR) --- src/crypto/chacha20.cpp | 132 +++++++++++++++++++++++++++++++++++++- src/crypto/chacha20.h | 18 ++++-- src/random.cpp | 2 +- src/random.h | 2 +- src/test/crypto_tests.cpp | 47 +++++++++++--- 5 files changed, 185 insertions(+), 16 deletions(-) diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 70c472b74b9d8..b147c7e2322b5 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -71,7 +71,7 @@ void ChaCha20::Seek(uint64_t pos) input[13] = pos >> 32; } -void ChaCha20::Output(unsigned char* c, size_t bytes) +void ChaCha20::Keystream(unsigned char* c, size_t bytes) { uint32_t x[16]; uint32_t j[16]; @@ -126,3 +126,133 @@ void ChaCha20::Output(unsigned char* c, size_t bytes) c += 64; } } + +void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = nullptr; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + // if m has fewer than 64 bytes available, copy m to tmp and + // read from tmp instead + for (i = 0;i < bytes;++i) tmp[i] = m[i]; + m = tmp; + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + x0 ^= ReadLE32(m + 0); + x1 ^= ReadLE32(m + 4); + x2 ^= ReadLE32(m + 8); + x3 ^= ReadLE32(m + 12); + x4 ^= ReadLE32(m + 16); + x5 ^= ReadLE32(m + 20); + x6 ^= ReadLE32(m + 24); + x7 ^= ReadLE32(m + 28); + x8 ^= ReadLE32(m + 32); + x9 ^= ReadLE32(m + 36); + x10 ^= ReadLE32(m + 40); + x11 ^= ReadLE32(m + 44); + x12 ^= ReadLE32(m + 48); + x13 ^= ReadLE32(m + 52); + x14 ^= ReadLE32(m + 56); + x15 ^= ReadLE32(m + 60); + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + m += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index a305977bcd5f4..5a4674f4a8afe 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -8,7 +8,8 @@ #include #include -/** A PRNG class for ChaCha20. */ +/** A class for ChaCha20 256-bit stream cipher developed by Daniel J. Bernstein + https://cr.yp.to/chacha/chacha-20080128.pdf */ class ChaCha20 { private: @@ -17,10 +18,17 @@ class ChaCha20 public: ChaCha20(); ChaCha20(const unsigned char* key, size_t keylen); - void SetKey(const unsigned char* key, size_t keylen); - void SetIV(uint64_t iv); - void Seek(uint64_t pos); - void Output(unsigned char* output, size_t bytes); + void SetKey(const unsigned char* key, size_t keylen); //!< set key with flexible keylength; 256bit recommended */ + void SetIV(uint64_t iv); // set the 64bit nonce + void Seek(uint64_t pos); // set the 64bit block counter + + /** outputs the keystream of size into */ + void Keystream(unsigned char* c, size_t bytes); + + /** enciphers the message of length and write the enciphered representation into + * Used for encryption and decryption (XOR) + */ + void Crypt(const unsigned char* input, unsigned char* output, size_t bytes); }; #endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/random.cpp b/src/random.cpp index c4209cbead5c7..31c308bd62658 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -702,7 +702,7 @@ std::vector FastRandomContext::randbytes(size_t len) if (requires_seed) RandomSeed(); std::vector ret(len); if (len > 0) { - rng.Output(&ret[0], len); + rng.Keystream(&ret[0], len); } return ret; } diff --git a/src/random.h b/src/random.h index 5c9f3d609f4fa..c6d1ceec0d736 100644 --- a/src/random.h +++ b/src/random.h @@ -114,7 +114,7 @@ class FastRandomContext { if (requires_seed) { RandomSeed(); } - rng.Output(bytebuf, sizeof(bytebuf)); + rng.Keystream(bytebuf, sizeof(bytebuf)); bytebuf_size = sizeof(bytebuf); } diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index b34c1c03763aa..181e3ec4518c1 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -66,17 +66,36 @@ void TestHMACSHA512(const std::string &hexkey, const std::string &hexin, const s TestVector(CHMAC_SHA512(&key[0], key.size()), ParseHex(hexin), ParseHex(hexout)); } -void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) +void TestChaCha20(const std::string &hex_message, const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) { std::vector key = ParseHex(hexkey); + std::vector m = ParseHex(hex_message); ChaCha20 rng(key.data(), key.size()); rng.SetIV(nonce); rng.Seek(seek); std::vector out = ParseHex(hexout); std::vector outres; outres.resize(out.size()); - rng.Output(outres.data(), outres.size()); + assert(hex_message.empty() || m.size() == out.size()); + + // perform the ChaCha20 round(s), if message is provided it will output the encrypted ciphertext otherwise the keystream + if (!hex_message.empty()) { + rng.Crypt(m.data(), outres.data(), outres.size()); + } else { + rng.Keystream(outres.data(), outres.size()); + } BOOST_CHECK(out == outres); + if (!hex_message.empty()) { + // Manually XOR with the keystream and compare the output + rng.SetIV(nonce); + rng.Seek(seek); + std::vector only_keystream(outres.size()); + rng.Keystream(only_keystream.data(), only_keystream.size()); + for (size_t i = 0; i != m.size(); i++) { + outres[i] = m[i] ^ only_keystream[i]; + } + BOOST_CHECK(out == outres); + } } void TestAES128(const std::string &hexkey, const std::string &hexin, const std::string &hexout) @@ -421,25 +440,37 @@ BOOST_AUTO_TEST_CASE(rfc6979_hmac_sha256) BOOST_AUTO_TEST_CASE(chacha20_testvector) { // Test vector from RFC 7539 - TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + + // test encryption + TestChaCha20("4c616469657320616e642047656e746c656d656e206f662074686520636c617373206f66202739393a204966204920636f756" + "c64206f6666657220796f75206f6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73637265656e" + "20776f756c642062652069742e", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + "6e2e359a2568f98041ba0728dd0d6981e97e7aec1d4360c20a27afccfd9fae0bf91b65c5524733ab8f593dabcd62b3571639d" + "624e65152ab8f530c359f0861d807ca0dbf500d6a6156a38e088a22b65e52bc514d16ccf806818ce91ab77937365af90bbf74" + "a35be6b40b8eedf2785e42874d" + ); + + // test keystream output + TestChaCha20("", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b9334794cb" "a40c63e34cdea212c4cf07d41b769a6749f3f630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53ac40c5945398b6eda1a" "832c89c167eacd901d7e2bf363"); // Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000000", 0, 0, "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b" "8f41518a11cc387b669b2ee6586"); - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000001", 0, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000001", 0, 0, "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d79" "2b1c43fea817e9ad275ae546963"); - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b52770" "62eb7a0433e445f41e3"); - TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 1, 0, + TestChaCha20("", "0000000000000000000000000000000000000000000000000000000000000000", 1, 0, "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc4" "97a0b466e7d6bbdb0041b2f586b"); - TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, + TestChaCha20("", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3b" "e59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc1" "18be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5" From 79e7fd3c823b991f90f51f08c4ad1451ffa61fe1 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Tue, 5 Mar 2019 01:50:50 -0800 Subject: [PATCH 27/55] Add ChaCha20 bench --- src/Makefile.bench.include | 1 + src/bench/chacha20.cpp | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 src/bench/chacha20.cpp diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 4c450080319e7..59bd3d3b0a9dc 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -15,6 +15,7 @@ bench_bench_pivx_SOURCES = \ bench/base58.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ + bench/chacha20.cpp \ bench/crypto_hash.cpp \ bench/lockedpool.cpp \ bench/perf.cpp \ diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp new file mode 100644 index 0000000000000..7c25adf9cbaf2 --- /dev/null +++ b/src/bench/chacha20.cpp @@ -0,0 +1,46 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include + +/* Number of bytes to process per iteration */ +static const uint64_t BUFFER_SIZE_TINY = 64; +static const uint64_t BUFFER_SIZE_SMALL = 256; +static const uint64_t BUFFER_SIZE_LARGE = 1024*1024; + +static void CHACHA20(benchmark::State& state, size_t buffersize) +{ + std::vector key(32,0); + ChaCha20 ctx(key.data(), key.size()); + ctx.SetIV(0); + ctx.Seek(0); + std::vector in(buffersize,0); + std::vector out(buffersize,0); + while (state.KeepRunning()) { + ctx.Crypt(in.data(), out.data(), in.size()); + } +} + +static void CHACHA20_64BYTES(benchmark::State& state) +{ + CHACHA20(state, BUFFER_SIZE_TINY); +} + +static void CHACHA20_256BYTES(benchmark::State& state) +{ + CHACHA20(state, BUFFER_SIZE_SMALL); +} + +static void CHACHA20_1MB(benchmark::State& state) +{ + CHACHA20(state, BUFFER_SIZE_LARGE); +} + +BENCHMARK(CHACHA20_64BYTES); +BENCHMARK(CHACHA20_256BYTES); +BENCHMARK(CHACHA20_1MB); From 22a71216df63f722044478edbd535ed9566840a5 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Mon, 29 Mar 2021 01:25:35 -0700 Subject: [PATCH 28/55] Fix non-deterministic coverage of test DoS_mapOrphans --- .../devtools/test_deterministic_coverage.sh | 2 +- src/test/DoS_tests.cpp | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index a9f02baa377dd..3695b62af0fd3 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -14,7 +14,7 @@ GCOV_EXECUTABLE="gcov" # Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. NON_DETERMINISTIC_TESTS=( - "DoS_tests/DoS_mapOrphans" # RandomOrphan() + "dummy_tests" # We currently don't have any unit tests that have determinism issues ) TEST_BITCOIN_BINARY="src/test/test_pivx" diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index ac21d069ba70e..e3d0f01076ddf 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -9,9 +9,11 @@ #include "test/test_pivx.h" +#include "arith_uint256.h" #include "keystore.h" #include "net_processing.h" #include "net.h" +#include "pubkey.h" #include "pow.h" #include "script/sign.h" #include "serialize.h" @@ -139,10 +141,26 @@ CTransactionRef RandomOrphan() return it->second.tx; } +static void MakeNewKeyWithFastRandomContext(CKey& key) +{ + std::vector keydata; + keydata = insecure_rand_ctx.randbytes(32); + key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn*/ true); + assert(key.IsValid()); +} + BOOST_AUTO_TEST_CASE(DoS_mapOrphans) { + // This test had non-deterministic coverage due to + // randomly selected seeds. + // This seed is chosen so that all branches of the function + // ecdsa_signature_parse_der_lax are executed during this test. + // Specifically branches that run only when an ECDSA + // signature's R and S values have leading zeros. + insecure_rand_ctx = FastRandomContext(ArithToUint256(arith_uint256(33))); + CKey key; - key.MakeNewKey(true); + MakeNewKeyWithFastRandomContext(key); CBasicKeyStore keystore; keystore.AddKey(key); From ada98689c9c8d533dc5465d137c287700ce017e9 Mon Sep 17 00:00:00 2001 From: fanquake Date: Sun, 13 Oct 2019 16:28:58 -0700 Subject: [PATCH 29/55] gui: remove OpenSSL PRNG seeding (Windows, Qt only) This removes the code introduced in [#4399](https://github.com/bitcoin/bitcoin/pull/4399) that attempts to add additional entroy to the OpenSSL PRNG using Windows messages. Note that this is specific to bitcoin-qt running on Windows. ``` RAND_event() collects the entropy from Windows events such as mouse movements and other user interaction. It should be called with the iMsg, wParam and lParam arguments of all messages sent to the window procedure. It will estimate the entropy contained in the event message (if any), and add it to the PRNG. The program can then process the messages as usual. ``` Besides BIP70, this is the last place we are directly using OpenSSL in the GUI code. All other OpenSSL usage is in random.cpp. Note that we are still also doing Windows specific entropy gathering in multiple other places. Such as [RandAddSeedPerfmon](https://github.com/bitcoin/bitcoin/blob/master/src/random.cpp#L268) and [RAND_screen()](https://github.com/bitcoin/bitcoin/blob/master/src/random.cpp#L600). Also note that if RAND_event returns 0 (PRNG has NOT been seeded with enough data), we're just logging a message and continuing on, which seems less than ideal. --- src/qt/winshutdownmonitor.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 082f5587bf86d..bdad740f7d2c4 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -7,14 +7,11 @@ #if defined(Q_OS_WIN) #include "init.h" -#include "util.h" #include #include -#include - // If we don't want a message to be processed by Qt, return true and set result to // the value that the window procedure should return. Otherwise return false. bool WinShutdownMonitor::nativeEventFilter(const QByteArray& eventType, void* pMessage, long* pnResult) @@ -23,16 +20,6 @@ bool WinShutdownMonitor::nativeEventFilter(const QByteArray& eventType, void* pM MSG* pMsg = static_cast(pMessage); - // Seed OpenSSL PRNG with Windows event data (e.g. mouse movements and other user interactions) - if (RAND_event(pMsg->message, pMsg->wParam, pMsg->lParam) == 0) { - // Warn only once as this is performance-critical - static bool warned = false; - if (!warned) { - LogPrintf("%s: OpenSSL RAND_event() failed to seed OpenSSL PRNG with enough data.\n", __func__); - warned = true; - } - } - switch (pMsg->message) { case WM_QUERYENDSESSION: { // Initiate a client shutdown after receiving a WM_QUERYENDSESSION and block From 5eed08c62b912813ea45e8e766775c48d2ecb3dd Mon Sep 17 00:00:00 2001 From: fanquake Date: Fri, 18 Oct 2019 14:16:13 -0400 Subject: [PATCH 30/55] random: remove call to RAND_screen() (Windows only) Follow up to https://github.com/bitcoin/bitcoin/pull/17151 where there were multiple calls to also remove our call to RAND_screen(). --- src/random.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 31c308bd62658..ed1ae35a15a4e 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -596,10 +596,6 @@ static void SeedSleep(CSHA512& hasher, RNGState& rng) static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept { -#ifdef WIN32 - RAND_screen(); -#endif - // Gather 256 bits of hardware randomness, if available SeedHardwareSlow(hasher); From 630931fa528e0e7d74dff2491af0bab2d47f7ad5 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Mon, 29 Mar 2021 21:21:25 -0700 Subject: [PATCH 31/55] break circular dependency: random/sync -> util -> random/sync --- src/random.cpp | 6 ++++-- src/sync.cpp | 9 +++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index ed1ae35a15a4e..9583014423ab0 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -12,8 +12,9 @@ #include "compat.h" // for Windows API #include #endif -#include "util.h" // for LogPrint() -#include "utilstrencodings.h" // for GetTime() +#include "logging.h" // for LogPrint() +#include "sync.h" +#include "utiltime.h" // for GetTime() #include #include @@ -23,6 +24,7 @@ #include "allocators.h" #ifndef WIN32 +#include #include #endif diff --git a/src/sync.cpp b/src/sync.cpp index 01403b72d4f3f..bef4212a53405 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -5,15 +5,16 @@ #include "sync.h" -#include -#include - -#include "util.h" +#include "logging.h" #include "utilstrencodings.h" #include "util/threadnames.h" #include +#include +#include +#include + #ifdef DEBUG_LOCKCONTENTION #if !defined(HAVE_THREAD_LOCAL) static_assert(false, "thread_local is not supported"); From 909473eb516504f203b25df74a83f958c13b32ea Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Mon, 29 Mar 2021 21:24:51 -0700 Subject: [PATCH 32/55] Fix FreeBSD build by including utilstrencodings.h `random.cpp` needs to explicitly include `utilstrencodings.h` to get `ARRAYLEN`. This fixes the FreeBSD build. --- src/random.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/random.cpp b/src/random.cpp index 9583014423ab0..df0f0ce87a33a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -39,6 +39,7 @@ #include #endif #ifdef HAVE_SYSCTL_ARND +#include "utilstrencodings.h" #include #endif From fccd2b8002614b4dda48b30956c6f0fb9f28b163 Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 16 Oct 2019 20:20:59 -0400 Subject: [PATCH 33/55] doc: correct function name in ReportHardwareRand() --- src/random.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/random.cpp b/src/random.cpp index df0f0ce87a33a..636fd677878e5 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -116,7 +116,7 @@ static void InitHardwareRand() static void ReportHardwareRand() { - // This must be done in a separate function, as HWRandInit() may be indirectly called + // This must be done in a separate function, as InitHardwareRand() may be indirectly called // from global constructors, before logging is initialized. if (g_rdseed_supported) { LogPrintf("Using RdSeed as additional entropy source\n"); From 27cf995d328be701d726e3a72f5ed22e2f524173 Mon Sep 17 00:00:00 2001 From: fanquake Date: Sat, 26 Oct 2019 05:05:59 -0700 Subject: [PATCH 34/55] doc: minor corrections in random.cpp This should have been part of #17151. --- src/random.cpp | 4 ++-- src/random.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 636fd677878e5..121e09ad3daff 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -13,7 +13,7 @@ #include #endif #include "logging.h" // for LogPrint() -#include "sync.h" +#include "sync.h" // for Mutex #include "utiltime.h" // for GetTime() #include @@ -716,7 +716,7 @@ bool Random_SanityCheck() uint64_t start = GetPerformanceCounter(); /* This does not measure the quality of randomness, but it does test that - * OSRandom() overwrites all 32 bytes of the output given a maximum + * GetOSRand() overwrites all 32 bytes of the output given a maximum * number of tries. */ static const ssize_t MAX_TRIES = 1024; diff --git a/src/random.h b/src/random.h index c6d1ceec0d736..d831144def8db 100644 --- a/src/random.h +++ b/src/random.h @@ -52,7 +52,6 @@ * sources used in the 'slow' seeder are included, but also: * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. - * - (On Windows) Through OpenSSL, the screen contents. * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and From 52b5336f2cf57d59bac5bc7832562f206f78f733 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 26 Oct 2019 12:20:12 -0700 Subject: [PATCH 35/55] [MOVEONLY] Move perfmon data gathering to new randomenv module --- CMakeLists.txt | 1 + src/Makefile.am | 2 ++ src/random.cpp | 51 +++++++---------------------------- src/randomenv.cpp | 69 +++++++++++++++++++++++++++++++++++++++++++++++ src/randomenv.h | 17 ++++++++++++ 5 files changed, 98 insertions(+), 42 deletions(-) create mode 100644 src/randomenv.cpp create mode 100644 src/randomenv.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7155f1b729623..2dca15e626e37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -431,6 +431,7 @@ set(UTIL_SOURCES ./src/fs.cpp ./src/logging.cpp ./src/random.cpp + ./src/randomenv.cpp ./src/rpc/protocol.cpp ./src/sync.cpp ./src/threadinterrupt.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 94dd80a921093..e4e2e58d96b31 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -239,6 +239,7 @@ BITCOIN_CORE_H = \ protocol.h \ pubkey.h \ random.h \ + randomenv.h \ reverselock.h \ reverse_iterate.h \ rpc/client.h \ @@ -538,6 +539,7 @@ libbitcoin_util_a_SOURCES = \ interfaces/handler.cpp \ logging.cpp \ random.cpp \ + randomenv.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ support/lockedpool.cpp \ diff --git a/src/random.cpp b/src/random.cpp index 121e09ad3daff..612bbb2ecec1a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -21,6 +21,8 @@ #include #include +#include "randomenv.h" + #include "allocators.h" #ifndef WIN32 @@ -268,44 +270,6 @@ static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA51 memory_cleanse(buffer, sizeof(buffer)); } -static void RandAddSeedPerfmon(CSHA512& hasher) -{ -#ifdef WIN32 - // Don't need this on Linux, OpenSSL automatically uses /dev/urandom - // Seed with the entire set of perfmon data - - // This can take up to 2 seconds, so only do it every 10 minutes - static int64_t nLastPerfmon; - if (GetTime() < nLastPerfmon + 10 * 60) - return; - nLastPerfmon = GetTime(); - - std::vector vData(250000, 0); - long ret = 0; - unsigned long nSize = 0; - const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data - while (true) { - nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", NULL, NULL, vData.data(), &nSize); - if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) - break; - vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially - } - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) { - hasher.Write(vData.data(), nSize); - memory_cleanse(vData.data(), nSize); - } else { - // Performance data is only a best-effort attempt at improving the - // situation when the OS randomness (and other sources) aren't - // adequate. As a result, failure to read it is isn't considered critical, - // so we don't call RandFailure(). - // TODO: Add logging when the logger is made functional before global - // constructors have been invoked. - } -#endif -} - #ifndef WIN32 /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. @@ -590,8 +554,8 @@ static void SeedSleep(CSHA512& hasher, RNGState& rng) // High-precision timestamp after sleeping (as we commit to both the time before and after, this measures the delay) SeedTimestamp(hasher); - // Windows performance monitor data (once every 10 minutes) - RandAddSeedPerfmon(hasher); + // Dynamic environment data (performance monitoring, ...; once every 10 minutes) + RandAddDynamicEnv(hasher); // Strengthen every minute SeedStrengthen(hasher, rng); @@ -605,8 +569,11 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept // Everything that the 'slow' seeder includes. SeedSlow(hasher); - // Windows performance monitor data. - RandAddSeedPerfmon(hasher); + // Dynamic environment data + RandAddDynamicEnv(hasher); + + // Static environment data + RandAddStaticEnv(hasher); // Strengthen SeedStrengthen(hasher, rng); diff --git a/src/randomenv.cpp b/src/randomenv.cpp new file mode 100644 index 0000000000000..b54ce38385c41 --- /dev/null +++ b/src/randomenv.cpp @@ -0,0 +1,69 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "randomenv.h" + +#include "crypto/sha512.h" +#include "support/cleanse.h" +#include "utiltime.h" // for GetTime() +#ifdef WIN32 +#include "compat.h" // for Windows API +#endif + +#include +#include + +#include + +namespace { + +void RandAddSeedPerfmon(CSHA512& hasher) +{ +#ifdef WIN32 + // Don't need this on Linux, OpenSSL automatically uses /dev/urandom + // Seed with the entire set of perfmon data + + // This can take up to 2 seconds, so only do it every 10 minutes + static int64_t nLastPerfmon; + if (GetTime() < nLastPerfmon + 10 * 60) + return; + nLastPerfmon = GetTime(); + + std::vector vData(250000, 0); + long ret = 0; + unsigned long nSize = 0; + const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data + while (true) { + nSize = vData.size(); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); + if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) + break; + vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially + } + RegCloseKey(HKEY_PERFORMANCE_DATA); + if (ret == ERROR_SUCCESS) { + hasher.Write(vData.data(), nSize); + memory_cleanse(vData.data(), nSize); + } else { + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it is isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. + } +#endif +} + +} // namespace + +void RandAddDynamicEnv(CSHA512& hasher) +{ + RandAddSeedPerfmon(hasher); +} + +void RandAddStaticEnv(CSHA512& hasher) +{ +} diff --git a/src/randomenv.h b/src/randomenv.h new file mode 100644 index 0000000000000..6c49d3deede07 --- /dev/null +++ b/src/randomenv.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOMENV_H +#define BITCOIN_RANDOMENV_H + +#include "crypto/sha512.h" + +/** Gather non-cryptographic environment data that changes over time. */ +void RandAddDynamicEnv(CSHA512& hasher); + +/** Gather non-cryptographic environment data that does not change over time. */ +void RandAddStaticEnv(CSHA512& hasher); + +#endif From 7bde8b72475e02d13cee0dd6f954c8e1c9dc491f Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Tue, 30 Mar 2021 04:53:49 -0700 Subject: [PATCH 36/55] [MOVEONLY] Move cpuid code from random to compat/cpuid --- src/Makefile.am | 1 + src/compat/cpuid.h | 24 ++++++++++++++++++++++++ src/random.cpp | 20 +++----------------- 3 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 src/compat/cpuid.h diff --git a/src/Makefile.am b/src/Makefile.am index e4e2e58d96b31..ec0037e22ded5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -165,6 +165,7 @@ BITCOIN_CORE_H = \ coins.h \ compat.h \ compat/byteswap.h \ + compat/cpuid.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h new file mode 100644 index 0000000000000..34ef9bb27f553 --- /dev/null +++ b/src/compat/cpuid.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_CPUID_H +#define BITCOIN_COMPAT_CPUID_H + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#define HAVE_GETCPUID + +#include + +// We can't use cpuid.h's __get_cpuid as it does not support subleafs. +void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + +#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#endif // BITCOIN_COMPAT_CPUID_H diff --git a/src/random.cpp b/src/random.cpp index 612bbb2ecec1a..14a7960f835db 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -6,6 +6,8 @@ #include "random.h" +#include "allocators.h" +#include "compat/cpuid.h" #include "crypto/sha512.h" #include "support/cleanse.h" #ifdef WIN32 @@ -23,8 +25,6 @@ #include "randomenv.h" -#include "allocators.h" - #ifndef WIN32 #include #include @@ -47,11 +47,6 @@ #include -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -#include -#include -#endif - #include #include #include @@ -82,7 +77,7 @@ static inline int64_t GetPerformanceCounter() noexcept #endif } -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#ifdef HAVE_GETCPUID static bool g_rdrand_supported = false; static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; @@ -93,15 +88,6 @@ static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND" #ifdef bit_RDSEED static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); #endif -static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ - // We can't use __get_cpuid as it doesn't support subleafs. -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} static void InitHardwareRand() { From 6142e1f504e4cc3a9a90d10c81b44c029d47e7bf Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 26 Oct 2019 17:14:21 -0700 Subject: [PATCH 37/55] Seed randomness with process id / thread id / various clocks This sort of data is also used by OpenSSL. --- src/randomenv.cpp | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index b54ce38385c41..3703dd875ba67 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -13,9 +13,22 @@ #endif #include +#include +#include #include #include +#include +#ifndef WIN32 +#include +#include +#include +#endif +#ifdef __MACH__ +#include +#include +#include +#endif namespace { @@ -57,13 +70,77 @@ void RandAddSeedPerfmon(CSHA512& hasher) #endif } +/** Helper to easily feed data into a CSHA512. + * + * Note that this does not serialize the passed object (like stream.h's << operators do). + * Its raw memory representation is used directly. + */ +template +CSHA512& operator<<(CSHA512& hasher, const T& data) { + static_assert(!std::is_same::type, char*>::value, "Calling operator<<(CSHA512, char*) is probably not what you want"); + static_assert(!std::is_same::type, unsigned char*>::value, "Calling operator<<(CSHA512, unsigned char*) is probably not what you want"); + static_assert(!std::is_same::type, const char*>::value, "Calling operator<<(CSHA512, const char*) is probably not what you want"); + static_assert(!std::is_same::type, const unsigned char*>::value, "Calling operator<<(CSHA512, const unsigned char*) is probably not what you want"); + hasher.Write((const unsigned char*)&data, sizeof(data)); + return hasher; +} + } // namespace void RandAddDynamicEnv(CSHA512& hasher) { RandAddSeedPerfmon(hasher); + + // Various clocks +#ifdef WIN32 + FILETIME ftime; + GetSystemTimeAsFileTime(&ftime); + hasher << ftime; +#else +# ifndef __MACH__ + // On non-MacOS systems, use various clock_gettime() calls. + struct timespec ts; +# ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &ts); + hasher << ts.tv_sec << ts.tv_nsec; +# endif +# ifdef CLOCK_REALTIME + clock_gettime(CLOCK_REALTIME, &ts); + hasher << ts.tv_sec << ts.tv_nsec; +# endif +# ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &ts); + hasher << ts.tv_sec << ts.tv_nsec; +# endif +# else + // On MacOS use mach_absolute_time (number of CPU ticks since boot) as a replacement for CLOCK_MONOTONIC, + // and clock_get_time for CALENDAR_CLOCK as a replacement for CLOCK_REALTIME. + hasher << mach_absolute_time(); + // From https://gist.github.com/jbenet/1087739 + clock_serv_t cclock; + mach_timespec_t mts; + if (host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock) == KERN_SUCCESS && clock_get_time(cclock, &mts) == KERN_SUCCESS) { + hasher << mts.tv_sec << mts.tv_nsec; + mach_port_deallocate(mach_task_self(), cclock); + } +# endif + // gettimeofday is available on all UNIX systems, but only has microsecond precision. + struct timeval tv; + gettimeofday(&tv, nullptr); + hasher << tv.tv_sec << tv.tv_usec; +#endif + // Probably redundant, but also use all the clocks C++11 provides: + hasher << std::chrono::system_clock::now().time_since_epoch().count(); + hasher << std::chrono::steady_clock::now().time_since_epoch().count(); + hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); } void RandAddStaticEnv(CSHA512& hasher) { +#ifdef WIN32 + hasher << GetCurrentProcessId() << GetCurrentThreadId(); +#else + hasher << getpid(); +#endif + hasher << std::this_thread::get_id(); } From 67de24653b439aac1392aee23da83fd5ffc71700 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 26 Oct 2019 18:05:10 -0700 Subject: [PATCH 38/55] Gather additional entropy from the environment This based on code by Gregory Maxwell. # Conflicts: # src/randomenv.cpp --- src/randomenv.cpp | 194 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 184 insertions(+), 10 deletions(-) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 3703dd875ba67..9c8fd4ed3bf27 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -3,8 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#if defined(HAVE_CONFIG_H) +#include "config/pivx-config.h" +#endif + #include "randomenv.h" +#include "clientversion.h" #include "crypto/sha512.h" #include "support/cleanse.h" #include "utiltime.h" // for GetTime() @@ -14,14 +19,21 @@ #include #include +#include #include #include #include #include #ifndef WIN32 +#include // must go before a number of other headers +#include +#include +#include +#include +#include #include -#include +#include #include #endif #ifdef __MACH__ @@ -29,6 +41,12 @@ #include #include #endif +#if HAVE_DECL_GETIFADDRS +#include +#endif + +//! Necessary on some platforms +extern char** environ; namespace { @@ -85,6 +103,53 @@ CSHA512& operator<<(CSHA512& hasher, const T& data) { return hasher; } +#ifndef WIN32 +void AddSockaddr(CSHA512& hasher, const struct sockaddr *addr) +{ + if (addr == nullptr) return; + switch (addr->sa_family) { + case AF_INET: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in)); + break; + case AF_INET6: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in6)); + break; + default: + hasher.Write((const unsigned char*)&addr->sa_family, sizeof(addr->sa_family)); + } +} + +void AddFile(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + int f = open(path, O_RDONLY); + size_t total = 0; + if (f != -1) { + unsigned char fbuf[4096]; + int n; + hasher.Write((const unsigned char*)&f, sizeof(f)); + if (fstat(f, &sb) == 0) hasher << sb; + do { + n = read(f, fbuf, sizeof(fbuf)); + if (n > 0) hasher.Write(fbuf, n); + total += n; + /* not bothering with EINTR handling. */ + } while (n == sizeof(fbuf) && total < 1048576); // Read only the first 1 Mbyte + close(f); + } +} + +void AddPath(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + if (stat(path, &sb) == 0) { + hasher.Write((const unsigned char*)path, strlen(path) + 1); + hasher << sb; + } +} +#endif + + } // namespace void RandAddDynamicEnv(CSHA512& hasher) @@ -99,18 +164,18 @@ void RandAddDynamicEnv(CSHA512& hasher) #else # ifndef __MACH__ // On non-MacOS systems, use various clock_gettime() calls. - struct timespec ts; + struct timespec ts = {}; # ifdef CLOCK_MONOTONIC clock_gettime(CLOCK_MONOTONIC, &ts); - hasher << ts.tv_sec << ts.tv_nsec; + hasher << ts; # endif # ifdef CLOCK_REALTIME clock_gettime(CLOCK_REALTIME, &ts); - hasher << ts.tv_sec << ts.tv_nsec; + hasher << ts; # endif # ifdef CLOCK_BOOTTIME clock_gettime(CLOCK_BOOTTIME, &ts); - hasher << ts.tv_sec << ts.tv_nsec; + hasher << ts; # endif # else // On MacOS use mach_absolute_time (number of CPU ticks since boot) as a replacement for CLOCK_MONOTONIC, @@ -118,29 +183,138 @@ void RandAddDynamicEnv(CSHA512& hasher) hasher << mach_absolute_time(); // From https://gist.github.com/jbenet/1087739 clock_serv_t cclock; - mach_timespec_t mts; + mach_timespec_t mts = {}; if (host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock) == KERN_SUCCESS && clock_get_time(cclock, &mts) == KERN_SUCCESS) { - hasher << mts.tv_sec << mts.tv_nsec; + hasher << mts; mach_port_deallocate(mach_task_self(), cclock); } # endif // gettimeofday is available on all UNIX systems, but only has microsecond precision. - struct timeval tv; + struct timeval tv = {}; gettimeofday(&tv, nullptr); - hasher << tv.tv_sec << tv.tv_usec; + hasher << tv; #endif // Probably redundant, but also use all the clocks C++11 provides: hasher << std::chrono::system_clock::now().time_since_epoch().count(); hasher << std::chrono::steady_clock::now().time_since_epoch().count(); hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); + +#ifndef WIN32 + // Current resource usage. + struct rusage usage = {}; + if (getrusage(RUSAGE_SELF, &usage) == 0) hasher << usage; +#endif + +#ifdef __linux__ + AddFile(hasher, "/proc/diskstats"); + AddFile(hasher, "/proc/vmstat"); + AddFile(hasher, "/proc/schedstat"); + AddFile(hasher, "/proc/zoneinfo"); + AddFile(hasher, "/proc/meminfo"); + AddFile(hasher, "/proc/softirqs"); + AddFile(hasher, "/proc/stat"); + AddFile(hasher, "/proc/self/schedstat"); + AddFile(hasher, "/proc/self/status"); +#endif + + // Stack and heap location + void* addr = malloc(4097); + hasher << &addr << addr; + free(addr); } void RandAddStaticEnv(CSHA512& hasher) { + // Some compile-time static properties + hasher << (CHAR_MIN < 0) << sizeof(void*) << sizeof(long) << sizeof(int); +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + hasher << __GNUC__ << __GNUC_MINOR__ << __GNUC_PATCHLEVEL__; +#endif +#ifdef _MSC_VER + hasher << _MSC_VER; +#endif + hasher << __cplusplus; +#ifdef _XOPEN_VERSION + hasher << _XOPEN_VERSION; +#endif +#ifdef __VERSION__ + const char* COMPILER_VERSION = __VERSION__; + hasher.Write((const unsigned char*)COMPILER_VERSION, strlen(COMPILER_VERSION) + 1); +#endif + + // Bitcoin client version + hasher << CLIENT_VERSION; + + // Memory locations + hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; + + // Hostname + char hname[256]; + if (gethostname(hname, 256) == 0) { + hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); + } + +#if HAVE_DECL_GETIFADDRS + // Network interfaces + struct ifaddrs *ifad = NULL; + getifaddrs(&ifad); + struct ifaddrs *ifit = ifad; + while (ifit != NULL) { + hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); + hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); + hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); + AddSockaddr(hasher, ifit->ifa_addr); + AddSockaddr(hasher, ifit->ifa_netmask); + AddSockaddr(hasher, ifit->ifa_dstaddr); + ifit = ifit->ifa_next; + } + freeifaddrs(ifad); +#endif + +#ifndef WIN32 + // UNIX kernel information + struct utsname name; + if (uname(&name) != -1) { + hasher.Write((const unsigned char*)&name.sysname, strlen(name.sysname) + 1); + hasher.Write((const unsigned char*)&name.nodename, strlen(name.nodename) + 1); + hasher.Write((const unsigned char*)&name.release, strlen(name.release) + 1); + hasher.Write((const unsigned char*)&name.version, strlen(name.version) + 1); + hasher.Write((const unsigned char*)&name.machine, strlen(name.machine) + 1); + } + + /* Path and filesystem provided data */ + AddPath(hasher, "/"); + AddPath(hasher, "."); + AddPath(hasher, "/tmp"); + AddPath(hasher, "/home"); + AddPath(hasher, "/proc"); +#ifdef __linux__ + AddFile(hasher, "/proc/cmdline"); + AddFile(hasher, "/proc/cpuinfo"); + AddFile(hasher, "/proc/version"); +#endif + AddFile(hasher, "/etc/passwd"); + AddFile(hasher, "/etc/group"); + AddFile(hasher, "/etc/hosts"); + AddFile(hasher, "/etc/resolv.conf"); + AddFile(hasher, "/etc/timezone"); + AddFile(hasher, "/etc/localtime"); + + /* TODO: sysctl's for OSX to fetch information not available from /proc */ +#endif + + // Env variables + if (environ) { + for (size_t i = 0; environ[i]; ++i) { + hasher.Write((const unsigned char*)environ[i], strlen(environ[i])); + } + } + + // Process, thread, user, session, group, ... ids. #ifdef WIN32 hasher << GetCurrentProcessId() << GetCurrentThreadId(); #else - hasher << getpid(); + hasher << getpid() << getppid() << getsid(0) << getpgid(0) << getuid() << geteuid() << getgid() << getegid(); #endif hasher << std::this_thread::get_id(); } From 8f5b9c98be6d622090a3db1912e31f689a523b63 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 28 Oct 2019 10:42:25 -0700 Subject: [PATCH 39/55] Use sysctl for seeding on MacOS/BSD --- configure.ac | 14 ++++- src/randomenv.cpp | 128 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index f540693063088..766988d647a24 100644 --- a/configure.ac +++ b/configure.ac @@ -749,7 +749,7 @@ if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([strnlen]) @@ -888,6 +888,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include [ AC_MSG_RESULT(no)] ) +AC_MSG_CHECKING(for sysctl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include + #include ]], + [[ static const int name[2] = {CTL_KERN, KERN_VERSION}; + #ifdef __linux__ + #error "Don't use sysctl on Linux, it's deprecated even when it works" + #endif + sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 9c8fd4ed3bf27..28f49c5fc452d 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -44,6 +44,18 @@ #if HAVE_DECL_GETIFADDRS #include #endif +#if HAVE_SYSCTL +#include +#if HAVE_VM_VM_PARAM_H +#include +#endif +#if HAVE_SYS_RESOURCES_H +#include +#endif +#if HAVE_SYS_VMMETER_H +#include +#endif +#endif //! Necessary on some platforms extern char** environ; @@ -149,6 +161,23 @@ void AddPath(CSHA512& hasher, const char *path) } #endif +#if HAVE_SYSCTL +template +void AddSysctl(CSHA512& hasher) +{ + int CTL[sizeof...(S)] = {S...}; + unsigned char buffer[65536]; + size_t siz = 65536; + int ret = sysctl(CTL, sizeof...(S), buffer, &siz, nullptr, 0); + if (ret == 0 || (ret == -1 && errno == ENOMEM)) { + hasher << sizeof(CTL); + hasher.Write((const unsigned char*)CTL, sizeof(CTL)); + if (siz > sizeof(buffer)) siz = sizeof(buffer); + hasher << siz; + hasher.Write(buffer, siz); + } +} +#endif } // namespace @@ -217,6 +246,30 @@ void RandAddDynamicEnv(CSHA512& hasher) AddFile(hasher, "/proc/self/status"); #endif +#if HAVE_SYSCTL +# ifdef CTL_KERN +# if defined(KERN_PROC) && defined(KERN_PROC_ALL) + AddSysctl(hasher); +# endif +# endif +# ifdef CTL_HW +# ifdef HW_DISKSTATS + AddSysctl(hasher); +# endif +# endif +# ifdef CTL_VM +# ifdef VM_LOADAVG + AddSysctl(hasher); +# endif +# ifdef VM_TOTAL + AddSysctl(hasher); +# endif +# ifdef VM_METER + AddSysctl(hasher); +# endif +# endif +#endif + // Stack and heap location void* addr = malloc(4097); hasher << &addr << addr; @@ -299,8 +352,81 @@ void RandAddStaticEnv(CSHA512& hasher) AddFile(hasher, "/etc/resolv.conf"); AddFile(hasher, "/etc/timezone"); AddFile(hasher, "/etc/localtime"); +#endif - /* TODO: sysctl's for OSX to fetch information not available from /proc */ + // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these + // will exist on every system. +#if HAVE_SYSCTL +# ifdef CTL_HW +# ifdef HW_MACHINE + AddSysctl(hasher); +# endif +# ifdef HW_MODEL + AddSysctl(hasher); +# endif +# ifdef HW_NCPU + AddSysctl(hasher); +# endif +# ifdef HW_PHYSMEM + AddSysctl(hasher); +# endif +# ifdef HW_USERMEM + AddSysctl(hasher); +# endif +# ifdef HW_MACHINE_ARCH + AddSysctl(hasher); +# endif +# ifdef HW_REALMEM + AddSysctl(hasher); +# endif +# ifdef HW_CPU_FREQ + AddSysctl(hasher); +# endif +# ifdef HW_BUS_FREQ + AddSysctl(hasher); +# endif +# ifdef HW_CACHELINE + AddSysctl(hasher); +# endif +# endif +# ifdef CTL_KERN +# ifdef KERN_BOOTFILE + AddSysctl(hasher); +# endif +# ifdef KERN_BOOTTIME + AddSysctl(hasher); +# endif +# ifdef KERN_CLOCKRATE + AddSysctl(hasher); +# endif +# ifdef KERN_HOSTID + AddSysctl(hasher); +# endif +# ifdef KERN_HOSTUUID + AddSysctl(hasher); +# endif +# ifdef KERN_HOSTNAME + AddSysctl(hasher); +# endif +# ifdef KERN_OSRELDATE + AddSysctl(hasher); +# endif +# ifdef KERN_OSRELEASE + AddSysctl(hasher); +# endif +# ifdef KERN_OSREV + AddSysctl(hasher); +# endif +# ifdef KERN_OSTYPE + AddSysctl(hasher); +# endif +# ifdef KERN_POSIX1 + AddSysctl(hasher); +# endif +# ifdef KERN_VERSION + AddSysctl(hasher); +# endif +# endif #endif // Env variables From 88d97d0cd7afc86d9985602cf82562e19ac4f2b4 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 26 Oct 2019 18:17:20 -0700 Subject: [PATCH 40/55] Feed CPUID data into RNG --- src/randomenv.cpp | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 28f49c5fc452d..cb6eb5a7d0a80 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -10,6 +10,7 @@ #include "randomenv.h" #include "clientversion.h" +#include "compat/cpuid.h" #include "crypto/sha512.h" #include "support/cleanse.h" #include "utiltime.h" // for GetTime() @@ -179,6 +180,36 @@ void AddSysctl(CSHA512& hasher) } #endif +#ifdef HAVE_GETCPUID +void inline AddCPUID(CSHA512& hasher, uint32_t leaf, uint32_t subleaf, uint32_t& ax, uint32_t& bx, uint32_t& cx, uint32_t& dx) +{ + GetCPUID(leaf, subleaf, ax, bx, cx, dx); + hasher << leaf << subleaf << ax << bx << cx << dx; +} + +void AddAllCPUID(CSHA512& hasher) +{ + uint32_t ax, bx, cx, dx; + // Iterate over all standard leaves + AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax + uint32_t max = ax; + for (uint32_t leaf = 1; leaf <= max; ++leaf) { + for (uint32_t subleaf = 0;; ++subleaf) { + AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); + // Iterate over subleaves for leaf 4, 11, 13 + if (leaf != 4 && leaf != 11 && leaf != 13) break; + if ((leaf == 4 || leaf == 13) && ax == 0) break; + if (leaf == 11 && (cx & 0xFF00) == 0) break; + } + } + // Iterate over all extended leaves + AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax + uint32_t ext_max = ax; + for (uint32_t leaf = 0x80000001; leaf <= ext_max; ++leaf) { + AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); + } +} +#endif } // namespace void RandAddDynamicEnv(CSHA512& hasher) @@ -298,6 +329,10 @@ void RandAddStaticEnv(CSHA512& hasher) // Bitcoin client version hasher << CLIENT_VERSION; +#ifdef HAVE_GETCPUID + AddAllCPUID(hasher); +#endif + // Memory locations hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; From 46791811b717afbf771ebe1eb9c09843ed6a20e3 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 27 Oct 2019 12:36:05 -0700 Subject: [PATCH 41/55] Add information gathered through getauxval() Suggested by Wladimir van der Laan. --- src/randomenv.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index cb6eb5a7d0a80..ca17f2482bd3f 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -57,6 +57,9 @@ #include #endif #endif +#ifdef __linux__ +#include +#endif //! Necessary on some platforms extern char** environ; @@ -329,6 +332,28 @@ void RandAddStaticEnv(CSHA512& hasher) // Bitcoin client version hasher << CLIENT_VERSION; +#ifdef __linux__ + // Information available through getauxval() +# ifdef AT_HWCAP + hasher << getauxval(AT_HWCAP); +# endif +# ifdef AT_HWCAP2 + hasher << getauxval(AT_HWCAP2); +# endif +# ifdef AT_RANDOM + const unsigned char* random_aux = (const unsigned char*)getauxval(AT_RANDOM); + if (random_aux) hasher.Write(random_aux, 16); +# endif +# ifdef AT_PLATFORM + const char* platform_str = (const char*)getauxval(AT_PLATFORM); + if (platform_str) hasher.Write((const unsigned char*)platform_str, strlen(platform_str) + 1); +# endif +# ifdef AT_EXECFN + const char* exec_str = (const char*)getauxval(AT_EXECFN); + if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); +# endif +#endif // __linux__ + #ifdef HAVE_GETCPUID AddAllCPUID(hasher); #endif From 7d6ddcb774732a75f49852850e206d47df7d4d6e Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 29 Oct 2019 11:55:59 -0700 Subject: [PATCH 42/55] Run background seeding periodically instead of unpredictably * Instead of calling RandAddSeedSleep anytime the scheduler goes idle, call its replacement (RandAddSeedPeriodic) just once per minute. This has better guarantees of actually being run, and helps limit how frequently the dynamic env data is gathered. * Since this code runs once per minute regardless now, we no longer need to keep track of the last time strengthening was run; just do it always. * Make strengthening time context dependent (100 ms at startup, 10 ms once per minute afterwards). --- src/init.cpp | 5 ++++ src/libzerocoin/bignum_gmp.cpp | 4 +-- src/random.cpp | 46 +++++++++++++--------------------- src/random.h | 4 +-- src/scheduler.cpp | 2 -- 5 files changed, 26 insertions(+), 35 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 9c4b89cc64bd1..caab0cdf5b333 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1254,6 +1254,11 @@ bool AppInitMain() CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(std::bind(&TraceThread, "scheduler", serviceLoop)); + // Gather some entropy once per minute. + scheduler.scheduleEvery([]{ + RandAddPeriodic(); + }, 60000); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); GetMainSignals().RegisterWithMempoolSignals(mempool); diff --git a/src/libzerocoin/bignum_gmp.cpp b/src/libzerocoin/bignum_gmp.cpp index 0de0e403e1f2e..5b199dbe3a301 100644 --- a/src/libzerocoin/bignum_gmp.cpp +++ b/src/libzerocoin/bignum_gmp.cpp @@ -63,7 +63,7 @@ CBigNum CBigNum::randBignum(const CBigNum& range) size_t size = (mpz_sizeinbase (range.bn, 2) + CHAR_BIT-1) / CHAR_BIT; std::vector buf(size); - RandAddSeedSleep(); + RandAddPeriodic(); GetRandBytes(buf.data(), size); CBigNum ret(buf); @@ -80,7 +80,7 @@ CBigNum CBigNum::randKBitBignum(const uint32_t k) { std::vector buf((k+7)/8); - RandAddSeedSleep(); + RandAddPeriodic(); GetRandBytes(buf.data(), (k+7)/8); CBigNum ret(buf); diff --git a/src/random.cpp b/src/random.cpp index 14a7960f835db..98ecb970b6694 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -511,22 +511,16 @@ static void SeedSlow(CSHA512& hasher) noexcept } /** Extract entropy from rng, strengthen it, and feed it into hasher. */ -static void SeedStrengthen(CSHA512& hasher, RNGState& rng) noexcept +static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noexcept { - static std::atomic last_strengthen{0}; - int64_t last_time = last_strengthen.load(); - int64_t current_time = GetTimeMicros(); - if (current_time > last_time + 60000000) { // Only run once a minute - // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. - unsigned char strengthen_seed[32]; - rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); - // Strengthen it for 10ms (100ms on first run), and feed it into hasher. - Strengthen(strengthen_seed, last_time == 0 ? 100000 : 10000, hasher); - last_strengthen = current_time; - } + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen the seed, and feed it into hasher. + Strengthen(strengthen_seed, microseconds, hasher); } -static void SeedSleep(CSHA512& hasher, RNGState& rng) +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -534,17 +528,11 @@ static void SeedSleep(CSHA512& hasher, RNGState& rng) // High-precision timestamp SeedTimestamp(hasher); - // Sleep for 1ms - MilliSleep(1); - - // High-precision timestamp after sleeping (as we commit to both the time before and after, this measures the delay) - SeedTimestamp(hasher); - - // Dynamic environment data (performance monitoring, ...; once every 10 minutes) + // Dynamic environment data (performance monitoring, ...) RandAddDynamicEnv(hasher); - // Strengthen every minute - SeedStrengthen(hasher, rng); + // Strengthen for 10 ms + SeedStrengthen(hasher, rng, 10000); } static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept @@ -555,20 +543,20 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept // Everything that the 'slow' seeder includes. SeedSlow(hasher); - // Dynamic environment data + // Dynamic environment data (performance monitoring, ...) RandAddDynamicEnv(hasher); // Static environment data RandAddStaticEnv(hasher); - // Strengthen - SeedStrengthen(hasher, rng); + // Strengthen for 100 ms + SeedStrengthen(hasher, rng, 100000); } enum class RNGLevel { FAST, //!< Automatically called by GetRandBytes SLOW, //!< Automatically called by GetStrongRandBytes - SLEEP, //!< Called by RandAddSeedSleep() + PERIODIC, //!< Called by RandAddPeriodic() }; static void ProcRand(unsigned char* out, int num, RNGLevel level) @@ -586,8 +574,8 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) case RNGLevel::SLOW: SeedSlow(hasher); break; - case RNGLevel::SLEEP: - SeedSleep(hasher, rng); + case RNGLevel::PERIODIC: + SeedPeriodic(hasher, rng); break; } @@ -610,7 +598,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } -void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } +void RandAddPeriodic() { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } bool g_mock_deterministic_tests{false}; diff --git a/src/random.h b/src/random.h index d831144def8db..8b1c945e62950 100644 --- a/src/random.h +++ b/src/random.h @@ -83,11 +83,11 @@ uint256 GetRandHash() noexcept; void GetStrongRandBytes(unsigned char* buf, int num) noexcept; /** - * Sleep for 1ms, gather entropy from various sources, and feed them to the PRNG state. + * Gather entropy from various expensive sources, and feed them to the PRNG state. * * Thread-safe. */ -void RandAddSeedSleep(); +void RandAddPeriodic(); /** * Fast randomness source. This is seeded once with secure random data, but diff --git a/src/scheduler.cpp b/src/scheduler.cpp index ed50cc177a2e8..ac52c43588ac6 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -32,8 +32,6 @@ void CScheduler::serviceQueue() try { if (!shouldStop() && taskQueue.empty()) { reverse_lock > rlock(lock); - // Use this chance to get more entropy - RandAddSeedSleep(); } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. From f363ea9ac668770b6d8ad98a716df4314262d35e Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sat, 23 Nov 2019 08:42:23 -0800 Subject: [PATCH 43/55] Seed RNG with precision timestamps on receipt of net messages. --- src/net.cpp | 7 +++++++ src/random.cpp | 33 +++++++++++++++++++++++++++++++++ src/random.h | 8 ++++++++ 3 files changed, 48 insertions(+) diff --git a/src/net.cpp b/src/net.cpp index c7bd86b4e0756..da4ea83079929 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -21,6 +21,7 @@ #include "netbase.h" #include "netmessagemaker.h" #include "primitives/transaction.h" +#include "random.h" #include "scheduler.h" #include "validation.h" @@ -379,6 +380,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char* pszDest, bool fCo pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); + // We're making a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); + return pnode; } else if (!proxyConnectionFailed) { // If connecting to the node failed, and failure is not caused by a problem connecting to @@ -1067,6 +1071,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { LOCK(cs_vNodes); vNodes.push_back(pnode); } + + // We received a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); } void CConnman::ThreadSocketHandler() diff --git a/src/random.cpp b/src/random.cpp index 98ecb970b6694..bcfc34a19544a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -8,6 +8,7 @@ #include "allocators.h" #include "compat/cpuid.h" +#include "crypto/sha256.h" #include "crypto/sha512.h" #include "support/cleanse.h" #ifdef WIN32 @@ -488,6 +489,23 @@ static void SeedFast(CSHA512& hasher) noexcept SeedTimestamp(hasher); } +// We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, +// since we want it to be fast as network peers may be able to trigger it repeatedly. +static Mutex events_mutex; +static CSHA256 events_hasher; +static void SeedEvents(CSHA512& hasher) +{ + LOCK(events_mutex); + + unsigned char events_hash[32]; + events_hasher.Finalize(events_hash); + hasher.Write(events_hash, 32); + + // Re-initialize the hasher with the finalized state to use later. + events_hasher.Reset(); + events_hasher.Write(events_hash, 32); +} + static void SeedSlow(CSHA512& hasher) noexcept { unsigned char buffer[32]; @@ -503,6 +521,9 @@ static void SeedSlow(CSHA512& hasher) noexcept RAND_bytes(buffer, sizeof(buffer)); hasher.Write(buffer, sizeof(buffer)); + // Add the events hasher into the mix + SeedEvents(hasher); + // High-precision timestamp. // // Note that we also commit to a timestamp in the Fast seeder, so we indirectly commit to a @@ -528,6 +549,9 @@ static void SeedPeriodic(CSHA512& hasher, RNGState& rng) // High-precision timestamp SeedTimestamp(hasher); + // Add the events hasher into the mix + SeedEvents(hasher); + // Dynamic environment data (performance monitoring, ...) RandAddDynamicEnv(hasher); @@ -600,6 +624,15 @@ void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNG void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } void RandAddPeriodic() { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddEvent(const uint32_t event_info) { + LOCK(events_mutex); + events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); + // Get the low four bytes of the performance counter. This translates to roughly the + // subsecond part. + uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); + events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); +} + bool g_mock_deterministic_tests{false}; uint64_t GetRand(uint64_t nMax) noexcept diff --git a/src/random.h b/src/random.h index 8b1c945e62950..04e30214da7ad 100644 --- a/src/random.h +++ b/src/random.h @@ -89,6 +89,14 @@ void GetStrongRandBytes(unsigned char* buf, int num) noexcept; */ void RandAddPeriodic(); +/** + * Gathers entropy from the low bits of the time at which events occur. Should + * be called with a uint32_t describing the event at the time an event occurs. + * + * Thread-safe. + */ +void RandAddEvent(const uint32_t event_info); + /** * Fast randomness source. This is seeded once with secure random data, but * is completely deterministic and does not gather more entropy after that. From 81d382fc5b47ef14136b49bb1629db50c28e48dd Mon Sep 17 00:00:00 2001 From: fanquake Date: Mon, 18 Nov 2019 09:58:10 -0500 Subject: [PATCH 44/55] doc: correct random.h docs after #17270 --- src/random.cpp | 2 +- src/random.h | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index bcfc34a19544a..716d4a31f4cad 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -187,7 +187,7 @@ static uint64_t GetRdSeed() noexcept /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). * Slower sources should probably be invoked separately, and/or only from - * RandAddSeedSleep (which is called during idle background operation). + * RandAddPeriodic (which is called once a minute). */ static void InitHardwareRand() {} static void ReportHardwareRand() {} diff --git a/src/random.h b/src/random.h index 04e30214da7ad..c1f680fd1206f 100644 --- a/src/random.h +++ b/src/random.h @@ -41,17 +41,17 @@ * These entropy sources are slower, but designed to make sure the RNG state contains * fresh data that is unpredictable to attackers. * - * - RandAddSeedSleep() seeds everything that fast seeding includes, but additionally: - * - A high-precision timestamp before and after sleeping 1ms. - * - (On Windows) Once every 10 minutes, performance monitoring data from the OS. - - - Once every minute, strengthen the entropy for 10 ms using repeated SHA512. - * These just exploit the fact the system is idle to improve the quality of the RNG - * slightly. + * - RandAddPeriodic() seeds everything that fast seeding includes, but additionally: + * - A high-precision timestamp + * - Dynamic environment data (performance monitoring, ...) + * - Strengthen the entropy for 10 ms using repeated SHA512. + * This is run once every minute. * * On first use of the RNG (regardless of what function is called first), all entropy * sources used in the 'slow' seeder are included, but also: * - 256 bits from the hardware RNG (rdseed or rdrand) when available. - * - (On Windows) Performance monitoring data from the OS. + * - Dynamic environment data (performance monitoring, ...) + * - Static environment data * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and From 88c2ae5f6e29533e05e52252c516a8008688c4be Mon Sep 17 00:00:00 2001 From: fanquake Date: Mon, 18 Nov 2019 10:21:28 -0500 Subject: [PATCH 45/55] random: mark RandAddPeriodic and SeedPeriodic as noexcept The usage of MilliSleep() in SeedPeriodic (previously SeedSleep) was removed in #17270, meaning it, and its users can now be marked noexcept. --- src/random.cpp | 16 +++------------- src/random.h | 2 +- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 716d4a31f4cad..01bd9b32edb13 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -455,17 +455,7 @@ void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THRE /* A note on the use of noexcept in the seeding functions below: * - * None of the RNG code should ever throw any exception, with the sole exception - * of MilliSleep in SeedSleep, which can (and does) support interruptions which - * cause a boost::thread_interrupted to be thrown. - * - * This means that SeedSleep, and all functions that invoke it are throwing. - * However, we know that GetRandBytes() and GetStrongRandBytes() never trigger - * this sleeping logic, so they are noexcept. The same is true for all the - * GetRand*() functions that use GetRandBytes() indirectly. - * - * TODO: After moving away from interruptible boost-based thread management, - * everything can become noexcept here. + * None of the RNG code should ever throw any exception. */ static void SeedTimestamp(CSHA512& hasher) noexcept @@ -541,7 +531,7 @@ static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noe Strengthen(strengthen_seed, microseconds, hasher); } -static void SeedPeriodic(CSHA512& hasher, RNGState& rng) +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -622,7 +612,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } -void RandAddPeriodic() { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } void RandAddEvent(const uint32_t event_info) { LOCK(events_mutex); diff --git a/src/random.h b/src/random.h index c1f680fd1206f..f4ba27ee01e84 100644 --- a/src/random.h +++ b/src/random.h @@ -87,7 +87,7 @@ void GetStrongRandBytes(unsigned char* buf, int num) noexcept; * * Thread-safe. */ -void RandAddPeriodic(); +void RandAddPeriodic() noexcept; /** * Gathers entropy from the low bits of the time at which events occur. Should From 8a9bbb17082ab2dcac8e41dcf45ff633185e48c4 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 4 Dec 2019 15:14:43 -0800 Subject: [PATCH 46/55] Move events_hasher into RNGState() --- src/random.cpp | 71 +++++++++++++++++++++++++++----------------------- src/random.h | 2 +- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/random.cpp b/src/random.cpp index 01bd9b32edb13..718fd141c5913 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -371,6 +371,9 @@ class RNGState { bool m_strongly_seeded GUARDED_BY(m_mutex) = false; std::unique_ptr m_mutex_openssl; + Mutex m_events_mutex; + CSHA256 m_events_hasher GUARDED_BY(m_events_mutex); + public: RNGState() noexcept { @@ -396,6 +399,35 @@ class RNGState { CRYPTO_set_locking_callback(nullptr); } + void AddEvent(uint32_t event_info) noexcept + { + LOCK(m_events_mutex); + + m_events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); + // Get the low four bytes of the performance counter. This translates to roughly the + // subsecond part. + uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); + m_events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); + } + + /** + * Feed (the hash of) all events added through AddEvent() to hasher. + */ + void SeedEvents(CSHA512& hasher) noexcept + { + // We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, + // since we want it to be fast as network peers may be able to trigger it repeatedly. + LOCK(m_events_mutex); + + unsigned char events_hash[32]; + m_events_hasher.Finalize(events_hash); + hasher.Write(events_hash, 32); + + // Re-initialize the hasher with the finalized state to use later. + m_events_hasher.Reset(); + m_events_hasher.Write(events_hash, 32); + } + /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. * * If this function has never been called with strong_seed = true, false is returned. @@ -479,24 +511,7 @@ static void SeedFast(CSHA512& hasher) noexcept SeedTimestamp(hasher); } -// We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, -// since we want it to be fast as network peers may be able to trigger it repeatedly. -static Mutex events_mutex; -static CSHA256 events_hasher; -static void SeedEvents(CSHA512& hasher) -{ - LOCK(events_mutex); - - unsigned char events_hash[32]; - events_hasher.Finalize(events_hash); - hasher.Write(events_hash, 32); - - // Re-initialize the hasher with the finalized state to use later. - events_hasher.Reset(); - events_hasher.Write(events_hash, 32); -} - -static void SeedSlow(CSHA512& hasher) noexcept +static void SeedSlow(CSHA512& hasher, RNGState& rng) noexcept { unsigned char buffer[32]; @@ -512,7 +527,7 @@ static void SeedSlow(CSHA512& hasher) noexcept hasher.Write(buffer, sizeof(buffer)); // Add the events hasher into the mix - SeedEvents(hasher); + rng.SeedEvents(hasher); // High-precision timestamp. // @@ -540,7 +555,7 @@ static void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept SeedTimestamp(hasher); // Add the events hasher into the mix - SeedEvents(hasher); + rng.SeedEvents(hasher); // Dynamic environment data (performance monitoring, ...) RandAddDynamicEnv(hasher); @@ -555,7 +570,7 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept SeedHardwareSlow(hasher); // Everything that the 'slow' seeder includes. - SeedSlow(hasher); + SeedSlow(hasher, rng); // Dynamic environment data (performance monitoring, ...) RandAddDynamicEnv(hasher); @@ -573,7 +588,7 @@ enum class RNGLevel { PERIODIC, //!< Called by RandAddPeriodic() }; -static void ProcRand(unsigned char* out, int num, RNGLevel level) +static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept { // Make sure the RNG is initialized first (as all Seed* function possibly need hwrand to be available). RNGState& rng = GetRNGState(); @@ -586,7 +601,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedFast(hasher); break; case RNGLevel::SLOW: - SeedSlow(hasher); + SeedSlow(hasher, rng); break; case RNGLevel::PERIODIC: SeedPeriodic(hasher, rng); @@ -613,15 +628,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } - -void RandAddEvent(const uint32_t event_info) { - LOCK(events_mutex); - events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); - // Get the low four bytes of the performance counter. This translates to roughly the - // subsecond part. - uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); - events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); -} +void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } bool g_mock_deterministic_tests{false}; diff --git a/src/random.h b/src/random.h index f4ba27ee01e84..632221bdae681 100644 --- a/src/random.h +++ b/src/random.h @@ -95,7 +95,7 @@ void RandAddPeriodic() noexcept; * * Thread-safe. */ -void RandAddEvent(const uint32_t event_info); +void RandAddEvent(const uint32_t event_info) noexcept; /** * Fast randomness source. This is seeded once with secure random data, but From 41ab1ffe2854c7d5b40b331eb2055ed5115f764b Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 19 Nov 2019 14:55:02 -0800 Subject: [PATCH 47/55] Fix CPUID subleaf iteration --- src/randomenv.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index ca17f2482bd3f..5dea90898c2da 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -197,12 +197,23 @@ void AddAllCPUID(CSHA512& hasher) AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax uint32_t max = ax; for (uint32_t leaf = 1; leaf <= max; ++leaf) { + uint32_t maxsub = 0; for (uint32_t subleaf = 0;; ++subleaf) { AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); - // Iterate over subleaves for leaf 4, 11, 13 - if (leaf != 4 && leaf != 11 && leaf != 13) break; - if ((leaf == 4 || leaf == 13) && ax == 0) break; - if (leaf == 11 && (cx & 0xFF00) == 0) break; + // Iterate subleafs for leaf values 4, 7, 11, 13 + if (leaf == 4) { + if ((ax & 0x1f) == 0) break; + } else if (leaf == 7) { + if (subleaf == 0) maxsub = ax; + if (subleaf == maxsub) break; + } else if (leaf == 11) { + if ((cx & 0xff00) == 0) break; + } else if (leaf == 13) { + if (ax == 0 && bx == 0 && cx == 0 && dx == 0) break; + } else { + // For any other leaf, stop after subleaf 0. + break; + } } } // Iterate over all extended leaves From fec460cf0a6f76ab26c2f8a71be90272aeb4c17c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 Nov 2019 10:54:08 -0800 Subject: [PATCH 48/55] Put bounds on the number of CPUID leaves explored --- src/randomenv.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 5dea90898c2da..483811aa60105 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -196,9 +196,9 @@ void AddAllCPUID(CSHA512& hasher) // Iterate over all standard leaves AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax uint32_t max = ax; - for (uint32_t leaf = 1; leaf <= max; ++leaf) { + for (uint32_t leaf = 1; leaf <= max && leaf <= 0xFF; ++leaf) { uint32_t maxsub = 0; - for (uint32_t subleaf = 0;; ++subleaf) { + for (uint32_t subleaf = 0; subleaf <= 0xFF; ++subleaf) { AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); // Iterate subleafs for leaf values 4, 7, 11, 13 if (leaf == 4) { @@ -219,7 +219,7 @@ void AddAllCPUID(CSHA512& hasher) // Iterate over all extended leaves AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax uint32_t ext_max = ax; - for (uint32_t leaf = 0x80000001; leaf <= ext_max; ++leaf) { + for (uint32_t leaf = 0x80000001; leaf <= ext_max && leaf <= 0x800000FF; ++leaf) { AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); } } From b70b26f38301377e3314d2315148b316a1d4ed4a Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Tue, 13 Apr 2021 18:24:48 -0700 Subject: [PATCH 49/55] Fix typo in comment in randomenv.cpp --- src/randomenv.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 483811aa60105..94438db790165 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -96,7 +96,7 @@ void RandAddSeedPerfmon(CSHA512& hasher) } else { // Performance data is only a best-effort attempt at improving the // situation when the OS randomness (and other sources) aren't - // adequate. As a result, failure to read it is isn't considered critical, + // adequate. As a result, failure to read it isn't considered critical, // so we don't call RandFailure(). // TODO: Add logging when the logger is made functional before global // constructors have been invoked. From 77bddd7ec7cdde39366eb1625742435f62c2e0ab Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Tue, 13 Apr 2021 18:25:45 -0700 Subject: [PATCH 50/55] Use GetStrongRandBytes in gmp bignum initialization --- src/libzerocoin/bignum_gmp.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libzerocoin/bignum_gmp.cpp b/src/libzerocoin/bignum_gmp.cpp index 5b199dbe3a301..dee024cfc2d15 100644 --- a/src/libzerocoin/bignum_gmp.cpp +++ b/src/libzerocoin/bignum_gmp.cpp @@ -63,8 +63,7 @@ CBigNum CBigNum::randBignum(const CBigNum& range) size_t size = (mpz_sizeinbase (range.bn, 2) + CHAR_BIT-1) / CHAR_BIT; std::vector buf(size); - RandAddPeriodic(); - GetRandBytes(buf.data(), size); + GetStrongRandBytes(buf.data(), size); CBigNum ret(buf); if (ret < 0) @@ -80,8 +79,7 @@ CBigNum CBigNum::randKBitBignum(const uint32_t k) { std::vector buf((k+7)/8); - RandAddPeriodic(); - GetRandBytes(buf.data(), (k+7)/8); + GetStrongRandBytes(buf.data(), (k+7)/8); CBigNum ret(buf); if (ret < 0) From 3a039d62cb460e8578f9923c64f989a0fb87a09a Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 19 Jul 2018 14:21:05 -0400 Subject: [PATCH 51/55] build: avoid getifaddrs when unavailable --- configure.ac | 4 ++++ src/net.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 766988d647a24..0345402f83a02 100644 --- a/configure.ac +++ b/configure.ac @@ -751,6 +751,10 @@ fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) +AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, + [#include + #include ] +) AC_CHECK_DECLS([strnlen]) # Check for daemon(3), unrelated to --with-daemon (although used by it) diff --git a/src/net.cpp b/src/net.cpp index da4ea83079929..c063172732608 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2007,7 +2007,7 @@ void Discover() } } } -#else +#elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) // Get local host ip struct ifaddrs* myaddrs; if (getifaddrs(&myaddrs) == 0) { From 414f405281c0be5a10351b35259a75b546434d58 Mon Sep 17 00:00:00 2001 From: fanquake Date: Sat, 20 Mar 2021 17:57:43 +0800 Subject: [PATCH 52/55] rand: only try and use freeifaddrs if available # Conflicts: # src/randomenv.cpp --- src/randomenv.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 94438db790165..eed4db22c0195 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -42,7 +42,7 @@ #include #include #endif -#if HAVE_DECL_GETIFADDRS +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS #include #endif #if HAVE_SYSCTL @@ -378,7 +378,7 @@ void RandAddStaticEnv(CSHA512& hasher) hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); } -#if HAVE_DECL_GETIFADDRS +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS // Network interfaces struct ifaddrs *ifad = NULL; getifaddrs(&ifad); From e906436e4fcbc1f5a04e3e9a9879bb8d382744f2 Mon Sep 17 00:00:00 2001 From: fanquake Date: Sun, 21 Mar 2021 08:41:26 +0800 Subject: [PATCH 53/55] build: check if -lsocket is required with *ifaddrs --- build-aux/m4/l_socket.m4 | 36 ++++++++++++++++++++++++++++++++++++ configure.ac | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 build-aux/m4/l_socket.m4 diff --git a/build-aux/m4/l_socket.m4 b/build-aux/m4/l_socket.m4 new file mode 100644 index 0000000000000..38923a98fc966 --- /dev/null +++ b/build-aux/m4/l_socket.m4 @@ -0,0 +1,36 @@ +# Illumos/SmartOS requires linking with -lsocket if +# using getifaddrs & freeifaddrs + +m4_define([_CHECK_SOCKET_testbody], [[ + #include + #include + + int main() { + struct ifaddrs *ifaddr; + getifaddrs(&ifaddr); + freeifaddrs(ifaddr); + } +]]) + +AC_DEFUN([CHECK_SOCKET], [ + + AC_LANG_PUSH(C++) + + AC_MSG_CHECKING([whether ifaddrs funcs can be used without link library]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + LIBS="$LIBS -lsocket" + AC_MSG_CHECKING([whether getifaddrs needs -lsocket]) + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_FAILURE([cannot figure out how to use getifaddrs]) + ]) + ]) + + AC_LANG_POP +]) diff --git a/configure.ac b/configure.ac index 0345402f83a02..24d322fbb089c 100644 --- a/configure.ac +++ b/configure.ac @@ -751,7 +751,7 @@ fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) -AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, +AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include #include ] ) From d9f67dad557ec8bf715c33738d5cfcad0a71ee23 Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 25 Mar 2021 21:53:05 -0700 Subject: [PATCH 54/55] net: add ifaddrs.h include --- src/net.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/net.cpp b/src/net.cpp index c063172732608..e865b241d91e2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -31,6 +31,10 @@ #include #endif +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS +#include +#endif + #ifdef USE_UPNP #include #include From cecbf6c0ae4a70dd05390bd6979f6f2cf893158f Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Tue, 13 Apr 2021 21:05:25 -0700 Subject: [PATCH 55/55] Use secure.h header for secure allocators #2276 split the `allocators.h` header and has been merged --- src/random.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/random.cpp b/src/random.cpp index 718fd141c5913..c033403df2723 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -6,11 +6,11 @@ #include "random.h" -#include "allocators.h" #include "compat/cpuid.h" #include "crypto/sha256.h" #include "crypto/sha512.h" #include "support/cleanse.h" +#include "support/allocators/secure.h" #ifdef WIN32 #include "compat.h" // for Windows API #include