From 839f14de0600445bea3261d38a91dc32f9289d8d Mon Sep 17 00:00:00 2001 From: random-zebra Date: Tue, 11 Jan 2022 12:47:58 -0300 Subject: [PATCH] P2P: Ensure intra-quorum connections and only relay to members --- CMakeLists.txt | 1 + src/Makefile.am | 2 + src/evo/deterministicmns.cpp | 11 -- src/evo/deterministicmns.h | 1 - src/evo/mnauth.cpp | 4 +- src/llmq/quorums_connections.cpp | 204 ++++++++++++++++++++++++ src/llmq/quorums_connections.h | 26 +++ src/llmq/quorums_dkgsession.cpp | 41 ++++- src/llmq/quorums_dkgsession.h | 1 + src/llmq/quorums_dkgsessionhandler.cpp | 7 +- src/llmq/quorums_utils.cpp | 26 +-- src/llmq/quorums_utils.h | 5 - test/lint/lint-circular-dependencies.sh | 2 +- 13 files changed, 280 insertions(+), 51 deletions(-) create mode 100644 src/llmq/quorums_connections.cpp create mode 100644 src/llmq/quorums_connections.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 53b6f575f429f..918cbcdca2358 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -419,6 +419,7 @@ set(COMMON_SOURCES ./src/evo/specialtx_validation.cpp ./src/llmq/quorums_blockprocessor.cpp ./src/llmq/quorums_commitment.cpp + ./src/llmq/quorums_connections.cpp ./src/llmq/quorums_debug.cpp ./src/llmq/quorums_dkgsessionhandler.cpp ./src/llmq/quorums_dkgsessionmgr.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 2f25321d24546..3bdb7af856ed2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -193,6 +193,7 @@ BITCOIN_CORE_H = \ flatdb.h \ llmq/quorums_blockprocessor.h \ llmq/quorums_commitment.h \ + llmq/quorums_connections.h \ llmq/quorums_debug.h \ llmq/quorums_dkgsessionhandler.h \ llmq/quorums_dkgsessionmgr.h \ @@ -367,6 +368,7 @@ libbitcoin_server_a_SOURCES = \ evo/specialtx_validation.cpp \ llmq/quorums_blockprocessor.cpp \ llmq/quorums_commitment.cpp \ + llmq/quorums_connections.cpp \ llmq/quorums_debug.cpp \ llmq/quorums_dkgsessionhandler.cpp \ llmq/quorums_dkgsessionmgr.cpp \ diff --git a/src/evo/deterministicmns.cpp b/src/evo/deterministicmns.cpp index f7bbaf6825821..2f8bc2f0e3716 100644 --- a/src/evo/deterministicmns.cpp +++ b/src/evo/deterministicmns.cpp @@ -14,7 +14,6 @@ #include "core_io.h" #include "key_io.h" #include "guiinterface.h" -#include "llmq/quorums_utils.h" #include "masternodeman.h" // for mnodeman (!TODO: remove) #include "script/standard.h" #include "spork.h" @@ -981,14 +980,4 @@ std::vector CDeterministicMNManager::GetAllQuorumMembers(C return allMns.CalculateQuorum(params.size, modifier); } -std::set CDeterministicMNManager::GetQuorumRelayMembers(Consensus::LLMQType llmqType, const CBlockIndex *pindexQuorum) -{ - // !TODO: complete me - auto mns = GetAllQuorumMembers(llmqType, pindexQuorum); - std::set result; - for (const auto& dmn : mns) { - result.emplace(dmn->proTxHash); - } - return result; -} diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index b91ada81038c2..f180294633c41 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -591,7 +591,6 @@ class CDeterministicMNManager // Get the list of members for a given quorum type and index std::vector GetAllQuorumMembers(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum); - std::set GetQuorumRelayMembers(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum); private: void CleanupCache(int nHeight); diff --git a/src/evo/mnauth.cpp b/src/evo/mnauth.cpp index ee5a53d3255b9..fb558d393f06a 100644 --- a/src/evo/mnauth.cpp +++ b/src/evo/mnauth.cpp @@ -10,7 +10,7 @@ #include "consensus/validation.h" #include "net.h" // for CSerializedNetMsg #include "netmessagemaker.h" -#include "llmq/quorums_utils.h" +#include "llmq/quorums_connections.h" #include "tiertwo/masternode_meta_manager.h" #include "tiertwo/net_masternodes.h" #include "tiertwo/tiertwo_sync_state.h" @@ -142,7 +142,7 @@ bool CMNAuth::ProcessMessage(CNode* pnode, const std::string& strCommand, CDataS if (pnode2->verifiedProRegTxHash == mnauth.proRegTxHash) { if (fMasterNode) { - auto deterministicOutbound = llmq::utils::DeterministicOutboundConnection(activeMnInfo->proTxHash, mnauth.proRegTxHash); + auto deterministicOutbound = llmq::DeterministicOutboundConnection(activeMnInfo->proTxHash, mnauth.proRegTxHash); LogPrint(BCLog::NET_MN, "CMNAuth::ProcessMessage -- Masternode %s has already verified as peer %d, deterministicOutbound=%s. peer=%d\n", mnauth.proRegTxHash.ToString(), pnode2->GetId(), deterministicOutbound.ToString(), pnode->GetId()); if (deterministicOutbound == activeMnInfo->proTxHash) { diff --git a/src/llmq/quorums_connections.cpp b/src/llmq/quorums_connections.cpp new file mode 100644 index 0000000000000..513eec5d71448 --- /dev/null +++ b/src/llmq/quorums_connections.cpp @@ -0,0 +1,204 @@ +// Copyright (c) 2018-2019 The Dash Core developers +// Copyright (c) 2022 The PIVX developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "llmq/quorums_connections.h" + +#include "evo/deterministicmns.h" +#include "net.h" +#include "tiertwo/masternode_meta_manager.h" // for g_mmetaman +#include "tiertwo/net_masternodes.h" +#include "validation.h" + +namespace llmq +{ + + +uint256 DeterministicOutboundConnection(const uint256& proTxHash1, const uint256& proTxHash2) +{ + // We need to deterministically select who is going to initiate the connection. The naive way would be to simply + // return the min(proTxHash1, proTxHash2), but this would create a bias towards MNs with a numerically low + // hash. To fix this, we return the proTxHash that has the lowest value of: + // hash(min(proTxHash1, proTxHash2), max(proTxHash1, proTxHash2), proTxHashX) + // where proTxHashX is the proTxHash to compare + uint256 h1; + uint256 h2; + if (proTxHash1 < proTxHash2) { + h1 = ::SerializeHash(std::make_tuple(proTxHash1, proTxHash2, proTxHash1)); + h2 = ::SerializeHash(std::make_tuple(proTxHash1, proTxHash2, proTxHash2)); + } else { + h1 = ::SerializeHash(std::make_tuple(proTxHash2, proTxHash1, proTxHash1)); + h2 = ::SerializeHash(std::make_tuple(proTxHash2, proTxHash1, proTxHash2)); + } + if (h1 < h2) { + return proTxHash1; + } + return proTxHash2; +} + +std::set GetQuorumRelayMembers(Consensus::LLMQType llmqType, const CBlockIndex *pindexQuorum, const uint256 &forMember, bool onlyOutbound) +{ + auto mns = deterministicMNManager->GetAllQuorumMembers(llmqType, pindexQuorum); + std::set result; + + auto calcOutbound = [&](size_t i, const uint256 proTxHash) { + // Relay to nodes at indexes (i+2^k)%n, where + // k: 0..max(1, floor(log2(n-1))-1) + // n: size of the quorum/ring + std::set r; + int gap = 1; + int gap_max = (int)mns.size() - 1; + int k = 0; + while ((gap_max >>= 1) || k <= 1) { + size_t idx = (i + gap) % mns.size(); + auto& otherDmn = mns[idx]; + if (otherDmn->proTxHash == proTxHash) { + continue; + } + r.emplace(otherDmn->proTxHash); + gap <<= 1; + k++; + } + return r; + }; + + for (size_t i = 0; i < mns.size(); i++) { + auto& dmn = mns[i]; + if (dmn->proTxHash == forMember) { + auto r = calcOutbound(i, dmn->proTxHash); + result.insert(r.begin(), r.end()); + } else if (!onlyOutbound) { + auto r = calcOutbound(i, dmn->proTxHash); + if (r.count(forMember)) { + result.emplace(dmn->proTxHash); + } + } + } + + return result; +} + +static std::set GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember, bool onlyOutbound) +{ + auto mns = deterministicMNManager->GetAllQuorumMembers(llmqType, pindexQuorum); + std::set result; + + for (auto& dmn : mns) { + if (dmn->proTxHash == forMember) { + continue; + } + // Determine which of the two MNs (forMember vs dmn) should initiate the outbound connection and which + // one should wait for the inbound connection. We do this in a deterministic way, so that even when we + // end up with both connecting to each other, we know which one to disconnect + uint256 deterministicOutbound = DeterministicOutboundConnection(forMember, dmn->proTxHash); + if (!onlyOutbound || deterministicOutbound == dmn->proTxHash) { + result.emplace(dmn->proTxHash); + } + } + return result; +} + +std::set CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount) +{ + static uint256 qwatchConnectionSeed; + static std::atomic qwatchConnectionSeedGenerated{false}; + static RecursiveMutex qwatchConnectionSeedCs; + if (!qwatchConnectionSeedGenerated) { + LOCK(qwatchConnectionSeedCs); + if (!qwatchConnectionSeedGenerated) { + qwatchConnectionSeed = GetRandHash(); + qwatchConnectionSeedGenerated = true; + } + } + + std::set result; + uint256 rnd = qwatchConnectionSeed; + for (size_t i = 0; i < connectionCount; i++) { + rnd = ::SerializeHash(std::make_pair(rnd, std::make_pair(static_cast(llmqType), pindexQuorum->GetBlockHash()))); + result.emplace(rnd.GetUint64(0) % memberCount); + } + return result; +} + +void EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash) +{ + auto members = deterministicMNManager->GetAllQuorumMembers(llmqType, pindexQuorum); + bool isMember = std::find_if(members.begin(), members.end(), [&](const CDeterministicMNCPtr& dmn) { return dmn->proTxHash == myProTxHash; }) != members.end(); + + if (!isMember) { // && !CLLMQUtils::IsWatchQuorumsEnabled()) { + return; + } + + std::set connections; + std::set relayMembers; + if (isMember) { + connections = GetQuorumConnections(llmqType, pindexQuorum, myProTxHash, true); + relayMembers = GetQuorumRelayMembers(llmqType, pindexQuorum, myProTxHash, true); + } else { + auto cindexes = CalcDeterministicWatchConnections(llmqType, pindexQuorum, members.size(), 1); + for (auto idx : cindexes) { + connections.emplace(members[idx]->proTxHash); + } + relayMembers = connections; + } + if (!connections.empty()) { + auto connman = g_connman->GetTierTwoConnMan(); + if (!connman->hasQuorumNodes(llmqType, pindexQuorum->GetBlockHash()) && LogAcceptCategory(BCLog::LLMQ)) { + auto mnList = deterministicMNManager->GetListAtChainTip(); + std::string debugMsg = strprintf("CLLMQUtils::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, pindexQuorum->GetBlockHash().ToString()); + for (auto& c : connections) { + auto dmn = mnList.GetValidMN(c); + if (!dmn) { + debugMsg += strprintf(" %s (not in valid MN set anymore)\n", c.ToString()); + } else { + debugMsg += strprintf(" %s (%s)\n", c.ToString(), dmn->pdmnState->addr.ToString()); + } + } + LogPrint(BCLog::LLMQ, debugMsg.c_str()); /* Continued */ + } + connman->setQuorumNodes(llmqType, pindexQuorum->GetBlockHash(), connections); + } + if (!relayMembers.empty()) { + auto connman = g_connman->GetTierTwoConnMan(); + connman->setMasternodeQuorumRelayMembers(llmqType, pindexQuorum->GetBlockHash(), relayMembers); + } +} + +void AddQuorumProbeConnections(Consensus::LLMQType llmqType, const CBlockIndex *pindexQuorum, const uint256 &myProTxHash) +{ + auto members = deterministicMNManager->GetAllQuorumMembers(llmqType, pindexQuorum); + auto curTime = GetAdjustedTime(); + + std::set probeConnections; + for (auto& dmn : members) { + if (dmn->proTxHash == myProTxHash) { + continue; + } + auto lastOutbound = g_mmetaman.GetMetaInfo(dmn->proTxHash)->GetLastOutboundSuccess(); + // re-probe after 50 minutes so that the "good connection" check in the DKG doesn't fail just because we're on + // the brink of timeout + if (curTime - lastOutbound > 50 * 60) { + probeConnections.emplace(dmn->proTxHash); + } + } + + if (!probeConnections.empty()) { + if (LogAcceptCategory(BCLog::LLMQ)) { + auto mnList = deterministicMNManager->GetListAtChainTip(); + std::string debugMsg = strprintf("CLLMQUtils::%s -- adding masternodes probes for quorum %s:\n", __func__, pindexQuorum->GetBlockHash().ToString()); + for (auto& c : probeConnections) { + auto dmn = mnList.GetValidMN(c); + if (!dmn) { + debugMsg += strprintf(" %s (not in valid MN set anymore)\n", c.ToString()); + } else { + debugMsg += strprintf(" %s (%s)\n", c.ToString(), dmn->pdmnState->addr.ToString()); + } + } + LogPrint(BCLog::LLMQ, debugMsg.c_str()); /* Continued */ + } + g_connman->GetTierTwoConnMan()->addPendingProbeConnections(probeConnections); + } +} + +} // namespace llmq diff --git a/src/llmq/quorums_connections.h b/src/llmq/quorums_connections.h new file mode 100644 index 0000000000000..8cfa8383edb89 --- /dev/null +++ b/src/llmq/quorums_connections.h @@ -0,0 +1,26 @@ +// Copyright (c) 2018-2019 The Dash Core developers +// Copyright (c) 2022 The PIVX developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_QUORUMS_CONNECTIONS_H +#define PIVX_QUORUMS_CONNECTIONS_H + +#include "consensus/params.h" + +class CBlockIndex; + +namespace llmq { + +// Deterministically selects which node should initiate the mnauth process +uint256 DeterministicOutboundConnection(const uint256& proTxHash1, const uint256& proTxHash2); + +std::set GetQuorumRelayMembers(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember, bool onlyOutbound); +std::set CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount); + +void EnsureQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash); +void AddQuorumProbeConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& myProTxHash); + +} // namespace llmq + +#endif // PIVX_QUORUMS_CONNECTIONS_H diff --git a/src/llmq/quorums_dkgsession.cpp b/src/llmq/quorums_dkgsession.cpp index 178e2a2089c00..7d93f432389a0 100644 --- a/src/llmq/quorums_dkgsession.cpp +++ b/src/llmq/quorums_dkgsession.cpp @@ -10,13 +10,14 @@ #include "cxxtimer.hpp" #include "evo/specialtx_validation.h" #include "init.h" +#include "llmq/quorums_connections.h" #include "llmq/quorums_commitment.h" #include "llmq/quorums_debug.h" #include "llmq/quorums_dkgsessionmgr.h" -#include "llmq/quorums_utils.h" #include "net.h" #include "netmessagemaker.h" #include "spork.h" +#include "tiertwo/masternode_meta_manager.h" #include "univalue.h" #include "validation.h" @@ -126,7 +127,7 @@ bool CDKGSession::Init(const CBlockIndex* _pindexQuorum, const std::vectorInitLocalSessionStatus(params.type, pindexQuorum->GetBlockHash(), pindexQuorum->nHeight); - relayMembers = deterministicMNManager->GetQuorumRelayMembers(params.type, pindexQuorum); + relayMembers = GetQuorumRelayMembers(params.type, pindexQuorum, myProTxHash, true); LogPrint(BCLog::DKG, "CDKGSession::%s: initialized as member. mns=%d\n", __func__, mns.size()); } @@ -435,9 +436,45 @@ void CDKGSession::VerifyAndComplain(CDKGPendingMessages& pendingMessages) logger.Batch("verified contributions. time=%d", t1.count()); logger.Flush(); + VerifyConnectionAndMinProtoVersions(); + SendComplaint(pendingMessages); } +void CDKGSession::VerifyConnectionAndMinProtoVersions() +{ + CDKGLogger logger(*this, __func__); + + std::unordered_map protoMap; + g_connman->ForEachNode([&](const CNode* pnode) { + if (pnode->verifiedProRegTxHash.IsNull()) { + return; + } + protoMap.emplace(pnode->verifiedProRegTxHash, pnode->nVersion); + }); + + for (auto& m : members) { + if (m->dmn->proTxHash == myProTxHash) { + continue; + } + + auto it = protoMap.find(m->dmn->proTxHash); + if (it == protoMap.end()) { + m->bad = true; + logger.Batch("%s is not connected to us", m->dmn->proTxHash.ToString()); + } else if (it != protoMap.end() && it->second < MNAUTH_NODE_VER_VERSION) { + m->bad = true; + logger.Batch("%s does not have min proto version %d (has %d)", m->dmn->proTxHash.ToString(), MNAUTH_NODE_VER_VERSION, it->second); + } + + auto lastOutbound = g_mmetaman.GetMetaInfo(m->dmn->proTxHash)->GetLastOutboundSuccess(); + if (GetAdjustedTime() - lastOutbound > 60 * 60) { + m->bad = true; + logger.Batch("%s no outbound connection since %d seconds", m->dmn->proTxHash.ToString(), GetAdjustedTime() - lastOutbound); + } + } +} + void CDKGSession::SendComplaint(CDKGPendingMessages& pendingMessages) { CDKGLogger logger(*this, __func__); diff --git a/src/llmq/quorums_dkgsession.h b/src/llmq/quorums_dkgsession.h index cf53f348382a3..93980e698690e 100644 --- a/src/llmq/quorums_dkgsession.h +++ b/src/llmq/quorums_dkgsession.h @@ -306,6 +306,7 @@ class CDKGSession // Phase 2: complaint void VerifyAndComplain(CDKGPendingMessages& pendingMessages); + void VerifyConnectionAndMinProtoVersions(); void SendComplaint(CDKGPendingMessages& pendingMessages); bool PreVerifyMessage(const uint256& hash, const CDKGComplaint& qc, bool& retBan) const; void ReceiveMessage(const uint256& hash, const CDKGComplaint& qc, bool& retBan); diff --git a/src/llmq/quorums_dkgsessionhandler.cpp b/src/llmq/quorums_dkgsessionhandler.cpp index ba6a009a31e8d..92eb729337db2 100644 --- a/src/llmq/quorums_dkgsessionhandler.cpp +++ b/src/llmq/quorums_dkgsessionhandler.cpp @@ -8,8 +8,8 @@ #include "activemasternode.h" #include "chainparams.h" #include "llmq/quorums_blockprocessor.h" +#include "llmq/quorums_connections.h" #include "llmq/quorums_debug.h" -#include "llmq/quorums_utils.h" #include "net_processing.h" #include "shutdown.h" #include "util/threadnames.h" @@ -522,11 +522,10 @@ void CDKGSessionHandler::HandleDKGRound() return changed; }); - /* TODO + EnsureQuorumConnections(params.type, pindexQuorum, curSession->myProTxHash); if (curSession->AreWeMember()) { - utils::EnsureQuorumConnections(params.type, pindexQuorum, curSession->myProTxHash); + AddQuorumProbeConnections(params.type, pindexQuorum, curSession->myProTxHash); } - */ WaitForNextPhase(QuorumPhase_Initialized, QuorumPhase_Contribute, curQuorumHash, []{return false;}); diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index 2186aa3ba3a8b..3b77a0d9da3c7 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -5,9 +5,7 @@ #include "llmq/quorums_utils.h" #include "bls/bls_wrapper.h" -#include "chainparams.h" -#include "random.h" -#include "spork.h" +#include "hash.h" namespace llmq { @@ -45,28 +43,6 @@ std::string ToHexStr(const std::vector& vBits) return HexStr(vBytes); } -uint256 DeterministicOutboundConnection(const uint256& proTxHash1, const uint256& proTxHash2) -{ - // We need to deterministically select who is going to initiate the connection. The naive way would be to simply - // return the min(proTxHash1, proTxHash2), but this would create a bias towards MNs with a numerically low - // hash. To fix this, we return the proTxHash that has the lowest value of: - // hash(min(proTxHash1, proTxHash2), max(proTxHash1, proTxHash2), proTxHashX) - // where proTxHashX is the proTxHash to compare - uint256 h1; - uint256 h2; - if (proTxHash1 < proTxHash2) { - h1 = ::SerializeHash(std::make_tuple(proTxHash1, proTxHash2, proTxHash1)); - h2 = ::SerializeHash(std::make_tuple(proTxHash1, proTxHash2, proTxHash2)); - } else { - h1 = ::SerializeHash(std::make_tuple(proTxHash2, proTxHash1, proTxHash1)); - h2 = ::SerializeHash(std::make_tuple(proTxHash2, proTxHash1, proTxHash2)); - } - if (h1 < h2) { - return proTxHash1; - } - return proTxHash2; -} - } // namespace llmq::utils } // namespace llmq diff --git a/src/llmq/quorums_utils.h b/src/llmq/quorums_utils.h index 87a72f8bb0a7a..789a716a80ebe 100644 --- a/src/llmq/quorums_utils.h +++ b/src/llmq/quorums_utils.h @@ -6,7 +6,6 @@ #define PIVX_QUORUMS_UTILS_H #include "consensus/params.h" -#include "net.h" #include @@ -27,11 +26,7 @@ uint256 BuildSignHash(const T& s) return BuildSignHash((Consensus::LLMQType)s.llmqType, s.quorumHash, s.id, s.msgHash); } -// Deterministically selects which node should initiate the mnauth process -uint256 DeterministicOutboundConnection(const uint256& proTxHash1, const uint256& proTxHash2); - std::string ToHexStr(const std::vector& vBits); - } // namespace llmq::utils } // namespace llmq diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index cf5845a8cb65b..94f91e4749dde 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -48,8 +48,8 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "chain -> legacy/stakemodifier -> validation -> checkpoints -> chain" "chain -> legacy/stakemodifier -> validation -> undo -> chain" "chain -> legacy/stakemodifier -> validation -> pow -> chain" + "evo/deterministicmns -> masternodeman -> net -> tiertwo/net_masternodes -> evo/deterministicmns" "evo/deterministicmns -> masternodeman -> validation -> validationinterface -> evo/deterministicmns" - "evo/deterministicmns -> llmq/quorums_utils -> net -> tiertwo/net_masternodes -> evo/deterministicmns" ) EXIT_CODE=0