backport: merge bitcoin#22689, #23381, #22674, #23804, #24310, #24537, #24582, #24404, #25013, #25029, #25254, #25388 (mempool backports)

doc/README.md

@@ -80,6 +80,7 @@ The Dash Core repo's [root README](/README.md) contains relevant information on
 - [Reduce Memory](reduce-memory.md)
 - [Reduce Traffic](reduce-traffic.md)
 - [Tor Support](tor.md)
+- [Transaction Relay Policy](policy/README.md)
 - [ZMQ](zmq.md)
 
 License

doc/policy/README.md

@@ -0,0 +1,10 @@
+# Transaction Relay Policy
+
+Policy is a set of validation rules, in addition to consensus, enforced for unconfirmed
+transactions.
+
+This documentation is not an exhaustive list of all policy rules.
+
+- [Packages](packages.md)
+
+

doc/policy/packages.md

@@ -0,0 +1,62 @@
+# Package Mempool Accept
+
+## Definitions
+
+A **package** is an ordered list of transactions, representable by a connected Directed Acyclic
+Graph (a directed edge exists between a transaction that spends the output of another transaction).
+
+For every transaction `t` in a **topologically sorted** package, if any of its parents are present
+in the package, they appear somewhere in the list before `t`.
+
+A **child-with-unconfirmed-parents** package is a topologically sorted package that consists of
+exactly one child and all of its unconfirmed parents (no other transactions may be present).
+The last transaction in the package is the child, and its package can be canonically defined based
+on the current state: each of its inputs must be available in the UTXO set as of the current chain
+tip or some preceding transaction in the package.
+
+## Package Mempool Acceptance Rules
+
+The following rules are enforced for all packages:
+
+* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_SIZE=101KvB` total size
+   (#20833)
+
+   - *Rationale*: This is already enforced as mempool ancestor/descendant limits. If
+     transactions in a package are all related, exceeding this limit would mean that the package
+     can either be split up or it wouldn't pass individual mempool policy.
+
+   - Note that, if these mempool limits change, package limits should be reconsidered. Users may
+     also configure their mempool limits differently.
+
+* Packages must be topologically sorted. (#20833)
+
+* Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend
+   the same inputs. Packages cannot have duplicate transactions. (#20833)
+
+* No transaction in a package can conflict with a mempool transaction.
+
+* When packages are evaluated against ancestor/descendant limits, the union of all transactions'
+  descendants and ancestors is considered. (#21800)
+
+   - *Rationale*: This is essentially a "worst case" heuristic intended for packages that are
+     heavily connected, i.e. some transaction in the package is the ancestor or descendant of all
+     the other transactions.
+
+The following rules are only enforced for packages to be submitted to the mempool (not enforced for
+test accepts):
+
+* Packages must be child-with-unconfirmed-parents packages. This also means packages must contain at
+  least 2 transactions. (#22674)
+
+* Transactions in the package that have the same txid as another transaction already in the mempool
+  will be removed from the package prior to submission ("deduplication").
+
+   - *Rationale*: Node operators are free to set their mempool policies however they please, nodes
+     may receive transactions in different orders, and malicious counterparties may try to take
+     advantage of policy differences to pin or delay propagation of transactions. As such, it's
+     possible for some package transaction(s) to already be in the mempool, and there is no need to
+     repeat validation for those transactions or double-count them in fees.
+
+   - *Rationale*: We want to prevent potential censorship vectors. We should not reject entire
+     packages because we already have one of the transactions. Also, if an attacker first broadcasts
+     a competing package, the honest package should still be considered for acceptance.

doc/release-notes-6499.md

@@ -0,0 +1,11 @@
+Updated RPCs
+------------
+
+- The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees`
+  returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`,
+  `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)`
+  are deprecated and will be removed in the next major version (use
+  `-deprecated=fees` if needed in this version). The same fee fields can be accessed
+  through the `fees` object in the result. WARNING: deprecated
+  fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all
+  fields in the `fees` object are denominated in DASH.

src/Makefile.am

@@ -300,6 +300,7 @@ BITCOIN_CORE_H = \
   rpc/blockchain.h \
   rpc/client.h \
   rpc/index_util.h \
+  rpc/mempool.h \
   rpc/mining.h \
   rpc/protocol.h \
   rpc/rawtransaction_util.h \
@@ -528,8 +529,10 @@ libbitcoin_server_a_SOURCES = \
   rpc/blockchain.cpp \
   rpc/coinjoin.cpp \
   rpc/evo.cpp \
+  rpc/fees.cpp \
   rpc/index_util.cpp \
   rpc/masternode.cpp \
+  rpc/mempool.cpp \
   rpc/governance.cpp \
   rpc/mining.cpp \
   rpc/misc.cpp \
@@ -538,6 +541,8 @@ libbitcoin_server_a_SOURCES = \
   rpc/rawtransaction.cpp \
   rpc/server.cpp \
   rpc/server_util.cpp \
+  rpc/signmessage.cpp \
+  rpc/txoutproof.cpp \
   script/sigcache.cpp \
   shutdown.cpp \
   spork.cpp \

src/Makefile.test.include

@@ -174,6 +174,7 @@ BITCOIN_TESTS =\
   test/torcontrol_tests.cpp \
   test/transaction_tests.cpp \
   test/txindex_tests.cpp \
+  test/txpackage_tests.cpp \
   test/txreconciliation_tests.cpp \
   test/txvalidation_tests.cpp \
   test/txvalidationcache_tests.cpp \

src/bench/rpc_mempool.cpp

@@ -3,7 +3,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <bench/bench.h>
-#include <rpc/blockchain.h>
+#include <rpc/mempool.h>
 #include <txmempool.h>
 
 #include <univalue.h>

src/net_processing.cpp

@@ -22,6 +22,7 @@
 #include <node/blockstorage.h>
 #include <node/txreconciliation.h>
 #include <policy/policy.h>
+#include <policy/settings.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
 #include <random.h>

src/policy/feerate.cpp

@@ -3,8 +3,8 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <consensus/amount.h>
 #include <policy/feerate.h>
-
 #include <tinyformat.h>
 
 CFeeRate::CFeeRate(const CAmount& nFeePaid, uint32_t num_bytes)

src/policy/feerate.h

@@ -9,7 +9,10 @@
 #include <consensus/amount.h>
 #include <serialize.h>
 
+
+#include <cstdint>
 #include <string>
+#include <type_traits>
 
 const std::string CURRENCY_UNIT = "DASH"; // One formatted unit
 const std::string CURRENCY_ATOM = "duff"; // One indivisible minimum value unit

src/policy/fees.cpp

@@ -6,12 +6,30 @@
 #include <policy/fees.h>
 
 #include <clientversion.h>
+#include <consensus/amount.h>
 #include <fs.h>
 #include <logging.h>
+#include <policy/feerate.h>
+#include <primitives/transaction.h>
+#include <random.h>
+#include <serialize.h>
 #include <streams.h>
+#include <sync.h>
+#include <tinyformat.h>
 #include <txmempool.h>
+#include <uint256.h>
 #include <util/serfloat.h>
 #include <util/system.h>
+#include <util/time.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <stdexcept>
+#include <utility>
 
 static const char* FEE_ESTIMATES_FILENAME = "fee_estimates.dat";
 

src/policy/fees.h

@@ -7,20 +7,20 @@
 
 #include <consensus/amount.h>
 #include <policy/feerate.h>
-#include <uint256.h>
 #include <random.h>
 #include <sync.h>
+#include <threadsafety.h>
+#include <uint256.h>
 
 #include <array>
 #include <map>
 #include <memory>
+#include <set>
 #include <string>
 #include <vector>
 
 class CAutoFile;
-class CFeeRate;
 class CTxMemPoolEntry;
-class CTxMemPool;
 class TxConfirmStats;
 
 /* Identifier for each of the 3 different TxConfirmStats which will track

src/policy/packages.cpp

@@ -2,12 +2,16 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-#include <consensus/validation.h>
 #include <policy/packages.h>
+#include <policy/policy.h>
 #include <primitives/transaction.h>
 #include <uint256.h>
 #include <util/hasher.h>
 
+#include <algorithm>
+#include <cassert>
+#include <iterator>
+#include <memory>
 #include <numeric>
 #include <unordered_set>
 
@@ -60,3 +64,20 @@ bool CheckPackage(const Package& txns, PackageValidationState& state)
     }
     return true;
 }
+
+bool IsChildWithParents(const Package& package)
+{
+    assert(std::all_of(package.cbegin(), package.cend(), [](const auto& tx){return tx != nullptr;}));
+    if (package.size() < 2) return false;
+
+    // The package is expected to be sorted, so the last transaction is the child.
+    const auto& child = package.back();
+    std::unordered_set<uint256, SaltedTxidHasher> input_txids;
+    std::transform(child->vin.cbegin(), child->vin.cend(),
+                   std::inserter(input_txids, input_txids.end()),
+                   [](const auto& input) { return input.prevout.hash; });
+
+    // Every transaction must be a parent of the last transaction in the package.
+    return std::all_of(package.cbegin(), package.cend() - 1,
+                       [&input_txids](const auto& ptx) { return input_txids.count(ptx->GetHash()) > 0; });
+}

src/policy/packages.h

@@ -5,10 +5,12 @@
 #ifndef BITCOIN_POLICY_PACKAGES_H
 #define BITCOIN_POLICY_PACKAGES_H
 
+#include <consensus/consensus.h>
 #include <consensus/validation.h>
 #include <policy/policy.h>
 #include <primitives/transaction.h>
 
+#include <cstdint>
 #include <vector>
 
 /** Default maximum number of transactions in a package. */
@@ -17,6 +19,15 @@ static constexpr uint32_t MAX_PACKAGE_COUNT{25};
 static constexpr uint32_t MAX_PACKAGE_SIZE{101};
 static_assert(MAX_PACKAGE_SIZE * 1000 >= MAX_STANDARD_TX_SIZE);
 
+// If a package is submitted, it must be within the mempool's ancestor/descendant limits. Since a
+// submitted package must be child-with-unconfirmed-parents (all of the transactions are an ancestor
+// of the child), package limits are ultimately bounded by mempool package limits. Ensure that the
+// defaults reflect this constraint.
+static_assert(DEFAULT_DESCENDANT_LIMIT >= MAX_PACKAGE_COUNT);
+static_assert(DEFAULT_ANCESTOR_LIMIT >= MAX_PACKAGE_COUNT);
+static_assert(DEFAULT_ANCESTOR_SIZE_LIMIT >= MAX_PACKAGE_SIZE);
+static_assert(DEFAULT_DESCENDANT_SIZE_LIMIT >= MAX_PACKAGE_SIZE);
+
 /** A "reason" why a package was invalid. It may be that one or more of the included
  * transactions is invalid or the package itself violates our rules.
  * We don't distinguish between consensus and policy violations right now.
@@ -25,6 +36,7 @@ enum class PackageValidationResult {
     PCKG_RESULT_UNSET = 0,        //!< Initial value. The package has not yet been rejected.
     PCKG_POLICY,                  //!< The package itself is invalid (e.g. too many transactions).
     PCKG_TX,                      //!< At least one tx is invalid.
+    PCKG_MEMPOOL_ERROR,           //!< Mempool logic error.
 };
 
 /** A package is an ordered list of transactions. The transactions cannot conflict with (spend the
@@ -41,4 +53,10 @@ class PackageValidationState : public ValidationState<PackageValidationResult> {
  */
 bool CheckPackage(const Package& txns, PackageValidationState& state);
 
+/** Context-free check that a package is exactly one child and its parents; not all parents need to
+ * be present, but the package must not contain any transactions that are not the child's parents.
+ * It is expected to be sorted, which means the last transaction must be the child.
+ */
+bool IsChildWithParents(const Package& package);
+
 #endif // BITCOIN_POLICY_PACKAGES_H

src/policy/policy.cpp

@@ -8,8 +8,20 @@
 #include <policy/policy.h>
 
 #include <coins.h>
-#include <policy/settings.h>
-
+#include <consensus/amount.h>
+#include <consensus/consensus.h>
+#include <consensus/validation.h>
+#include <policy/feerate.h>
+#include <primitives/transaction.h>
+#include <script/interpreter.h>
+#include <script/script.h>
+#include <script/standard.h>
+#include <serialize.h>
+#include <span.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <vector>
 
 CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn)
 {