fix: special handle Bitcoin Testnet gas price estimator (#2452)

* special handle bitcoin testnet gas price estimator

* let specialHandleFeeRate return the fee rate number and calls PostGasPrice in single spot

* added some explanations for specialHandleFeeRate

---------

Co-authored-by: Lucas Bertrand <[email protected]>

Author: ws4charlie

changelog.md

@@ -90,6 +90,7 @@
 * [2327](https://github.com/zeta-chain/node/pull/2327) - partially cherry picked the fix to Bitcoin outbound dust amount
 * [2362](https://github.com/zeta-chain/node/pull/2362) - set 1000 satoshis as minimum BTC amount that can be withdrawn from zEVM
 * [2382](https://github.com/zeta-chain/node/pull/2382) - add tx input and gas in rpc methods for synthetic eth txs
+* [2396](https://github.com/zeta-chain/node/issues/2386) - special handle bitcoin testnet gas price estimator
 * [2434](https://github.com/zeta-chain/node/pull/2434) - the default database when running `zetacored init` is now pebbledb
 
 ### CI

zetaclient/chains/bitcoin/observer/observer.go

@@ -23,6 +23,7 @@ import (
 	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 	"github.com/zeta-chain/zetacore/zetaclient/chains/base"
 	"github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin"
+	"github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/rpc"
 	"github.com/zeta-chain/zetacore/zetaclient/chains/interfaces"
 	"github.com/zeta-chain/zetacore/zetaclient/context"
 	"github.com/zeta-chain/zetacore/zetaclient/metrics"
@@ -341,44 +342,44 @@ func (ob *Observer) WatchGasPrice() {
 // PostGasPrice posts gas price to zetacore
 // TODO(revamp): move to gas price file
 func (ob *Observer) PostGasPrice() error {
-	// hardcode gas price here since this RPC is not available on regtest
-	if chains.IsBitcoinRegnet(ob.Chain().ChainId) {
-		blockNumber, err := ob.btcClient.GetBlockCount()
+	var err error
+	feeRateEstimated := uint64(0)
+
+	// special handle regnet and testnet gas rate
+	// regnet:  RPC 'EstimateSmartFee' is not available
+	// testnet: RPC 'EstimateSmartFee' returns unreasonable high gas rate
+	if ob.Chain().NetworkType != chains.NetworkType_mainnet {
+		feeRateEstimated, err = ob.specialHandleFeeRate()
 		if err != nil {
+			ob.logger.GasPrice.Err(err).Msg("error specialHandleFeeRate")
 			return err
 		}
-
-		// #nosec G701 always in range
-		_, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), 1, "100", uint64(blockNumber))
+	} else {
+		// EstimateSmartFee returns the fees per kilobyte (BTC/kb) targeting given block confirmation
+		feeResult, err := ob.btcClient.EstimateSmartFee(1, &btcjson.EstimateModeEconomical)
 		if err != nil {
-			ob.logger.GasPrice.Err(err).Msg("PostGasPrice:")
+			ob.logger.GasPrice.Err(err).Msg("error EstimateSmartFee")
 			return err
 		}
-		return nil
-	}
-
-	// EstimateSmartFee returns the fees per kilobyte (BTC/kb) targeting given block confirmation
-	feeResult, err := ob.btcClient.EstimateSmartFee(1, &btcjson.EstimateModeEconomical)
-	if err != nil {
-		return err
-	}
-	if feeResult.Errors != nil || feeResult.FeeRate == nil {
-		return fmt.Errorf("error getting gas price: %s", feeResult.Errors)
-	}
-	if *feeResult.FeeRate > math.MaxInt64 {
-		return fmt.Errorf("gas price is too large: %f", *feeResult.FeeRate)
+		if feeResult.Errors != nil || feeResult.FeeRate == nil {
+			return fmt.Errorf("error getting gas price: %s", feeResult.Errors)
+		}
+		if *feeResult.FeeRate > math.MaxInt64 {
+			return fmt.Errorf("gas price is too large: %f", *feeResult.FeeRate)
+		}
+		feeRateEstimated = bitcoin.FeeRateToSatPerByte(*feeResult.FeeRate).Uint64()
 	}
-	feeRatePerByte := bitcoin.FeeRateToSatPerByte(*feeResult.FeeRate)
 
+	// query the current block number
 	blockNumber, err := ob.btcClient.GetBlockCount()
 	if err != nil {
 		return err
 	}
 
 	// #nosec G701 always positive
-	_, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), feeRatePerByte.Uint64(), "100", uint64(blockNumber))
+	_, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), feeRateEstimated, "100", uint64(blockNumber))
 	if err != nil {
-		ob.logger.GasPrice.Err(err).Msg("PostGasPrice:")
+		ob.logger.GasPrice.Err(err).Msg("err PostGasPrice")
 		return err
 	}
 
@@ -644,6 +645,23 @@ func (ob *Observer) LoadBroadcastedTxMap() error {
 	return nil
 }
 
+// specialHandleFeeRate handles the fee rate for regnet and testnet
+func (ob *Observer) specialHandleFeeRate() (uint64, error) {
+	switch ob.Chain().NetworkType {
+	case chains.NetworkType_privnet:
+		// hardcode gas price for regnet
+		return 1, nil
+	case chains.NetworkType_testnet:
+		feeRateEstimated, err := rpc.GetRecentFeeRate(ob.btcClient, ob.netParams)
+		if err != nil {
+			return 0, errors.Wrapf(err, "error GetRecentFeeRate")
+		}
+		return feeRateEstimated, nil
+	default:
+		return 0, fmt.Errorf(" unsupported bitcoin network type %d", ob.Chain().NetworkType)
+	}
+}
+
 // isTssTransaction checks if a given transaction was sent by TSS itself.
 // An unconfirmed transaction is safe to spend only if it was sent by TSS and verified by ourselves.
 func (ob *Observer) isTssTransaction(txid string) bool {

zetaclient/chains/bitcoin/rpc/rpc.go

@@ -4,14 +4,24 @@ import (
 	"fmt"
 
 	"github.com/btcsuite/btcd/btcjson"
+	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/pkg/errors"
 
+	"github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin"
 	"github.com/zeta-chain/zetacore/zetaclient/chains/interfaces"
 	"github.com/zeta-chain/zetacore/zetaclient/config"
 )
 
+const (
+	// feeRateCountBackBlocks is the default number of blocks to look back for fee rate estimation
+	feeRateCountBackBlocks = 2
+
+	// defaultTestnetFeeRate is the default fee rate for testnet, 10 sat/byte
+	defaultTestnetFeeRate = 10
+)
+
 // NewRPCClient creates a new RPC client by the given config.
 func NewRPCClient(btcConfig config.BTCConfig) (*rpcclient.Client, error) {
 	connCfg := &rpcclient.ConnConfig{
@@ -107,3 +117,43 @@ func GetRawTxResult(
 	// res.Confirmations < 0 (meaning not included)
 	return btcjson.TxRawResult{}, fmt.Errorf("GetRawTxResult: tx %s not included yet", hash)
 }
+
+// GetRecentFeeRate gets the highest fee rate from recent blocks
+// Note: this method is only used for testnet
+func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Params) (uint64, error) {
+	blockNumber, err := rpcClient.GetBlockCount()
+	if err != nil {
+		return 0, err
+	}
+
+	// get the highest fee rate among recent 'countBack' blocks to avoid underestimation
+	highestRate := int64(0)
+	for i := int64(0); i < feeRateCountBackBlocks; i++ {
+		// get the block
+		hash, err := rpcClient.GetBlockHash(blockNumber - i)
+		if err != nil {
+			return 0, err
+		}
+		block, err := rpcClient.GetBlockVerboseTx(hash)
+		if err != nil {
+			return 0, err
+		}
+
+		// computes the average fee rate of the block and take the higher rate
+		avgFeeRate, err := bitcoin.CalcBlockAvgFeeRate(block, netParams)
+		if err != nil {
+			return 0, err
+		}
+		if avgFeeRate > highestRate {
+			highestRate = avgFeeRate
+		}
+	}
+
+	// use 10 sat/byte as default estimation if recent fee rate drops to 0
+	if highestRate == 0 {
+		highestRate = defaultTestnetFeeRate
+	}
+
+	// #nosec G701 always in range
+	return uint64(highestRate), nil
+}

zetaclient/chains/bitcoin/rpc/rpc_live_test.go

@@ -59,7 +59,7 @@ func (suite *BitcoinObserverTestSuite) SetupTest() {
 		base.DefaultLogger(), nil)
 	suite.Require().NoError(err)
 	suite.Require().NotNil(ob)
-	suite.rpcClient, err = getRPCClient(18332)
+	suite.rpcClient, err = createRPCClient(18332)
 	suite.Require().NoError(err)
 	skBytes, err := hex.DecodeString(skHex)
 	suite.Require().NoError(err)
@@ -91,13 +91,14 @@ func (suite *BitcoinObserverTestSuite) SetupTest() {
 func (suite *BitcoinObserverTestSuite) TearDownSuite() {
 }
 
-func getRPCClient(chainID int64) (*rpcclient.Client, error) {
+// createRPCClient creates a new Bitcoin RPC client for given chainID
+func createRPCClient(chainID int64) (*rpcclient.Client, error) {
 	var connCfg *rpcclient.ConnConfig
 	rpcMainnet := os.Getenv("BTC_RPC_MAINNET")
 	rpcTestnet := os.Getenv("BTC_RPC_TESTNET")
 
 	// mainnet
-	if chainID == 8332 {
+	if chainID == chains.BitcoinMainnet.ChainId {
 		connCfg = &rpcclient.ConnConfig{
 			Host:         rpcMainnet, // mainnet endpoint goes here
 			User:         "user",
@@ -108,7 +109,7 @@ func getRPCClient(chainID int64) (*rpcclient.Client, error) {
 		}
 	}
 	// testnet3
-	if chainID == 18332 {
+	if chainID == chains.BitcoinTestnet.ChainId {
 		connCfg = &rpcclient.ConnConfig{
 			Host:         rpcTestnet, // testnet endpoint goes here
 			User:         "user",
@@ -218,6 +219,7 @@ func TestBitcoinObserverLive(t *testing.T) {
 	// LiveTestBitcoinFeeRate(t)
 	// LiveTestAvgFeeRateMainnetMempoolSpace(t)
 	// LiveTestAvgFeeRateTestnetMempoolSpace(t)
+	// LiveTestGetRecentFeeRate(t)
 	// LiveTestGetSenderByVin(t)
 }
 
@@ -243,7 +245,7 @@ func LiveTestNewRPCClient(t *testing.T) {
 // LiveTestGetBlockHeightByHash queries Bitcoin block height by hash
 func LiveTestGetBlockHeightByHash(t *testing.T) {
 	// setup Bitcoin client
-	client, err := getRPCClient(8332)
+	client, err := createRPCClient(chains.BitcoinMainnet.ChainId)
 	require.NoError(t, err)
 
 	// the block hashes to test
@@ -265,7 +267,7 @@ func LiveTestGetBlockHeightByHash(t *testing.T) {
 // and compares Conservative and Economical fee rates for different block targets (1 and 2)
 func LiveTestBitcoinFeeRate(t *testing.T) {
 	// setup Bitcoin client
-	client, err := getRPCClient(8332)
+	client, err := createRPCClient(chains.BitcoinMainnet.ChainId)
 	require.NoError(t, err)
 	bn, err := client.GetBlockCount()
 	if err != nil {
@@ -390,7 +392,7 @@ func compareAvgFeeRate(t *testing.T, client *rpcclient.Client, startBlock int, e
 // LiveTestAvgFeeRateMainnetMempoolSpace compares calculated fee rate with mempool.space fee rate for mainnet
 func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) {
 	// setup Bitcoin client
-	client, err := getRPCClient(8332)
+	client, err := createRPCClient(chains.BitcoinMainnet.ChainId)
 	require.NoError(t, err)
 
 	// test against mempool.space API for 10000 blocks
@@ -404,7 +406,7 @@ func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) {
 // LiveTestAvgFeeRateTestnetMempoolSpace compares calculated fee rate with mempool.space fee rate for testnet
 func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) {
 	// setup Bitcoin client
-	client, err := getRPCClient(18332)
+	client, err := createRPCClient(chains.BitcoinTestnet.ChainId)
 	require.NoError(t, err)
 
 	// test against mempool.space API for 10000 blocks
@@ -415,11 +417,23 @@ func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) {
 	compareAvgFeeRate(t, client, startBlock, endBlock, true)
 }
 
+// LiveTestGetRecentFeeRate gets the highest fee rate from recent blocks
+func LiveTestGetRecentFeeRate(t *testing.T) {
+	// setup Bitcoin testnet client
+	client, err := createRPCClient(chains.BitcoinTestnet.ChainId)
+	require.NoError(t, err)
+
+	// get fee rate from recent blocks
+	feeRate, err := rpc.GetRecentFeeRate(client, &chaincfg.TestNet3Params)
+	require.NoError(t, err)
+	require.Greater(t, feeRate, uint64(0))
+}
+
 // LiveTestGetSenderByVin gets sender address for each vin and compares with mempool.space sender address
 func LiveTestGetSenderByVin(t *testing.T) {
 	// setup Bitcoin client
-	chainID := int64(8332)
-	client, err := getRPCClient(chainID)
+	chainID := chains.BitcoinMainnet.ChainId
+	client, err := createRPCClient(chainID)
 	require.NoError(t, err)
 
 	// net params