feat: added new cmd "zetatool" and sub cmd filterdeposits

Makefile

@@ -98,6 +98,10 @@ install-zetaclient-race-test-only-build: go.sum
 		@echo "--> Installing zetaclientd"
 		@go install -race -mod=readonly $(BUILD_FLAGS) ./cmd/zetaclientd
 
+install-zetatool: go.sum
+		@echo "--> Installing zetatool"
+		@go install -mod=readonly $(BUILD_FLAGS) ./cmd/zetatool
+
 ###############################################################################
 ###                             Local network                               ###
 ###############################################################################
@@ -286,4 +290,17 @@ mainnet-bitcoind-node:
 	cd contrib/mainnet/bitcoind && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
 
 athens3-zetarpc-node:
-	cd contrib/athens3/zetacored && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
+	cd contrib/athens3/zetacored && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
+
+###############################################################################
+###                               Debug Tools                               ###
+###############################################################################
+
+filter-missed-btc: install-zetatool
+	zetatool filterdeposit btc --config ./tool/filter_missed_deposits/zetatool_config.json
+
+filter-missed-eth: install-zetatool
+	zetatool filterdeposit eth \
+		--config ./tool/filter_missed_deposits/zetatool_config.json \
+		--evm-max-range 1000 \
+		--evm-start-block 19464041

changelog.md

@@ -25,6 +25,7 @@
 * [1789](https://github.com/zeta-chain/node/issues/1789) - block cross-chain transactions that involve restricted addresses
 * [1755](https://github.com/zeta-chain/node/issues/1755) - use evm JSON RPC for inbound tx (including blob tx) observation.
 * [1815](https://github.com/zeta-chain/node/pull/1815) - add authority module for authorized actions
+* [1884](https://github.com/zeta-chain/node/pull/1884) - added zetatool cmd, added subcommand to filter deposits
 
 ### Tests
 

cmd/zetatool/config/config.go

@@ -0,0 +1,71 @@
+package config
+
+import (
+	"encoding/json"
+
+	"github.com/spf13/afero"
+)
+
+var AppFs = afero.NewOsFs()
+
+const (
+	FlagConfig         = "config"
+	defaultCfgFileName = "zetatool_config.json"
+	ZetaURL            = "127.0.0.1:1317"
+	BtcExplorerURL     = "https://blockstream.info/api/"
+	EthRPCURL          = "https://ethereum-rpc.publicnode.com"
+	ConnectorAddress   = "0x000007Cf399229b2f5A4D043F20E90C9C98B7C6a"
+	CustodyAddress     = "0x0000030Ec64DF25301d8414eE5a29588C4B0dE10"
+)
+
+// Config is a struct the defines the configuration fields used by zetatool
+type Config struct {
+	ZetaURL          string
+	BtcExplorerURL   string
+	EthRPCURL        string
+	EtherscanAPIkey  string
+	ConnectorAddress string
+	CustodyAddress   string
+}
+
+func DefaultConfig() *Config {
+	return &Config{
+		ZetaURL:          ZetaURL,
+		BtcExplorerURL:   BtcExplorerURL,
+		EthRPCURL:        EthRPCURL,
+		ConnectorAddress: ConnectorAddress,
+		CustodyAddress:   CustodyAddress,
+	}
+}
+
+func (c *Config) Save() error {
+	file, err := json.MarshalIndent(c, "", " ")
+	if err != nil {
+		return err
+	}
+	err = afero.WriteFile(AppFs, defaultCfgFileName, file, 0600)
+	return err
+}
+
+func (c *Config) Read(filename string) error {
+	data, err := afero.ReadFile(AppFs, filename)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(data, c)
+	return err
+}
+
+func GetConfig(filename string) (*Config, error) {
+	//Check if cfgFile is empty, if so return default Config and save to file
+	if filename == "" {
+		cfg := DefaultConfig()
+		err := cfg.Save()
+		return cfg, err
+	}
+
+	//if file is specified, open file and return struct
+	cfg := &Config{}
+	err := cfg.Read(filename)
+	return cfg, err
+}

cmd/zetatool/config/config_test.go

@@ -0,0 +1,77 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultConfig(t *testing.T) {
+	cfg := DefaultConfig()
+	require.Equal(t, cfg.EthRPCURL, EthRPCURL)
+	require.Equal(t, cfg.ZetaURL, ZetaURL)
+	require.Equal(t, cfg.BtcExplorerURL, BtcExplorerURL)
+	require.Equal(t, cfg.ConnectorAddress, ConnectorAddress)
+	require.Equal(t, cfg.CustodyAddress, CustodyAddress)
+}
+
+func TestGetConfig(t *testing.T) {
+	AppFs = afero.NewMemMapFs()
+	defaultCfg := DefaultConfig()
+
+	t.Run("No config file specified", func(t *testing.T) {
+		cfg, err := GetConfig("")
+		require.NoError(t, err)
+		require.Equal(t, cfg, defaultCfg)
+
+		exists, err := afero.Exists(AppFs, defaultCfgFileName)
+		require.NoError(t, err)
+		require.True(t, exists)
+	})
+
+	t.Run("config file specified", func(t *testing.T) {
+		cfg, err := GetConfig(defaultCfgFileName)
+		require.NoError(t, err)
+		require.Equal(t, cfg, defaultCfg)
+	})
+}
+
+func TestConfig_Read(t *testing.T) {
+	AppFs = afero.NewMemMapFs()
+	cfg, err := GetConfig("")
+	require.NoError(t, err)
+
+	t.Run("read existing file", func(t *testing.T) {
+		c := &Config{}
+		err := c.Read(defaultCfgFileName)
+		require.NoError(t, err)
+		require.Equal(t, c, cfg)
+	})
+
+	t.Run("read non-existent file", func(t *testing.T) {
+		err := AppFs.Remove(defaultCfgFileName)
+		require.NoError(t, err)
+		c := &Config{}
+		err = c.Read(defaultCfgFileName)
+		require.ErrorContains(t, err, "file does not exist")
+		require.NotEqual(t, c, cfg)
+	})
+}
+
+func TestConfig_Save(t *testing.T) {
+	AppFs = afero.NewMemMapFs()
+	cfg := DefaultConfig()
+	cfg.EtherscanAPIkey = "DIFFERENTAPIKEY"
+
+	t.Run("save modified cfg", func(t *testing.T) {
+		err := cfg.Save()
+		require.NoError(t, err)
+
+		newCfg, err := GetConfig(defaultCfgFileName)
+		require.NoError(t, err)
+		require.Equal(t, cfg, newCfg)
+	})
+
+	// Should test invalid json encoding but currently not able to without interface
+}

cmd/zetatool/filterdeposit/btc.go

@@ -0,0 +1,188 @@
+package filterdeposit
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/zeta-chain/zetacore/cmd/zetatool/config"
+	"github.com/zeta-chain/zetacore/common"
+)
+
+func NewBtcCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "btc",
+		Short: "Filter inbound btc deposits",
+		RunE:  FilterBTCTransactions,
+	}
+}
+
+// FilterBTCTransactions is a command that queries the bitcoin explorer for inbound transactions that qualify for
+// cross chain transactions.
+func FilterBTCTransactions(cmd *cobra.Command, _ []string) error {
+	configFile, err := cmd.Flags().GetString(config.FlagConfig)
+	fmt.Println("config file name: ", configFile)
+	if err != nil {
+		return err
+	}
+	btcChainID, err := cmd.Flags().GetString(BTCChainIDFlag)
+	if err != nil {
+		return err
+	}
+	cfg, err := config.GetConfig(configFile)
+	if err != nil {
+		return err
+	}
+	fmt.Println("getting tss Address")
+	res, err := GetTssAddress(cfg, btcChainID)
+	if err != nil {
+		return err
+	}
+	fmt.Println("got tss Address")
+	list, err := getHashList(cfg, res.Btc)
+	if err != nil {
+		return err
+	}
+
+	_, err = CheckForCCTX(list, cfg)
+	return err
+}
+
+// getHashList is called by FilterBTCTransactions to help query and filter inbound transactions on btc
+func getHashList(cfg *config.Config, tssAddress string) ([]Deposit, error) {
+	var list []Deposit
+	lastHash := ""
+
+	// Setup URL for query
+	btcURL, err := url.JoinPath(cfg.BtcExplorerURL, "address", tssAddress, "txs")
+	if err != nil {
+		return list, err
+	}
+
+	// This loop will query the bitcoin explorer for transactions associated with the TSS address. Since the api only
+	// allows a response of 25 transactions per request, several requests will be required in order to retrieve a
+	// complete list.
+	for {
+		// The Next Query is determined by the last transaction hash provided by the previous response.
+		nextQuery := btcURL
+		if lastHash != "" {
+			nextQuery, err = url.JoinPath(btcURL, "chain", lastHash)
+			if err != nil {
+				return list, err
+			}
+		}
+		// #nosec G107 url must be variable
+		res, getErr := http.Get(nextQuery)
+		if getErr != nil {
+			return list, getErr
+		}
+
+		body, readErr := ioutil.ReadAll(res.Body)
+		if readErr != nil {
+			return list, readErr
+		}
+		closeErr := res.Body.Close()
+		if closeErr != nil {
+			return list, closeErr
+		}
+
+		// NOTE: decoding json from request dynamically is not ideal, however there isn't a detailed, defined data structure
+		// provided by blockstream. Will need to create one in the future using following definition:
+		// https://github.com/Blockstream/esplora/blob/master/API.md#transaction-format
+		var txns []map[string]interface{}
+		err := json.Unmarshal(body, &txns)
+		if err != nil {
+			return list, err
+		}
+
+		if len(txns) == 0 {
+			break
+		}
+
+		fmt.Println("Length of txns: ", len(txns))
+
+		// The "/address" blockstream api provides a maximum of 25 transactions associated with a given address. This
+		// loop will iterate over that list of transactions to determine whether each transaction can be considered
+		// a deposit to ZetaChain.
+		for _, txn := range txns {
+			// Get tx hash of the current transaction
+			hash := txn["txid"].(string)
+
+			// Read the first output of the transaction and parse the destination address.
+			// This address should be the TSS address.
+			vout := txn["vout"].([]interface{})
+			vout0 := vout[0].(map[string]interface{})
+			var vout1 map[string]interface{}
+			if len(vout) > 1 {
+				vout1 = vout[1].(map[string]interface{})
+			} else {
+				continue
+			}
+			_, found := vout0["scriptpubkey"]
+			scriptpubkey := ""
+			if found {
+				scriptpubkey = vout0["scriptpubkey"].(string)
+			}
+			_, found = vout0["scriptpubkey_address"]
+			targetAddr := ""
+			if found {
+				targetAddr = vout0["scriptpubkey_address"].(string)
+			}
+
+			//Check if txn is confirmed
+			status := txn["status"].(map[string]interface{})
+			confirmed := status["confirmed"].(bool)
+			if !confirmed {
+				continue
+			}
+
+			//Filter out deposits less than min base fee
+			if vout0["value"].(float64) < 1360 {
+				continue
+			}
+
+			//Check if Deposit is a donation
+			scriptpubkey1 := vout1["scriptpubkey"].(string)
+			if len(scriptpubkey1) >= 4 && scriptpubkey1[:2] == "6a" {
+				memoSize, err := strconv.ParseInt(scriptpubkey1[2:4], 16, 32)
+				if err != nil {
+					continue
+				}
+				if int(memoSize) != (len(scriptpubkey1)-4)/2 {
+					continue
+				}
+				memoBytes, err := hex.DecodeString(scriptpubkey1[4:])
+				if err != nil {
+					continue
+				}
+				if bytes.Equal(memoBytes, []byte(common.DonationMessage)) {
+					continue
+				}
+			} else {
+				continue
+			}
+
+			//Make sure Deposit is sent to correct tss address
+			if strings.Compare("0014", scriptpubkey[:4]) == 0 && targetAddr == tssAddress {
+				entry := Deposit{
+					hash,
+					// #nosec G701 parsing json requires float64 type from blockstream
+					uint64(vout0["value"].(float64)),
+				}
+				list = append(list, entry)
+			}
+		}
+
+		lastTxn := txns[len(txns)-1]
+		lastHash = lastTxn["txid"].(string)
+	}
+
+	return list, nil
+}