feat: added new cmd "zetatool" and sub cmd filterdeposits

Makefile

@@ -98,6 +98,10 @@ install-zetaclient-race-test-only-build: go.sum
 		@echo "--> Installing zetaclientd"
 		@go install -race -mod=readonly $(BUILD_FLAGS) ./cmd/zetaclientd
 
+install-zetatool: go.sum
+		@echo "--> Installing zetatool"
+		@go install -mod=readonly $(BUILD_FLAGS) ./cmd/zetatool
+
 ###############################################################################
 ###                             Local network                               ###
 ###############################################################################
@@ -286,4 +290,14 @@ mainnet-bitcoind-node:
 	cd contrib/mainnet/bitcoind && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
 
 athens3-zetarpc-node:
-	cd contrib/athens3/zetacored && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
+	cd contrib/athens3/zetacored && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
+
+###############################################################################
+###                               Debug Tools                               ###
+###############################################################################
+
+filter-missed-btc: install-zetatool
+	./tool/filter_missed_deposits/filter_missed_btc.sh
+
+filter-missed-eth: install-zetatool
+	./tool/filter_missed_deposits/filter_missed_eth.sh

changelog.md

@@ -23,6 +23,7 @@
 * [1789](https://github.com/zeta-chain/node/issues/1789) - block cross-chain transactions that involve restricted addresses
 * [1755](https://github.com/zeta-chain/node/issues/1755) - use evm JSON RPC for inbound tx (including blob tx) observation.
 * [1815](https://github.com/zeta-chain/node/pull/1815) - add authority module for authorized actions
+* [1884](https://github.com/zeta-chain/node/pull/1884) - added zetatool cmd, added subcommand to filter deposits
 
 ### Tests
 

cmd/zetatool/config/config.go

@@ -0,0 +1,81 @@
+package config
+
+import (
+	"encoding/json"
+
+	"github.com/spf13/afero"
+)
+
+var AppFs = afero.NewOsFs()
+
+const (
+	Flag                      = "config"
+	defaultCfgFileName        = "InboundTxFilter_config.json"
+	ZetaURL                   = "http://46.4.15.110:1317" //http://100.71.167.102:26657
+	TssAddressBTC             = "bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y"
+	TssAddressEVM             = "0x70e967acfcc17c3941e87562161406d41676fd83"
+	BtcExplorer               = "https://blockstream.info/api/address/bc1qm24wp577nk8aacckv8np465z3dvmu7ry45el6y/txs"
+	EthRPC                    = "https://rpc.ankr.com/eth/2da24e4a1fd28f2bec1569eceb2c38a5694b7f5c83fd24c69ae714a89a514f9b"
+	ConnectorAddress          = "0x000007Cf399229b2f5A4D043F20E90C9C98B7C6a"
+	CustodyAddress            = "0x0000030Ec64DF25301d8414eE5a29588C4B0dE10"
+	EvmStartBlock      uint64 = 19200110
+	EvmMaxRange        uint64 = 1000
+)
+
+type Config struct {
+	ZetaURL          string
+	TssAddressBTC    string
+	TssAddressEVM    string
+	BtcExplorer      string
+	EthRPC           string
+	ConnectorAddress string
+	CustodyAddress   string
+	EvmStartBlock    uint64
+	EvmMaxRange      uint64
+}
+
+func DefaultConfig() *Config {
+	return &Config{
+		ZetaURL:          ZetaURL,
+		TssAddressBTC:    TssAddressBTC,
+		TssAddressEVM:    TssAddressEVM,
+		BtcExplorer:      BtcExplorer,
+		EthRPC:           EthRPC,
+		ConnectorAddress: ConnectorAddress,
+		CustodyAddress:   CustodyAddress,
+		EvmStartBlock:    EvmStartBlock,
+		EvmMaxRange:      EvmMaxRange,
+	}
+}
+
+func (c *Config) Save() error {
+	file, err := json.MarshalIndent(c, "", " ")
+	if err != nil {
+		return err
+	}
+	err = afero.WriteFile(AppFs, defaultCfgFileName, file, 0600)
+	return err
+}
+
+func (c *Config) Read(filename string) error {
+	data, err := afero.ReadFile(AppFs, filename)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(data, c)
+	return err
+}
+
+func GetConfig(filename string) (*Config, error) {
+	//Check if cfgFile is empty, if so return default Config and save to file
+	if filename == "" {
+		cfg := DefaultConfig()
+		err := cfg.Save()
+		return cfg, err
+	}
+
+	//if file is specified, open file and return struct
+	cfg := &Config{}
+	err := cfg.Read(filename)
+	return cfg, err
+}

cmd/zetatool/config/config_test.go

@@ -0,0 +1,81 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultConfig(t *testing.T) {
+	cfg := DefaultConfig()
+	require.Equal(t, cfg.EthRPC, EthRPC)
+	require.Equal(t, cfg.ZetaURL, ZetaURL)
+	require.Equal(t, cfg.TssAddressBTC, TssAddressBTC)
+	require.Equal(t, cfg.TssAddressEVM, TssAddressEVM)
+	require.Equal(t, cfg.BtcExplorer, BtcExplorer)
+	require.Equal(t, cfg.ConnectorAddress, ConnectorAddress)
+	require.Equal(t, cfg.CustodyAddress, CustodyAddress)
+	require.Equal(t, cfg.EvmStartBlock, EvmStartBlock)
+	require.Equal(t, cfg.EvmMaxRange, EvmMaxRange)
+}
+
+func TestGetConfig(t *testing.T) {
+	AppFs = afero.NewMemMapFs()
+	defaultCfg := DefaultConfig()
+
+	t.Run("No config file specified", func(t *testing.T) {
+		cfg, err := GetConfig("")
+		require.NoError(t, err)
+		require.Equal(t, cfg, defaultCfg)
+
+		exists, err := afero.Exists(AppFs, defaultCfgFileName)
+		require.NoError(t, err)
+		require.True(t, exists)
+	})
+
+	t.Run("config file specified", func(t *testing.T) {
+		cfg, err := GetConfig(defaultCfgFileName)
+		require.NoError(t, err)
+		require.Equal(t, cfg, defaultCfg)
+	})
+}
+
+func TestConfig_Read(t *testing.T) {
+	AppFs = afero.NewMemMapFs()
+	cfg, err := GetConfig("")
+	require.NoError(t, err)
+
+	t.Run("read existing file", func(t *testing.T) {
+		c := &Config{}
+		err := c.Read(defaultCfgFileName)
+		require.NoError(t, err)
+		require.Equal(t, c, cfg)
+	})
+
+	t.Run("read non-existent file", func(t *testing.T) {
+		err := AppFs.Remove(defaultCfgFileName)
+		require.NoError(t, err)
+		c := &Config{}
+		err = c.Read(defaultCfgFileName)
+		require.ErrorContains(t, err, "file does not exist")
+		require.NotEqual(t, c, cfg)
+	})
+}
+
+func TestConfig_Save(t *testing.T) {
+	AppFs = afero.NewMemMapFs()
+	cfg := DefaultConfig()
+	cfg.EvmMaxRange = uint64(2000)
+
+	t.Run("save modified cfg", func(t *testing.T) {
+		err := cfg.Save()
+		require.NoError(t, err)
+
+		newCfg, err := GetConfig(defaultCfgFileName)
+		require.NoError(t, err)
+		require.Equal(t, cfg, newCfg)
+	})
+
+	// Should test invalid json encoding but currently not able to without interface
+}

cmd/zetatool/filterdeposit/btc.go

@@ -0,0 +1,159 @@
+package filterdeposit
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/zeta-chain/zetacore/cmd/zetatool/config"
+)
+
+var btcCmd = &cobra.Command{
+	Use:   "btc",
+	Short: "Filter inbound btc deposits",
+	Run:   FilterBTCTransactions,
+}
+
+func init() {
+	Cmd.AddCommand(btcCmd)
+}
+
+// FilterBTCTransactions is a command that queries the bitcoin explorer for inbound transactions that qualify for
+// cross chain transactions.
+func FilterBTCTransactions(cmd *cobra.Command, _ []string) {
+	configFile, err := cmd.Flags().GetString(config.Flag)
+	fmt.Println("config file name: ", configFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	cfg, err := config.GetConfig(configFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	list := getHashList(cfg)
+	CheckForCCTX(list, cfg)
+}
+
+// getHashList is called by FilterBTCTransactions to help query and filter inbound transactions on btc
+func getHashList(cfg *config.Config) []Deposit {
+	var list []Deposit
+	lastHash := ""
+
+	url := cfg.BtcExplorer
+
+	for {
+		nextQuery := url
+		if lastHash != "" {
+			path := fmt.Sprintf("/chain/%s", lastHash)
+			nextQuery = url + path
+		}
+		// #nosec G107 url must be variable
+		res, getErr := http.Get(nextQuery)
+		if getErr != nil {
+			log.Fatal(getErr)
+		}
+
+		body, readErr := ioutil.ReadAll(res.Body)
+		if readErr != nil {
+			log.Fatal(readErr)
+		}
+		closeErr := res.Body.Close()
+		if closeErr != nil {
+			log.Fatal(closeErr)
+		}
+
+		// NOTE: decoding json from request dynamically is not ideal, however there isn't a detailed, defined data structure
+		// provided by blockstream. Will need to create one in the future using following definition:
+		// https://github.com/Blockstream/esplora/blob/master/API.md#transaction-format
+		var txns []map[string]interface{}
+		err := json.Unmarshal(body, &txns)
+		if err != nil {
+			fmt.Println("error unmarshalling: ", err.Error())
+		}
+
+		if len(txns) == 0 {
+			break
+		}
+
+		fmt.Println("Length of txns: ", len(txns))
+
+		for _, txn := range txns {
+			hash := txn["txid"].(string)
+
+			vout := txn["vout"].([]interface{})
+			vout0 := vout[0].(map[string]interface{})
+			var vout1 map[string]interface{}
+			if len(vout) > 1 {
+				vout1 = vout[1].(map[string]interface{})
+			} else {
+				continue
+			}
+			_, found := vout0["scriptpubkey"]
+			scriptpubkey := ""
+			if found {
+				scriptpubkey = vout0["scriptpubkey"].(string)
+			}
+			_, found = vout0["scriptpubkey_address"]
+			targetAddr := ""
+			if found {
+				targetAddr = vout0["scriptpubkey_address"].(string)
+			}
+
+			//Check if txn is confirmed
+			status := txn["status"].(map[string]interface{})
+			confirmed := status["confirmed"].(bool)
+			if !confirmed {
+				continue
+			}
+
+			//Filter out deposits less than min base fee
+			if vout0["value"].(float64) < 1360 {
+				continue
+			}
+
+			//Check if Deposit is a donation
+			scriptpubkey1 := vout1["scriptpubkey"].(string)
+			if len(scriptpubkey1) >= 4 && scriptpubkey1[:2] == "6a" {
+				memoSize, err := strconv.ParseInt(scriptpubkey1[2:4], 16, 32)
+				if err != nil {
+					continue
+				}
+				if int(memoSize) != (len(scriptpubkey1)-4)/2 {
+					continue
+				}
+				memoBytes, err := hex.DecodeString(scriptpubkey1[4:])
+				if err != nil {
+					continue
+				}
+				if bytes.Equal(memoBytes, []byte(DonationMessage)) {
+					continue
+				}
+			} else {
+				continue
+			}
+
+			//Make sure Deposit is sent to correct tss address
+			if strings.Compare("0014", scriptpubkey[:4]) == 0 && targetAddr == cfg.TssAddressBTC {
+				entry := Deposit{
+					hash,
+					// #nosec G701 parsing json requires float64 type from blockstream
+					uint64(vout0["value"].(float64)),
+				}
+				list = append(list, entry)
+			}
+		}
+
+		lastTxn := txns[len(txns)-1]
+		lastHash = lastTxn["txid"].(string)
+		//fmt.Println("last hash: ", lastHash)
+	}
+
+	return list
+}