Merge pull request #3495 from TrueBlocks/feature/slurp-source

Updating slurp source

docs/content/chifra/other.md

@@ -87,7 +87,7 @@ Flags:
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
   -a, --articulate      articulate the retrieved data if ABIs can be found
   -S, --source string   the source of the slurped data
-                        One of [ etherscan | key ]
+                        One of [ etherscan | key ] (default "etherscan")
   -U, --count           for --appearances mode only, display only the count of records
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)

docs/readmes/other-slurp.md

@@ -29,7 +29,7 @@ Flags:
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
   -a, --articulate      articulate the retrieved data if ABIs can be found
   -S, --source string   the source of the slurped data
-                        One of [ etherscan | key ]
+                        One of [ etherscan | key ] (default "etherscan")
   -U, --count           for --appearances mode only, display only the count of records
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)

src/apps/chifra/cmd/slurp.go

@@ -68,7 +68,7 @@ func init() {
 One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]`)
 	slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Appearances, "appearances", "p", false, "show only the blocknumber.tx_id appearances of the exported transactions")
 	slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Articulate, "articulate", "a", false, "articulate the retrieved data if ABIs can be found")
-	slurpCmd.Flags().StringVarP(&slurpPkg.GetOptions().Source, "source", "S", "", `the source of the slurped data
+	slurpCmd.Flags().StringVarP(&slurpPkg.GetOptions().Source, "source", "S", "etherscan", `the source of the slurped data
 One of [ etherscan | key ]`)
 	slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Count, "count", "U", false, "for --appearances mode only, display only the count of records")
 	slurpCmd.Flags().Uint64VarP(&slurpPkg.GetOptions().PerPage, "per_page", "P", 5000, "the number of records to request on each page")

src/apps/chifra/internal/slurp/README.md

@@ -29,7 +29,7 @@ Flags:
   -p, --appearances     show only the blocknumber.tx_id appearances of the exported transactions
   -a, --articulate      articulate the retrieved data if ABIs can be found
   -S, --source string   the source of the slurped data
-                        One of [ etherscan | key ]
+                        One of [ etherscan | key ] (default "etherscan")
   -U, --count           for --appearances mode only, display only the count of records
   -P, --per_page uint   the number of records to request on each page (default 5000)
   -s, --sleep float     seconds to sleep between requests (default 0.25)

src/apps/chifra/internal/slurp/handle_appearances.go

@@ -39,7 +39,7 @@ func (opts *SlurpOptions) HandleAppearances() error {
 				})
 
 				for !done {
-					txs, nFetched, err := opts.Conn.GetESTransactionByAddress(opts.Globals.Chain, addr, tt, &paginator)
+					txs, nFetched, err := opts.Conn.SlurpTxsByAddress(opts.Globals.Chain, opts.Source, addr, tt, &paginator)
 					done = nFetched < paginator.PerPage
 					totalFetched += nFetched
 					if err != nil {

src/apps/chifra/internal/slurp/handle_count.go

@@ -0,0 +1,89 @@
+package slurpPkg
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
+)
+
+func (opts *SlurpOptions) HandleCount() error {
+	testMode := opts.Globals.TestMode
+	paginator := rpc.Paginator{
+		Page:    1,
+		PerPage: int(opts.PerPage),
+	}
+	if opts.Globals.TestMode {
+		paginator.PerPage = 100
+	}
+
+	ctx := context.Background()
+	fetchData := func(modelChan chan types.Modeler[types.RawMonitor], errorChan chan error) {
+		totalFetched := 0
+		totalFiltered := 0
+		for _, addr := range opts.Addrs {
+			for _, tt := range opts.Types {
+				paginator.Page = 1
+				done := false
+
+				bar := logger.NewBar(logger.BarOptions{
+					Type:    logger.Expanding,
+					Enabled: !testMode, // && !utils.IsTerminal(),
+					Prefix:  fmt.Sprintf("%s %s", utils.FormattedHash(false, addr), tt),
+				})
+
+				for !done {
+					txs, nFetched, err := opts.Conn.SlurpTxsByAddress(opts.Globals.Chain, opts.Source, addr, tt, &paginator)
+					done = nFetched < paginator.PerPage
+					totalFetched += nFetched
+					if err != nil {
+						errorChan <- err
+						continue
+					}
+
+					for _, tx := range txs {
+						tx := tx
+						if !opts.isInRange(uint(tx.BlockNumber), errorChan) {
+							continue
+						}
+						bar.Tick()
+						totalFiltered++
+					}
+
+					// Without this Etherscan chokes
+					sleep := opts.Sleep
+					if sleep > 0 {
+						ms := time.Duration(sleep*1000) * time.Millisecond
+						logger.Progress(!opts.Globals.TestMode, fmt.Sprintf("Sleeping for %g seconds", sleep))
+						time.Sleep(ms)
+					}
+				}
+				bar.Finish(true /* newLine */)
+			}
+
+			if totalFiltered == 0 {
+				msg := fmt.Sprintf("zero transactions reported, %d fetched", totalFetched)
+				errorChan <- fmt.Errorf(msg)
+			} else {
+				s := types.SimpleMonitor{
+					Address:  addr,
+					NRecords: totalFiltered,
+					FileSize: int64(totalFetched),
+				}
+				if testMode {
+					s.FileSize = 0xdead
+				}
+				modelChan <- &s
+			}
+		}
+	}
+
+	return output.StreamMany(ctx, fetchData, opts.Globals.OutputOpts())
+}
+
+// const maxTestingBlock = 17000000

src/apps/chifra/internal/slurp/handle_show.go

@@ -42,7 +42,7 @@ func (opts *SlurpOptions) HandleShow() error {
 					Prefix:  fmt.Sprintf("%s %s", utils.FormattedHash(false, addr), tt),
 				})
 				for !done {
-					txs, nFetched, err := opts.Conn.GetESTransactionByAddress(opts.Globals.Chain, addr, tt, &paginator)
+					txs, nFetched, err := opts.Conn.SlurpTxsByAddress(opts.Globals.Chain, opts.Source, addr, tt, &paginator)
 					done = nFetched < paginator.PerPage
 					totalFetched += nFetched
 					if err != nil {

src/apps/chifra/internal/slurp/options.go

@@ -41,6 +41,7 @@ type SlurpOptions struct {
 }
 
 var defaultSlurpOptions = SlurpOptions{
+	Source:  "etherscan",
 	PerPage: 5000,
 }
 
@@ -51,7 +52,7 @@ func (opts *SlurpOptions) testLog() {
 	logger.TestLog(len(opts.Types) > 0, "Types: ", opts.Types)
 	logger.TestLog(opts.Appearances, "Appearances: ", opts.Appearances)
 	logger.TestLog(opts.Articulate, "Articulate: ", opts.Articulate)
-	logger.TestLog(len(opts.Source) > 0, "Source: ", opts.Source)
+	logger.TestLog(len(opts.Source) > 0 && opts.Source != "etherscan", "Source: ", opts.Source)
 	logger.TestLog(opts.Count, "Count: ", opts.Count)
 	logger.TestLog(opts.PerPage != 5000, "PerPage: ", opts.PerPage)
 	logger.TestLog(opts.Sleep != float64(.25), "Sleep: ", opts.Sleep)

src/apps/chifra/internal/slurp/output.go

@@ -54,6 +54,8 @@ func (opts *SlurpOptions) SlurpInternal() error {
 	// EXISTING_CODE
 	if opts.Globals.Decache {
 		err = opts.HandleDecache()
+	} else if opts.Count {
+		err = opts.HandleCount()
 	} else if opts.Appearances {
 		err = opts.HandleAppearances()
 	} else {

src/apps/chifra/pkg/rpc/etherscan.go

@@ -21,7 +21,18 @@ type Paginator struct {
 	PerPage int
 }
 
-func (conn *Connection) GetESTransactionByAddress(chain, addr, requestType string, paginator *Paginator) ([]types.SimpleSlurp, int, error) {
+func (conn *Connection) SlurpTxsByAddress(chain, source, addr, requestType string, paginator *Paginator) ([]types.SimpleSlurp, int, error) {
+	switch source {
+	case "key":
+		return []types.SimpleSlurp{}, 0, nil
+	case "etherscan":
+		fallthrough
+	default:
+		return conn.getTxsByAddressEs(chain, addr, requestType, paginator)
+	}
+}
+
+func (conn *Connection) getTxsByAddressEs(chain, addr, requestType string, paginator *Paginator) ([]types.SimpleSlurp, int, error) {
 	url, err := getEtherscanUrl(chain, addr, requestType, paginator)
 	if err != nil {
 		return []types.SimpleSlurp{}, 0, err

src/apps/chifra/pkg/rpc/get_log.go

@@ -80,6 +80,10 @@ func (conn *Connection) getLogsSimple(filter types.SimpleLogFilter) ([]types.Sim
 
 	if rawLogs, err := query.Query[[]types.RawLog](conn.Chain, method, params); err != nil {
 		return []types.SimpleLog{}, err
+
+	} else if rawLogs == nil || len(*rawLogs) == 0 {
+		return []types.SimpleLog{}, nil
+
 	} else {
 		curBlock := utils.NOPOS
 		curTs := utils.NOPOSI

src/apps/chifra/pkg/rpc/get_receipt.go

@@ -123,7 +123,7 @@ func (conn *Connection) getReceiptsSimple(bn base.Blknum) ([]types.SimpleReceipt
 	if rawReceipts, err := query.Query[[]types.RawReceipt](conn.Chain, method, params); err != nil {
 		return []types.SimpleReceipt{}, err
 
-	} else if len(*rawReceipts) == 0 {
+	} else if rawReceipts == nil || len(*rawReceipts) == 0 {
 		return []types.SimpleReceipt{}, nil
 
 	} else {

src/apps/chifra/pkg/rpc/get_traces.go

@@ -32,6 +32,10 @@ func (conn *Connection) GetTracesByBlockNumber(bn uint64) ([]types.SimpleTrace,
 
 	if rawTraces, err := query.Query[[]types.RawTrace](conn.Chain, method, params); err != nil {
 		return []types.SimpleTrace{}, err
+
+	} else if rawTraces == nil || len(*rawTraces) == 0 {
+		return []types.SimpleTrace{}, nil
+
 	} else {
 		curApp := types.SimpleAppearance{BlockNumber: uint32(^uint32(0))}
 		curTs := conn.GetBlockTimestamp(bn)
@@ -140,6 +144,9 @@ func (conn *Connection) GetTracesByTransactionHash(txHash string, transaction *t
 	if rawTraces, err := query.Query[[]types.RawTrace](conn.Chain, method, params); err != nil {
 		return ret, ethereum.NotFound
 
+	} else if rawTraces == nil || len(*rawTraces) == 0 {
+		return []types.SimpleTrace{}, nil
+
 	} else {
 		curApp := types.SimpleAppearance{BlockNumber: uint32(^uint32(0))}
 		var idx uint64
@@ -237,6 +244,10 @@ func (conn *Connection) GetTracesByFilter(filter string) ([]types.SimpleTrace, e
 	var ret []types.SimpleTrace
 	if rawTraces, err := query.Query[[]types.RawTrace](conn.Chain, method, params); err != nil {
 		return ret, fmt.Errorf("trace filter %s returned an error: %w", filter, ethereum.NotFound)
+
+	} else if rawTraces == nil || len(*rawTraces) == 0 {
+		return []types.SimpleTrace{}, nil
+
 	} else {
 		curApp := types.SimpleAppearance{BlockNumber: uint32(^uint32(0))}
 		curTs := conn.GetBlockTimestamp(utils.MustParseUint(f.FromBlock))