From 39beb2f23b8352dfa47f21906edc0f7198aa451f Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Mon, 5 Feb 2024 21:34:30 -0500 Subject: [PATCH 1/2] Updating slurp source --- docs/content/chifra/other.md | 2 +- docs/readmes/other-slurp.md | 2 +- src/apps/chifra/cmd/slurp.go | 2 +- src/apps/chifra/internal/slurp/README.md | 2 +- .../internal/slurp/handle_appearances.go | 2 +- .../chifra/internal/slurp/handle_count.go | 89 ++++++++++ src/apps/chifra/internal/slurp/handle_show.go | 2 +- src/apps/chifra/internal/slurp/options.go | 3 +- src/apps/chifra/internal/slurp/output.go | 2 + src/apps/chifra/pkg/rpc/etherscan.go | 14 +- src/apps/chifra/pkg/rpc/get_log.go | 4 + src/apps/chifra/pkg/rpc/get_receipt.go | 2 +- src/apps/chifra/pkg/rpc/get_traces.go | 11 ++ src/apps/chifra/pkg/rpc/query/query.go | 155 +++++++++--------- src/apps/chifra/pkg/rpc/query/query_test.go | 17 +- src/cmd-line-options.csv | 2 +- .../testRunner/testCases/tools/ethslurp.csv | 10 +- test/gold/apps/chifra/chifra_help_slurp.txt | 2 +- .../tools/ethslurp/ethslurp_bad_source.txt | 8 +- .../ethslurp/ethslurp_cache_and_decache.txt | 2 +- .../tools/ethslurp/ethslurp_caps_allowed.txt | 2 +- .../ethslurp/ethslurp_caps_disallowed_1.txt | 2 +- test/gold/tools/ethslurp/ethslurp_help.txt | 2 +- .../tools/ethslurp/ethslurp_help_long.txt | 2 +- .../ethslurp/ethslurp_invalid_address.txt | 2 +- .../tools/ethslurp/ethslurp_invalid_param.txt | 2 +- .../ethslurp/ethslurp_invalid_type_1.txt | 2 +- .../ethslurp/ethslurp_invalid_type_2.txt | 2 +- .../tools/ethslurp/ethslurp_no_params.txt | 2 +- .../tools/ethslurp/ethslurp_not_apps_key.txt | 10 +- 30 files changed, 242 insertions(+), 119 deletions(-) create mode 100644 src/apps/chifra/internal/slurp/handle_count.go diff --git a/docs/content/chifra/other.md b/docs/content/chifra/other.md index e3b2f90cb2..9486dc1656 100644 --- a/docs/content/chifra/other.md +++ b/docs/content/chifra/other.md @@ -87,7 +87,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/docs/readmes/other-slurp.md b/docs/readmes/other-slurp.md index 34e5d1b1d3..3695e54ab0 100644 --- a/docs/readmes/other-slurp.md +++ b/docs/readmes/other-slurp.md @@ -29,7 +29,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/src/apps/chifra/cmd/slurp.go b/src/apps/chifra/cmd/slurp.go index 96c41dec79..a7ffb121b5 100644 --- a/src/apps/chifra/cmd/slurp.go +++ b/src/apps/chifra/cmd/slurp.go @@ -68,7 +68,7 @@ func init() { One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ]`) slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Appearances, "appearances", "p", false, "show only the blocknumber.tx_id appearances of the exported transactions") slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Articulate, "articulate", "a", false, "articulate the retrieved data if ABIs can be found") - slurpCmd.Flags().StringVarP(&slurpPkg.GetOptions().Source, "source", "S", "", `the source of the slurped data + slurpCmd.Flags().StringVarP(&slurpPkg.GetOptions().Source, "source", "S", "etherscan", `the source of the slurped data One of [ etherscan | key ]`) slurpCmd.Flags().BoolVarP(&slurpPkg.GetOptions().Count, "count", "U", false, "for --appearances mode only, display only the count of records") slurpCmd.Flags().Uint64VarP(&slurpPkg.GetOptions().PerPage, "per_page", "P", 5000, "the number of records to request on each page") diff --git a/src/apps/chifra/internal/slurp/README.md b/src/apps/chifra/internal/slurp/README.md index 91f8fac20c..99adf31c84 100644 --- a/src/apps/chifra/internal/slurp/README.md +++ b/src/apps/chifra/internal/slurp/README.md @@ -29,7 +29,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/src/apps/chifra/internal/slurp/handle_appearances.go b/src/apps/chifra/internal/slurp/handle_appearances.go index f3dee8178a..2a035a7110 100644 --- a/src/apps/chifra/internal/slurp/handle_appearances.go +++ b/src/apps/chifra/internal/slurp/handle_appearances.go @@ -39,7 +39,7 @@ func (opts *SlurpOptions) HandleAppearances() error { }) for !done { - txs, nFetched, err := opts.Conn.GetESTransactionByAddress(opts.Globals.Chain, addr, tt, &paginator) + txs, nFetched, err := opts.Conn.SlurpTxsByAddress(opts.Globals.Chain, opts.Source, addr, tt, &paginator) done = nFetched < paginator.PerPage totalFetched += nFetched if err != nil { diff --git a/src/apps/chifra/internal/slurp/handle_count.go b/src/apps/chifra/internal/slurp/handle_count.go new file mode 100644 index 0000000000..9688a0e66b --- /dev/null +++ b/src/apps/chifra/internal/slurp/handle_count.go @@ -0,0 +1,89 @@ +package slurpPkg + +import ( + "context" + "fmt" + "time" + + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils" +) + +func (opts *SlurpOptions) HandleCount() error { + testMode := opts.Globals.TestMode + paginator := rpc.Paginator{ + Page: 1, + PerPage: int(opts.PerPage), + } + if opts.Globals.TestMode { + paginator.PerPage = 100 + } + + ctx := context.Background() + fetchData := func(modelChan chan types.Modeler[types.RawMonitor], errorChan chan error) { + totalFetched := 0 + totalFiltered := 0 + for _, addr := range opts.Addrs { + for _, tt := range opts.Types { + paginator.Page = 1 + done := false + + bar := logger.NewBar(logger.BarOptions{ + Type: logger.Expanding, + Enabled: !testMode, // && !utils.IsTerminal(), + Prefix: fmt.Sprintf("%s %s", utils.FormattedHash(false, addr), tt), + }) + + for !done { + txs, nFetched, err := opts.Conn.SlurpTxsByAddress(opts.Globals.Chain, opts.Source, addr, tt, &paginator) + done = nFetched < paginator.PerPage + totalFetched += nFetched + if err != nil { + errorChan <- err + continue + } + + for _, tx := range txs { + tx := tx + if !opts.isInRange(uint(tx.BlockNumber), errorChan) { + continue + } + bar.Tick() + totalFiltered++ + } + + // Without this Etherscan chokes + sleep := opts.Sleep + if sleep > 0 { + ms := time.Duration(sleep*1000) * time.Millisecond + logger.Progress(!opts.Globals.TestMode, fmt.Sprintf("Sleeping for %g seconds", sleep)) + time.Sleep(ms) + } + } + bar.Finish(true /* newLine */) + } + + if totalFiltered == 0 { + msg := fmt.Sprintf("zero transactions reported, %d fetched", totalFetched) + errorChan <- fmt.Errorf(msg) + } else { + s := types.SimpleMonitor{ + Address: addr, + NRecords: totalFiltered, + FileSize: int64(totalFetched), + } + if testMode { + s.FileSize = 0xdead + } + modelChan <- &s + } + } + } + + return output.StreamMany(ctx, fetchData, opts.Globals.OutputOpts()) +} + +// const maxTestingBlock = 17000000 diff --git a/src/apps/chifra/internal/slurp/handle_show.go b/src/apps/chifra/internal/slurp/handle_show.go index c5e9c937d7..be08c2d4b7 100644 --- a/src/apps/chifra/internal/slurp/handle_show.go +++ b/src/apps/chifra/internal/slurp/handle_show.go @@ -42,7 +42,7 @@ func (opts *SlurpOptions) HandleShow() error { Prefix: fmt.Sprintf("%s %s", utils.FormattedHash(false, addr), tt), }) for !done { - txs, nFetched, err := opts.Conn.GetESTransactionByAddress(opts.Globals.Chain, addr, tt, &paginator) + txs, nFetched, err := opts.Conn.SlurpTxsByAddress(opts.Globals.Chain, opts.Source, addr, tt, &paginator) done = nFetched < paginator.PerPage totalFetched += nFetched if err != nil { diff --git a/src/apps/chifra/internal/slurp/options.go b/src/apps/chifra/internal/slurp/options.go index 64a400e69e..53ebf45ef9 100644 --- a/src/apps/chifra/internal/slurp/options.go +++ b/src/apps/chifra/internal/slurp/options.go @@ -41,6 +41,7 @@ type SlurpOptions struct { } var defaultSlurpOptions = SlurpOptions{ + Source: "etherscan", PerPage: 5000, } @@ -51,7 +52,7 @@ func (opts *SlurpOptions) testLog() { logger.TestLog(len(opts.Types) > 0, "Types: ", opts.Types) logger.TestLog(opts.Appearances, "Appearances: ", opts.Appearances) logger.TestLog(opts.Articulate, "Articulate: ", opts.Articulate) - logger.TestLog(len(opts.Source) > 0, "Source: ", opts.Source) + logger.TestLog(len(opts.Source) > 0 && opts.Source != "etherscan", "Source: ", opts.Source) logger.TestLog(opts.Count, "Count: ", opts.Count) logger.TestLog(opts.PerPage != 5000, "PerPage: ", opts.PerPage) logger.TestLog(opts.Sleep != float64(.25), "Sleep: ", opts.Sleep) diff --git a/src/apps/chifra/internal/slurp/output.go b/src/apps/chifra/internal/slurp/output.go index 928f9ea01c..c33b4f1b33 100644 --- a/src/apps/chifra/internal/slurp/output.go +++ b/src/apps/chifra/internal/slurp/output.go @@ -54,6 +54,8 @@ func (opts *SlurpOptions) SlurpInternal() error { // EXISTING_CODE if opts.Globals.Decache { err = opts.HandleDecache() + } else if opts.Count { + err = opts.HandleCount() } else if opts.Appearances { err = opts.HandleAppearances() } else { diff --git a/src/apps/chifra/pkg/rpc/etherscan.go b/src/apps/chifra/pkg/rpc/etherscan.go index a260d150de..5655a1281d 100644 --- a/src/apps/chifra/pkg/rpc/etherscan.go +++ b/src/apps/chifra/pkg/rpc/etherscan.go @@ -12,6 +12,7 @@ import ( "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/debug" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc/query" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils" ) @@ -21,7 +22,18 @@ type Paginator struct { PerPage int } -func (conn *Connection) GetESTransactionByAddress(chain, addr, requestType string, paginator *Paginator) ([]types.SimpleSlurp, int, error) { +func (conn *Connection) SlurpTxsByAddress(chain, source, addr, requestType string, paginator *Paginator) ([]types.SimpleSlurp, int, error) { + switch source { + case "key": + return []types.SimpleSlurp{}, 0, nil + case "etherscan": + fallthrough + default: + return conn.getTxsByAddressEs(chain, addr, requestType, paginator) + } +} + +func (conn *Connection) getTxsByAddressEs(chain, addr, requestType string, paginator *Paginator) ([]types.SimpleSlurp, int, error) { url, err := getEtherscanUrl(chain, addr, requestType, paginator) if err != nil { return []types.SimpleSlurp{}, 0, err diff --git a/src/apps/chifra/pkg/rpc/get_log.go b/src/apps/chifra/pkg/rpc/get_log.go index 08bd1e255a..bd456aa053 100644 --- a/src/apps/chifra/pkg/rpc/get_log.go +++ b/src/apps/chifra/pkg/rpc/get_log.go @@ -80,6 +80,10 @@ func (conn *Connection) getLogsSimple(filter types.SimpleLogFilter) ([]types.Sim if rawLogs, err := query.Query[[]types.RawLog](conn.Chain, method, params); err != nil { return []types.SimpleLog{}, err + + } else if rawLogs == nil || len(*rawLogs) == 0 { + return []types.SimpleLog{}, nil + } else { curBlock := utils.NOPOS curTs := utils.NOPOSI diff --git a/src/apps/chifra/pkg/rpc/get_receipt.go b/src/apps/chifra/pkg/rpc/get_receipt.go index 7a2acef82c..085cf60f41 100644 --- a/src/apps/chifra/pkg/rpc/get_receipt.go +++ b/src/apps/chifra/pkg/rpc/get_receipt.go @@ -123,7 +123,7 @@ func (conn *Connection) getReceiptsSimple(bn base.Blknum) ([]types.SimpleReceipt if rawReceipts, err := query.Query[[]types.RawReceipt](conn.Chain, method, params); err != nil { return []types.SimpleReceipt{}, err - } else if len(*rawReceipts) == 0 { + } else if rawReceipts == nil || len(*rawReceipts) == 0 { return []types.SimpleReceipt{}, nil } else { diff --git a/src/apps/chifra/pkg/rpc/get_traces.go b/src/apps/chifra/pkg/rpc/get_traces.go index 5557236b5c..3e406dd52a 100644 --- a/src/apps/chifra/pkg/rpc/get_traces.go +++ b/src/apps/chifra/pkg/rpc/get_traces.go @@ -32,6 +32,10 @@ func (conn *Connection) GetTracesByBlockNumber(bn uint64) ([]types.SimpleTrace, if rawTraces, err := query.Query[[]types.RawTrace](conn.Chain, method, params); err != nil { return []types.SimpleTrace{}, err + + } else if rawTraces == nil || len(*rawTraces) == 0 { + return []types.SimpleTrace{}, nil + } else { curApp := types.SimpleAppearance{BlockNumber: uint32(^uint32(0))} curTs := conn.GetBlockTimestamp(bn) @@ -140,6 +144,9 @@ func (conn *Connection) GetTracesByTransactionHash(txHash string, transaction *t if rawTraces, err := query.Query[[]types.RawTrace](conn.Chain, method, params); err != nil { return ret, ethereum.NotFound + } else if rawTraces == nil || len(*rawTraces) == 0 { + return []types.SimpleTrace{}, nil + } else { curApp := types.SimpleAppearance{BlockNumber: uint32(^uint32(0))} var idx uint64 @@ -237,6 +244,10 @@ func (conn *Connection) GetTracesByFilter(filter string) ([]types.SimpleTrace, e var ret []types.SimpleTrace if rawTraces, err := query.Query[[]types.RawTrace](conn.Chain, method, params); err != nil { return ret, fmt.Errorf("trace filter %s returned an error: %w", filter, ethereum.NotFound) + + } else if rawTraces == nil || len(*rawTraces) == 0 { + return []types.SimpleTrace{}, nil + } else { curApp := types.SimpleAppearance{BlockNumber: uint32(^uint32(0))} curTs := conn.GetBlockTimestamp(utils.MustParseUint(f.FromBlock)) diff --git a/src/apps/chifra/pkg/rpc/query/query.go b/src/apps/chifra/pkg/rpc/query/query.go index 016a20e3b8..9fab463054 100644 --- a/src/apps/chifra/pkg/rpc/query/query.go +++ b/src/apps/chifra/pkg/rpc/query/query.go @@ -19,8 +19,9 @@ type Params []interface{} // Payload is used to make calls to the RPC. type Payload struct { - Method string `json:"method"` - Params `json:"params"` + Headers map[string]string `json:"headers,omitempty"` + Method string `json:"method"` + Params `json:"params"` } type rpcResponse[T any] struct { @@ -33,27 +34,6 @@ type eip1474Error struct { Message string `json:"message"` } -// Query returns a single result for given method and params. -func Query[T any](chain string, method string, params Params) (*T, error) { - var response rpcResponse[T] - - payload := Payload{ - Method: method, - Params: params, - } - - provider := config.GetChain(chain).RpcProvider - err := fromRpc(provider, &payload, &response) - if err != nil { - return nil, err - } - if response.Error != nil { - return nil, fmt.Errorf("%d: %s", response.Error.Code, response.Error.Message) - } - - return &response.Result, err -} - var rpcCounter uint32 type rpcPayload struct { @@ -63,60 +43,74 @@ type rpcPayload struct { ID int `json:"id"` } -// fromRpc Returns all traces for a given block. -func fromRpc(rpcProvider string, payload *Payload, ret any) error { +// BatchPayload is a wrapper around Payload type that allows us +// to associate a name (Key) to given request. +type BatchPayload struct { + Key string + *Payload +} + +// Query returns a single result for given method and params. +func Query[T any](chain string, method string, params Params) (*T, error) { + url := config.GetChain(chain).RpcProvider + return QueryWithHeaders[T](url, map[string]string{}, method, params) +} + +// QueryWithHeaders returns a single result for given method and params. +func QueryWithHeaders[T any](url string, headers map[string]string, method string, params Params) (*T, error) { payloadToSend := rpcPayload{ Jsonrpc: "2.0", - Method: payload.Method, - Params: payload.Params, + Method: method, + Params: params, ID: int(atomic.AddUint32(&rpcCounter, 1)), } debug.DebugCurl(rpcDebug{ + url: url, payload: payloadToSend, - url: rpcProvider}, - ) + headers: headers, + }) - plBytes, err := json.Marshal(payloadToSend) - if err != nil { - return err + if plBytes, err := json.Marshal(payloadToSend); err != nil { + return nil, err + } else { + var result rpcResponse[T] + body := bytes.NewReader(plBytes) + if response, err := http.Post(url, "application/json", body); err != nil { + return nil, err + } else { + defer response.Body.Close() + if theBytes, err := io.ReadAll(response.Body); err != nil { + return nil, err + } else { + err = json.Unmarshal(theBytes, &result) + if err != nil { + return nil, err + } + if result.Error != nil { + return nil, fmt.Errorf("%d: %s", result.Error.Code, result.Error.Message) + } + return &result.Result, nil + } + } } - - return sendRpcRequest(rpcProvider, plBytes, ret) -} - -// BatchPayload is a wrapper around Payload type that allows us -// to associate a name (Key) to given request. -type BatchPayload struct { - Key string - *Payload } // QueryBatch batches requests to the node. Returned values are stored in map, with the same keys as defined // in `batchPayload` (this way we don't have to operate on array indices) func QueryBatch[T any](chain string, batchPayload []BatchPayload) (map[string]*T, error) { - var response []rpcResponse[T] + return QueryBatchWithHeaders[T](chain, map[string]string{}, batchPayload) +} +func QueryBatchWithHeaders[T any](chain string, headers map[string]string, batchPayload []BatchPayload) (map[string]*T, error) { keys := make([]string, 0, len(batchPayload)) payloads := make([]Payload, 0, len(batchPayload)) - for _, config := range batchPayload { - keys = append(keys, config.Key) - payloads = append(payloads, *config.Payload) + for _, bpl := range batchPayload { + keys = append(keys, bpl.Key) + payloads = append(payloads, *bpl.Payload) } - provider := config.GetChain(chain).RpcProvider - err := fromRpcBatch(provider, payloads, &response) - if err != nil { - return nil, err - } - results := make(map[string]*T, len(batchPayload)) - for index, key := range keys { - results[key] = &response[index].Result - } - return results, err -} - -func fromRpcBatch(rpcProvider string, payloads []Payload, ret interface{}) error { + url := config.GetChain(chain).RpcProvider payloadToSend := make([]rpcPayload, 0, len(payloads)) for _, payload := range payloads { @@ -127,34 +121,37 @@ func fromRpcBatch(rpcProvider string, payloads []Payload, ret interface{}) error ID: int(atomic.AddUint32(&rpcCounter, 1)), } debug.DebugCurl(rpcDebug{ + url: url, payload: theLoad, - url: rpcProvider}, - ) + headers: headers, + }) payloadToSend = append(payloadToSend, theLoad) } plBytes, err := json.Marshal(payloadToSend) if err != nil { - return err - } - - return sendRpcRequest(rpcProvider, plBytes, ret) -} - -func sendRpcRequest(rpcProvider string, marshalled []byte, result any) error { - body := bytes.NewReader(marshalled) - resp, err := http.Post(rpcProvider, "application/json", body) - if err != nil { - return err + return nil, err } - defer resp.Body.Close() - theBytes, err := io.ReadAll(resp.Body) - if err != nil { - return err + var result []rpcResponse[T] + body := bytes.NewReader(plBytes) + if response, err := http.Post(url, "application/json", body); err != nil { + return nil, err + } else { + defer response.Body.Close() + if theBytes, err := io.ReadAll(response.Body); err != nil { + return nil, err + } else { + if err = json.Unmarshal(theBytes, &result); err != nil { + return nil, err + } + results := make(map[string]*T, len(batchPayload)) + for index, key := range keys { + results[key] = &result[index].Result + } + return results, err + } } - - return json.Unmarshal(theBytes, result) } func init() { @@ -170,6 +167,7 @@ func init() { type rpcDebug struct { payload rpcPayload url string + headers map[string]string } func (c rpcDebug) Url() string { @@ -182,6 +180,9 @@ func (c rpcDebug) Body() string { func (c rpcDebug) Headers() string { ret := `-H "Content-Type: application/json"` + for key, value := range c.headers { + ret += fmt.Sprintf(` -H "%s: %s"`, key, value) + } return ret } diff --git a/src/apps/chifra/pkg/rpc/query/query_test.go b/src/apps/chifra/pkg/rpc/query/query_test.go index 9093c28e68..e839b93ed7 100644 --- a/src/apps/chifra/pkg/rpc/query/query_test.go +++ b/src/apps/chifra/pkg/rpc/query/query_test.go @@ -1,3 +1,6 @@ +//go:build integration +// +build integration + // Copyright 2021 The TrueBlocks Authors. All rights reserved. // Use of this source code is governed by a license that can // be found in the LICENSE file. @@ -9,6 +12,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils" ) func TestFromRpcCounter(t *testing.T) { @@ -26,14 +31,12 @@ func TestFromRpcCounter(t *testing.T) { })) defer server.Close() - var result map[string]string for i := 0; i < 20; i++ { - // TODO: Use rpc.Query - err := fromRpc( - server.URL, - &Payload{}, - &result, - ) + payload := Payload{ + Method: "eth_blockNumber", + Params: []interface{}{}, + } + _, err := Query[string](utils.GetTestChain(), payload.Method, payload.Params) if err != nil { t.Fatal(err) } diff --git a/src/cmd-line-options.csv b/src/cmd-line-options.csv index 6b201d3ff6..4b15ad3773 100644 --- a/src/cmd-line-options.csv +++ b/src/cmd-line-options.csv @@ -315,7 +315,7 @@ num,group,tags,api_route,tool,longName,hotKey,def_val,is_required,is_customizabl 12160,tools,Other,slurp,ethslurp,types,t,,false,false,true,true,gocmd,flag,list,which types of transactions to request 12180,tools,Other,slurp,ethslurp,appearances,p,,false,false,true,true,gocmd,switch,,show only the blocknumber.tx_id appearances of the exported transactions 12185,tools,Other,slurp,ethslurp,articulate,a,,false,false,true,true,gocmd,switch,,articulate the retrieved data if ABIs can be found -12200,tools,Other,slurp,ethslurp,source,S,,false,false,true,true,gocmd,flag,enum[etherscan*|key],the source of the slurped data +12200,tools,Other,slurp,ethslurp,source,S,etherscan,false,false,true,true,gocmd,flag,enum[etherscan*|key],the source of the slurped data 12202,tools,Other,slurp,ethslurp,count,U,,false,false,true,true,gocmd,switch,,for --appearances mode only, display only the count of records 12205,tools,Other,slurp,ethslurp,per_page,P,5000,false,false,true,true,gocmd,flag,,the number of records to request on each page 12210,tools,Other,slurp,ethslurp,sleep,s,.25,false,false,true,true,gocmd,flag,,seconds to sleep between requests diff --git a/src/dev_tools/testRunner/testCases/tools/ethslurp.csv b/src/dev_tools/testRunner/testCases/tools/ethslurp.csv index c650d7a62e..03f8e91227 100644 --- a/src/dev_tools/testRunner/testCases/tools/ethslurp.csv +++ b/src/dev_tools/testRunner/testCases/tools/ethslurp.csv @@ -30,12 +30,12 @@ on ,both ,slow ,slurp ,tools/ethslurp ,appearances_csv ,n ,addrs = on ,both ,slow ,slurp ,tools/ethslurp ,bad_source ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & source = bad & fmt = json on ,both ,slow ,slurp ,tools/ethslurp ,not_apps_es ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & source = etherscan & fmt = json -on ,both ,slow ,slurp ,tools/ethslurp ,not_apps_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & source = key & fmt = json +local ,both ,slow ,slurp ,tools/ethslurp ,not_apps_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & source = key & fmt = json on ,both ,slow ,slurp ,tools/ethslurp ,apps_es ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = etherscan & fmt = json -on ,both ,slow ,slurp ,tools/ethslurp ,apps_key_type ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & types = ext & appearances & source = key & fmt = json -on ,both ,slow ,slurp ,tools/ethslurp ,apps_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = key & fmt = json -on ,both ,slow ,slurp ,tools/ethslurp ,count_not_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = etherscan & count & fmt = json -on ,both ,slow ,slurp ,tools/ethslurp ,count_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = key & count & fmt = json +local ,both ,slow ,slurp ,tools/ethslurp ,apps_key_type ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & types = ext & appearances & source = key & fmt = json +local ,both ,slow ,slurp ,tools/ethslurp ,apps_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = key & fmt = json +local ,both ,slow ,slurp ,tools/ethslurp ,count_not_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = etherscan & count & fmt = json +local ,both ,slow ,slurp ,tools/ethslurp ,count_key ,y ,addrs = 0x9519afbc60719a83c470ceec41e32d3e4b8b262e & blocks = 0-1500000 & appearances & source = key & count & fmt = json on ,both ,medi ,slurp ,tools/ethslurp ,ens_test ,y ,addrs = wolfofethereum.eth & types = int & blocks = 12524380-13675400 & fmt = json on ,both ,medi ,slurp ,tools/ethslurp ,block_range ,y ,addrs = 0x63c8c29af409bd31ec7ddeea58ff14f21e8980b0 & blocks = 4186279-4186301 & fmt = json diff --git a/test/gold/apps/chifra/chifra_help_slurp.txt b/test/gold/apps/chifra/chifra_help_slurp.txt index 32c0dc30d4..8c4e09a06a 100644 --- a/test/gold/apps/chifra/chifra_help_slurp.txt +++ b/test/gold/apps/chifra/chifra_help_slurp.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_bad_source.txt b/test/gold/tools/ethslurp/ethslurp_bad_source.txt index 380aac6d88..44e2159532 100644 --- a/test/gold/tools/ethslurp/ethslurp_bad_source.txt +++ b/test/gold/tools/ethslurp/ethslurp_bad_source.txt @@ -18,9 +18,9 @@ Flags: One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ] -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found - -S, --source string the API url to use as the source of the data - One of [ etherscan | key ] - -U, --count only available for --appearances mode when --source==key, return only the number of records + -S, --source string the source of the slurped data + One of [ etherscan | key ] (default "etherscan") + -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) -H, --ether specify value in ether @@ -35,5 +35,5 @@ Notes: - An address must be either an ENS name or start with '0x' and be forty-two characters long. - Portions of this software are Powered by Etherscan.io APIs. - The withdrawals option is only available on certain chains. It is ignored otherwise. - - If the value of --source is key, --types is ignored and only appearances are returned. + - If the value of --source is key, --types is ignored and only appearances or counts are returned. diff --git a/test/gold/tools/ethslurp/ethslurp_cache_and_decache.txt b/test/gold/tools/ethslurp/ethslurp_cache_and_decache.txt index f5e05e5d0d..af95026bd0 100644 --- a/test/gold/tools/ethslurp/ethslurp_cache_and_decache.txt +++ b/test/gold/tools/ethslurp/ethslurp_cache_and_decache.txt @@ -21,7 +21,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_caps_allowed.txt b/test/gold/tools/ethslurp/ethslurp_caps_allowed.txt index 6cea1c6ceb..f25d633f29 100644 --- a/test/gold/tools/ethslurp/ethslurp_caps_allowed.txt +++ b/test/gold/tools/ethslurp/ethslurp_caps_allowed.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_caps_disallowed_1.txt b/test/gold/tools/ethslurp/ethslurp_caps_disallowed_1.txt index 83192b3e83..87a99d5ad1 100644 --- a/test/gold/tools/ethslurp/ethslurp_caps_disallowed_1.txt +++ b/test/gold/tools/ethslurp/ethslurp_caps_disallowed_1.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_help.txt b/test/gold/tools/ethslurp/ethslurp_help.txt index e052a238c0..d1fd87cc78 100644 --- a/test/gold/tools/ethslurp/ethslurp_help.txt +++ b/test/gold/tools/ethslurp/ethslurp_help.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_help_long.txt b/test/gold/tools/ethslurp/ethslurp_help_long.txt index df1ad000f3..c559dde385 100644 --- a/test/gold/tools/ethslurp/ethslurp_help_long.txt +++ b/test/gold/tools/ethslurp/ethslurp_help_long.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_invalid_address.txt b/test/gold/tools/ethslurp/ethslurp_invalid_address.txt index 5854a193e1..6009d07732 100644 --- a/test/gold/tools/ethslurp/ethslurp_invalid_address.txt +++ b/test/gold/tools/ethslurp/ethslurp_invalid_address.txt @@ -17,7 +17,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_invalid_param.txt b/test/gold/tools/ethslurp/ethslurp_invalid_param.txt index b07a087248..e7585cfd47 100644 --- a/test/gold/tools/ethslurp/ethslurp_invalid_param.txt +++ b/test/gold/tools/ethslurp/ethslurp_invalid_param.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_invalid_type_1.txt b/test/gold/tools/ethslurp/ethslurp_invalid_type_1.txt index afcb627805..a9524ccb2e 100644 --- a/test/gold/tools/ethslurp/ethslurp_invalid_type_1.txt +++ b/test/gold/tools/ethslurp/ethslurp_invalid_type_1.txt @@ -17,7 +17,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_invalid_type_2.txt b/test/gold/tools/ethslurp/ethslurp_invalid_type_2.txt index 98b45378e8..f62615bc44 100644 --- a/test/gold/tools/ethslurp/ethslurp_invalid_type_2.txt +++ b/test/gold/tools/ethslurp/ethslurp_invalid_type_2.txt @@ -15,7 +15,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_no_params.txt b/test/gold/tools/ethslurp/ethslurp_no_params.txt index 4296bc41a2..70a33a39cd 100644 --- a/test/gold/tools/ethslurp/ethslurp_no_params.txt +++ b/test/gold/tools/ethslurp/ethslurp_no_params.txt @@ -16,7 +16,7 @@ Flags: -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found -S, --source string the source of the slurped data - One of [ etherscan | key ] + One of [ etherscan | key ] (default "etherscan") -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) diff --git a/test/gold/tools/ethslurp/ethslurp_not_apps_key.txt b/test/gold/tools/ethslurp/ethslurp_not_apps_key.txt index 160792bada..069d3902cd 100644 --- a/test/gold/tools/ethslurp/ethslurp_not_apps_key.txt +++ b/test/gold/tools/ethslurp/ethslurp_not_apps_key.txt @@ -5,7 +5,7 @@ TEST[DATE|TIME] Types: [ext] TEST[DATE|TIME] Source: key TEST[DATE|TIME] Caps: cache,decache,raw,ether TEST[DATE|TIME] Format: json -Error: The --source=key option is only available with --appearances. +Error: The --source=key option is only available with --appearances or --count. Usage: chifra slurp [flags]
[address...] [block...] @@ -18,9 +18,9 @@ Flags: One or more of [ ext | int | token | nfts | 1155 | miner | uncles | withdrawals | all ] -p, --appearances show only the blocknumber.tx_id appearances of the exported transactions -a, --articulate articulate the retrieved data if ABIs can be found - -S, --source string the API url to use as the source of the data - One of [ etherscan | key ] - -U, --count only available for --appearances mode when --source==key, return only the number of records + -S, --source string the source of the slurped data + One of [ etherscan | key ] (default "etherscan") + -U, --count for --appearances mode only, display only the count of records -P, --per_page uint the number of records to request on each page (default 5000) -s, --sleep float seconds to sleep between requests (default 0.25) -H, --ether specify value in ether @@ -35,5 +35,5 @@ Notes: - An address must be either an ENS name or start with '0x' and be forty-two characters long. - Portions of this software are Powered by Etherscan.io APIs. - The withdrawals option is only available on certain chains. It is ignored otherwise. - - If the value of --source is key, --types is ignored and only appearances are returned. + - If the value of --source is key, --types is ignored and only appearances or counts are returned. From ab733c3c39828ccdd5e1d3d8ad23e30333a689c8 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Mon, 5 Feb 2024 21:45:01 -0500 Subject: [PATCH 2/2] Catching up --- src/apps/chifra/pkg/rpc/etherscan.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/apps/chifra/pkg/rpc/etherscan.go b/src/apps/chifra/pkg/rpc/etherscan.go index 5655a1281d..e92af5f1d8 100644 --- a/src/apps/chifra/pkg/rpc/etherscan.go +++ b/src/apps/chifra/pkg/rpc/etherscan.go @@ -12,7 +12,6 @@ import ( "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/debug" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger" - "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc/query" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils" )