Final 2 (#3718)

* Updating version

* Updating version

* Updating version

* Cleaning up OpenApi.yaml and examples

* Adding a few examples to OpenAPI.yaml file

* Adding a few examples to OpenAPI.yaml file

* Trying to capture control of the progress bar

* Trying to capture control of the progress bar

* Adds logging to traces

* Removes --load from chifra blocks as unused

* Updates

* Removes --seed from chifra tranactions as unused

* Fixes tests

* Fixes off by one error

* Fixes off by one error

* Messaging while scraping

* Fixes issue #3721

* Fixes issue

* Fixes tests

.markdownlint.yaml

@@ -2,3 +2,4 @@ MD013: false
 MD033: false
 MD036: false
 MD041: false
+MD024: false

.prettierrc

@@ -0,0 +1,3 @@
+{
+  "trailingComma": "none"
+}

VERSION

@@ -1 +1 @@
-2.5.11
+2.9.0

docs/content/chifra/globals.md

@@ -78,7 +78,7 @@ Where:
 
 ### Group 4
 
-**As of version 2.6.0, the `--r aw` option has been removed in  its entirity.
+**As of version 2.6.0, the `--raw` option has been removed in  its entirity.
 Prior to that version, the following tools had this option which would pass
 the data received directly from the node without modification.**
 

docs/content/data-model/admin.md

@@ -312,17 +312,18 @@ Chains consist of the following fields:
 
 This documentation mentions the following basic data types.
 
-| Type     | Description                         | Notes       |
-| -------- | ----------------------------------- | ----------- |
-| address  | an '0x'-prefixed 20-byte hex string | lowercase   |
-| blkrange | a pair of nine-digit block numbers  | zero padded |
-| bool     | either `true`, `false`, `1`, or `0` |             |
-| datetime | a JSON formatted date               | as a string |
-| float64  | a double precision float            | 64 bits     |
-| hash     | an '0x'-prefixed 32-byte hex string | lowercase   |
-| int64    | a 64-bit signed integer             |             |
-| ipfshash | a multi-hash produced by IPFS       | mixed-case  |
-| string   | a normal character string           |             |
-| uint64   | a 64-bit unsigned integer           |             |
+| Type     | Description                              | Notes       |
+| -------- | ---------------------------------------- | ----------- |
+| address  | an '0x'-prefixed 20-byte hex string      | lowercase   |
+| any      | any cache item found in the binary cache |             |
+| blkrange | a pair of nine-digit block numbers       | zero padded |
+| bool     | either `true`, `false`, `1`, or `0`      |             |
+| datetime | a JSON formatted date                    | as a string |
+| float64  | a double precision float                 | 64 bits     |
+| hash     | an '0x'-prefixed 32-byte hex string      | lowercase   |
+| int64    | a 64-bit signed integer                  |             |
+| ipfshash | a multi-hash produced by IPFS            | mixed-case  |
+| string   | a normal character string                |             |
+| uint64   | a 64-bit unsigned integer                |             |
 
 *Copyright (c) 2024, TrueBlocks, LLC. All rights reserved. Generated with goMaker.*

docs/content/data-model/chainstate.md

@@ -58,22 +58,23 @@ The following commands produce and manage Tokens:
 
 Tokens consist of the following fields:
 
-| Field            | Description                                                                | Type      |
-| ---------------- | -------------------------------------------------------------------------- | --------- |
-| blockNumber      | the block at which the report is made                                      | blknum    |
-| transactionIndex | the transaction index (if applicable) at which the report is made          | txnum     |
-| timestamp        | the timestamp of the block                                                 | timestamp |
-| date             | the timestamp as a date (calculated)                                       | datetime  |
-| totalSupply      | the total supply of the token contract                                     | int256    |
-| address          | the address of the token contract                                          | address   |
-| holder           | the holder address for which we are reporting                              | address   |
-| priorBalance     | the holder's asset balance at its prior appearance                         | int256    |
-| balance          | the holder's asset balance at the given block height                       | int256    |
-| diff             | the difference, if any, between the prior and current balance (calculated) | int256    |
-| name             | the name of the token contract, if available                               | string    |
-| symbol           | the symbol of the token contract                                           | string    |
-| decimals         | the number of decimals for the token contract                              | uint64    |
-| type             | the type of token (ERC20 or ERC721) or none                                | TokenType |
+| Field            | Description                                                                  | Type      |
+| ---------------- | ---------------------------------------------------------------------------- | --------- |
+| blockNumber      | the block at which the report is made                                        | blknum    |
+| transactionIndex | the transaction index (if applicable) at which the report is made            | txnum     |
+| timestamp        | the timestamp of the block                                                   | timestamp |
+| date             | the timestamp as a date (calculated)                                         | datetime  |
+| totalSupply      | the total supply of the token contract                                       | int256    |
+| address          | the address of the token contract                                            | address   |
+| holder           | the holder address for which we are reporting                                | address   |
+| priorBalance     | the holder's asset balance at its prior appearance                           | int256    |
+| balance          | the holder's asset balance at the given block height                         | int256    |
+| balanceDec       | the holder's asset balance (in Ether) at the given block height (calculated) | float64   |
+| diff             | the difference, if any, between the prior and current balance (calculated)   | int256    |
+| name             | the name of the token contract, if available                                 | string    |
+| symbol           | the symbol of the token contract                                             | string    |
+| decimals         | the number of decimals for the token contract                                | uint64    |
+| type             | the type of token (ERC20 or ERC721) or none                                  | TokenType |
 
 ## Result
 
@@ -108,6 +109,7 @@ This documentation mentions the following basic data types.
 | blknum    | an alias for a uint64                  |                |
 | datetime  | a JSON formatted date                  | as a string    |
 | ether     | a big number float                     | as a string    |
+| float64   | a double precision float               | 64 bits        |
 | int256    | a signed big number                    | as a string    |
 | string    | a normal character string              |                |
 | timestamp | a 64-bit unsigned integer              | Unix timestamp |

sdk/export.go

@@ -36,7 +36,6 @@ type ExportOptions struct {
 	Flow        ExportFlow  `json:"flow,omitempty"`
 	Factory     bool        `json:"factory,omitempty"`
 	Unripe      bool        `json:"unripe,omitempty"`
-	Load        string      `json:"load,omitempty"`
 	Reversed    bool        `json:"reversed,omitempty"`
 	NoZero      bool        `json:"noZero,omitempty"`
 	FirstBlock  base.Blknum `json:"firstBlock,omitempty"`

sdk/export_internal.go

@@ -48,7 +48,6 @@ type exportOptionsInternal struct {
 	Flow        ExportFlow  `json:"flow,omitempty"`
 	Factory     bool        `json:"factory,omitempty"`
 	Unripe      bool        `json:"unripe,omitempty"`
-	Load        string      `json:"load,omitempty"`
 	Reversed    bool        `json:"reversed,omitempty"`
 	NoZero      bool        `json:"noZero,omitempty"`
 	FirstBlock  base.Blknum `json:"firstBlock,omitempty"`
@@ -165,7 +164,6 @@ func (opts *ExportOptions) toInternal() *exportOptionsInternal {
 		Flow:        opts.Flow,
 		Factory:     opts.Factory,
 		Unripe:      opts.Unripe,
-		Load:        opts.Load,
 		Reversed:    opts.Reversed,
 		NoZero:      opts.NoZero,
 		FirstBlock:  opts.FirstBlock,

sdk/transactions.go

@@ -14,7 +14,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
 	// EXISTING_CODE
 )
@@ -62,13 +61,6 @@ func (opts *TransactionsOptions) TransactionsLogs() ([]types.Log, *types.MetaDat
 	return queryTransactions[types.Log](in)
 }
 
-// TransactionsSeed implements the chifra transactions --seed command.
-func (opts *TransactionsOptions) TransactionsSeed() ([]base.Address, *types.MetaData, error) {
-	in := opts.toInternal()
-	in.Seed = true
-	return queryTransactions[base.Address](in)
-}
-
 type TransactionsFlow int
 
 const (

sdk/transactions_internal.go

@@ -32,7 +32,6 @@ type transactionsOptionsInternal struct {
 	Emitter        []string         `json:"emitter,omitempty"`
 	Topic          []string         `json:"topic,omitempty"`
 	CacheTraces    bool             `json:"cacheTraces,omitempty"`
-	Seed           bool             `json:"seed,omitempty"`
 	Globals
 }
 

src/apps/chifra/cmd/export.go

@@ -91,14 +91,10 @@ func init() {
 One of [ in | out | zero ]`)
 	exportCmd.Flags().BoolVarP(&exportPkg.GetOptions().Factory, "factory", "y", false, `for --traces only, report addresses created by (or self-destructed by) the given address(es)`)
 	exportCmd.Flags().BoolVarP(&exportPkg.GetOptions().Unripe, "unripe", "u", false, `export transactions labeled unripe (i.e. less than 28 blocks old)`)
-	exportCmd.Flags().StringVarP(&exportPkg.GetOptions().Load, "load", "O", "", `a comma separated list of dynamic traversers to load (hidden)`)
 	exportCmd.Flags().BoolVarP(&exportPkg.GetOptions().Reversed, "reversed", "E", false, `produce results in reverse chronological order`)
 	exportCmd.Flags().BoolVarP(&exportPkg.GetOptions().NoZero, "no_zero", "z", false, `for the --count option only, suppress the display of zero appearance accounts`)
 	exportCmd.Flags().Uint64VarP((*uint64)(&exportPkg.GetOptions().FirstBlock), "first_block", "F", 0, `first block to process (inclusive)`)
 	exportCmd.Flags().Uint64VarP((*uint64)(&exportPkg.GetOptions().LastBlock), "last_block", "L", 0, `last block to process (inclusive)`)
-	if os.Getenv("TEST_MODE") != "true" {
-		_ = exportCmd.Flags().MarkHidden("load")
-	}
 	globals.InitGlobals("export", exportCmd, &exportPkg.GetOptions().Globals, capabilities)
 
 	exportCmd.SetUsageTemplate(UsageWithNotes(notesExport))

src/apps/chifra/cmd/transactions.go

@@ -68,10 +68,8 @@ One of [ from | to ]`)
 	transactionsCmd.Flags().StringSliceVarP(&transactionsPkg.GetOptions().Emitter, "emitter", "m", nil, `for the --logs option only, filter logs to show only those logs emitted by the given address(es)`)
 	transactionsCmd.Flags().StringSliceVarP(&transactionsPkg.GetOptions().Topic, "topic", "B", nil, `for the --logs option only, filter logs to show only those with this topic(s)`)
 	transactionsCmd.Flags().BoolVarP(&transactionsPkg.GetOptions().CacheTraces, "cache_traces", "", false, `force the transaction's traces into the cache (hidden)`)
-	transactionsCmd.Flags().BoolVarP(&transactionsPkg.GetOptions().Seed, "seed", "S", false, `find the source of the funds sent to the receiver (hidden)`)
 	if os.Getenv("TEST_MODE") != "true" {
 		_ = transactionsCmd.Flags().MarkHidden("cache_traces")
-		_ = transactionsCmd.Flags().MarkHidden("seed")
 	}
 	globals.InitGlobals("transactions", transactionsCmd, &transactionsPkg.GetOptions().Globals, capabilities)
 

src/apps/chifra/internal/abis/handle_encode.go

@@ -20,7 +20,7 @@ func (opts *AbisOptions) HandleEncode() error {
 	fetchData := func(modelChan chan types.Modeler, errorChan chan error) {
 		funcs := abi.ExtractSigs(opts.Encode)
 		if len(funcs) == 0 {
-			errorChan <- fmt.Errorf("not a valid string in HandleEncode")
+			errorChan <- fmt.Errorf("not a valid function signature in HandleEncode: %s", opts.Encode)
 			return
 		}
 		for _, f := range funcs {

src/apps/chifra/internal/blocks/handle_decache.go

@@ -14,16 +14,15 @@ import (
 )
 
 func (opts *BlocksOptions) HandleDecache() error {
-	silent := !opts.Globals.ShowProgress()
-
 	itemsToRemove, err := decache.LocationsFromBlockIds(opts.Conn, opts.BlockIds, opts.Logs, opts.Traces)
 	if err != nil {
 		return err
 	}
 
 	ctx := context.Background()
 	fetchData := func(modelChan chan types.Modeler, errorChan chan error) {
-		if msg, err := decache.Decache(opts.Conn, itemsToRemove, silent, opts.getCacheType()); err != nil {
+		showProgress := opts.Globals.ShowProgress()
+		if msg, err := decache.Decache(opts.Conn, itemsToRemove, showProgress, opts.getCacheType()); err != nil {
 			errorChan <- err
 		} else {
 			s := types.Message{

src/apps/chifra/internal/blocks/handle_hashes.go

@@ -39,8 +39,9 @@ func (opts *BlocksOptions) HandleHashes() error {
 			cancel()
 
 		} else {
+			showProgress := opts.Globals.ShowProgress()
 			bar := logger.NewBar(logger.BarOptions{
-				Enabled: !testMode && !logger.IsTerminal(),
+				Enabled: showProgress,
 				Total:   int64(cnt),
 			})
 

src/apps/chifra/internal/blocks/handle_logs.go

@@ -44,8 +44,9 @@ func (opts *BlocksOptions) HandleLogs() error {
 			cancel()
 
 		} else {
+			showProgress := opts.Globals.ShowProgress()
 			bar := logger.NewBar(logger.BarOptions{
-				Enabled: !testMode && !logger.IsTerminal(),
+				Enabled: showProgress,
 				Total:   int64(cnt),
 			})
 

src/apps/chifra/internal/blocks/handle_show.go

@@ -42,8 +42,9 @@ func (opts *BlocksOptions) HandleShow() error {
 			cancel()
 
 		} else {
+			showProgress := opts.Globals.ShowProgress()
 			bar := logger.NewBar(logger.BarOptions{
-				Enabled: !testMode && !logger.IsTerminal(),
+				Enabled: showProgress,
 				Total:   int64(cnt),
 			})
 

src/apps/chifra/internal/blocks/handle_traces.go

@@ -6,54 +6,119 @@ package blocksPkg
 
 import (
 	"context"
-	"errors"
+	"fmt"
+	"sort"
 
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/articulate"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/identifiers"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
-	"github.com/ethereum/go-ethereum"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
 )
 
 func (opts *BlocksOptions) HandleTraces() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
+	nErrors := 0
+
+	abiCache := articulate.NewAbiCache(opts.Conn, opts.Articulate)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler, errorChan chan error) {
-		for _, br := range opts.BlockIds {
-			blockNums, err := br.ResolveBlocks(chain)
-			if err != nil {
-				errorChan <- err
-				if errors.Is(err, ethereum.NotFound) {
-					continue
+		apps, _, err := identifiers.IdsToApps(chain, opts.BlockIds)
+		if err != nil {
+			errorChan <- err
+			cancel()
+		}
+
+		if sliceOfMaps, cnt, err := types.AsSliceOfMaps[types.Transaction](apps, false); err != nil {
+			errorChan <- err
+			cancel()
+
+		} else if cnt == 0 {
+			errorChan <- fmt.Errorf("no blocks found for the query")
+			cancel()
+
+		} else {
+			showProgress := opts.Globals.ShowProgress()
+			bar := logger.NewBar(logger.BarOptions{
+				Enabled: showProgress,
+				Total:   int64(cnt),
+			})
+
+			for _, thisMap := range sliceOfMaps {
+				for app := range thisMap {
+					thisMap[app] = new(types.Transaction)
 				}
-				cancel()
-				return
-			}
 
-			for _, bn := range blockNums {
-				var traces []types.Trace
-				traces, err = opts.Conn.GetTracesByBlockNumber(bn)
-				if err != nil {
-					errorChan <- err
-					if errors.Is(err, ethereum.NotFound) {
-						continue
+				iterFunc := func(app types.Appearance, value *types.Transaction) error {
+					bn := base.Blknum(app.BlockNumber)
+					if traces, err := opts.Conn.GetTracesByBlockNumber(bn); err != nil {
+						errMutex.Lock()
+						defer errMutex.Unlock()
+						delete(thisMap, app)
+						return fmt.Errorf("block at %d returned an error: %w", bn, err)
+
+					} else if len(traces) == 0 {
+						errMutex.Lock()
+						defer errMutex.Unlock()
+						delete(thisMap, app)
+						return fmt.Errorf("block at %d has no traces", bn)
+
+					} else {
+						l := make([]types.Trace, 0, len(traces))
+						for index := range traces {
+							if opts.Articulate {
+								if err = abiCache.ArticulateTrace(&traces[index]); err != nil {
+									errorChan <- err // continue even with an error
+								}
+							}
+							l = append(l, traces[index])
+						}
+						value.Traces = append(value.Traces, l...)
+						bar.Tick()
+						return nil
+					}
+				}
+
+				iterErrorChan := make(chan error)
+				iterCtx, iterCancel := context.WithCancel(context.Background())
+				defer iterCancel()
+				go utils.IterateOverMap(iterCtx, iterErrorChan, thisMap, iterFunc)
+				for err := range iterErrorChan {
+					if !testMode || nErrors == 0 {
+						errorChan <- err
+						nErrors++
 					}
-					cancel()
-					return
 				}
 
-				for _, trace := range traces {
-					modelChan <- &trace
+				items := make([]types.Trace, 0, len(thisMap))
+				for _, tx := range thisMap {
+					items = append(items, tx.Traces...)
+				}
+				sort.Slice(items, func(i, j int) bool {
+					if items[i].BlockNumber == items[j].BlockNumber {
+						if items[i].TransactionIndex == items[j].TransactionIndex {
+							return items[i].TraceIndex < items[j].TraceIndex
+						}
+						return items[i].TransactionIndex < items[j].TransactionIndex
+					}
+					return items[i].BlockNumber < items[j].BlockNumber
+				})
+
+				for _, item := range items {
+					modelChan <- &item
 				}
 			}
+			bar.Finish(true /* newLine */)
 		}
 	}
 
 	extraOpts := map[string]any{
-		"uncles":     opts.Uncles,
-		"logs":       opts.Logs,
-		"traces":     opts.Traces,
-		"addresses":  opts.Uniq,
 		"articulate": opts.Articulate,
 	}
+
 	return output.StreamMany(ctx, fetchData, opts.Globals.OutputOptsWithExtra(extraOpts))
 }

src/apps/chifra/internal/blocks/handle_uncles.go

@@ -39,8 +39,9 @@ func (opts *BlocksOptions) HandleUncles() error {
 			cancel()
 
 		} else {
+			showProgress := opts.Globals.ShowProgress()
 			bar := logger.NewBar(logger.BarOptions{
-				Enabled: !testMode && !logger.IsTerminal(),
+				Enabled: showProgress,
 				Total:   int64(cnt),
 			})