Merge pull request #3428 from TrueBlocks/develop

Better Concurrency 1
TrueBlocks · Nov 29, 2023 · 4e530b4 · 4e530b4
2 parents a12ec59 + dc5ce3d
commit 4e530b4
Show file tree

Hide file tree

Showing 51 changed files with 726 additions and 707 deletions.
diff --git a/docs/content/api/openapi.yaml b/docs/content/api/openapi.yaml
@@ -7,7 +7,7 @@ info:
   license:
     name: GPL 3.0
     url: http://www.gnu.org/licenses/
-  version: 2.1.0-release
+  version: 2.2.0-release
   description: >
 
     A REST layer over the TrueBlocks application. With `chifra daemon`, you can
@@ -3918,6 +3918,9 @@ components:
       type: string
     string:
       type: string
+    uint64:
+      type: number
+      format: uint64
     topic:
       type: string
       format: bytes

diff --git a/docs/templates/api/components.txt b/docs/templates/api/components.txt
@@ -1531,6 +1531,9 @@ components:
       type: string
     string:
       type: string
+    uint64:
+      type: number
+      format: uint64
     topic:
       type: string
       format: bytes

diff --git a/.../chifra/internal/abis/handle_addresses.go → src/apps/chifra/internal/abis/handle_show.go b/.../chifra/internal/abis/handle_addresses.go → src/apps/chifra/internal/abis/handle_show.go
@@ -14,7 +14,7 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
 )
 
-func (opts *AbisOptions) HandleAddresses() (err error) {
+func (opts *AbisOptions) HandleShow() (err error) {
 	abiCache := articulate.NewAbiCache(opts.Conn, opts.Known)
 
 	ctx, cancel := context.WithCancel(context.Background())

diff --git a/src/apps/chifra/internal/abis/output.go b/src/apps/chifra/internal/abis/output.go
@@ -59,7 +59,7 @@ func (opts *AbisOptions) AbisInternal() error {
 	} else if len(opts.Encode) > 0 {
 		err = opts.HandleEncode()
 	} else {
-		err = opts.HandleAddresses()
+		err = opts.HandleShow()
 	}
 	// EXISTING_CODE
 	timer.Report(msg)

diff --git a/src/apps/chifra/internal/blocks/handle_hashes.go b/src/apps/chifra/internal/blocks/handle_hashes.go
@@ -19,28 +19,28 @@ import (
 
 func (opts *BlocksOptions) HandleHashes() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
 	nErrors := 0
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawBlock], errorChan chan error) {
+		// var cnt int
 		var err error
-		var appMap map[identifiers.ResolvedId]*types.SimpleBlock[string]
+		var appMap map[types.SimpleAppearance]*types.SimpleBlock[string]
 		if appMap, _, err = identifiers.AsMap[types.SimpleBlock[string]](chain, opts.BlockIds); err != nil {
 			errorChan <- err
 			cancel()
 		}
 
-		iterCtx, iterCancel := context.WithCancel(context.Background())
-		defer iterCancel()
-
 		bar := logger.NewBar(logger.BarOptions{
 			Type:    logger.Expanding,
 			Enabled: !opts.Globals.TestMode,
 			Total:   int64(len(appMap)),
 		})
 
-		iterFunc := func(app identifiers.ResolvedId, value *types.SimpleBlock[string]) error {
-			if block, err := opts.Conn.GetBlockHeaderByNumber(app.BlockNumber); err != nil {
+		iterFunc := func(app types.SimpleAppearance, value *types.SimpleBlock[string]) error {
+			bn := uint64(app.BlockNumber)
+			if block, err := opts.Conn.GetBlockHeaderByNumber(bn); err != nil {
 				errorChan <- err
 				if errors.Is(err, ethereum.NotFound) {
 					errorChan <- errors.New("uncles not found")
@@ -55,9 +55,11 @@ func (opts *BlocksOptions) HandleHashes() error {
 		}
 
 		iterErrorChan := make(chan error)
+		iterCtx, iterCancel := context.WithCancel(context.Background())
+		defer iterCancel()
 		go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
 		for err := range iterErrorChan {
-			if !opts.Globals.TestMode || nErrors == 0 {
+			if !testMode || nErrors == 0 {
 				errorChan <- err
 				nErrors++
 			}

diff --git a/src/apps/chifra/internal/blocks/handle_logs.go b/src/apps/chifra/internal/blocks/handle_logs.go
@@ -20,6 +20,9 @@ import (
 
 func (opts *BlocksOptions) HandleLogs() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
+	nErrors := 0
+
 	abiCache := articulate.NewAbiCache(opts.Conn, opts.Articulate)
 	emitters := []base.Address{}
 	for _, e := range opts.Emitter {
@@ -34,35 +37,34 @@ func (opts *BlocksOptions) HandleLogs() error {
 		Topics:   topics,
 	}
 
-	nErrors := 0
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawLog], errorChan chan error) {
-		txMap, _, err := identifiers.AsMap[types.SimpleTransaction](chain, opts.BlockIds)
-		if err != nil {
+		// var cnt int
+		var err error
+		var appMap map[types.SimpleAppearance]*types.SimpleTransaction
+		if appMap, _, err = identifiers.AsMap[types.SimpleTransaction](chain, opts.BlockIds); err != nil {
 			errorChan <- err
 			cancel()
 		}
 
 		bar := logger.NewBar(logger.BarOptions{
 			Enabled: !opts.Globals.TestMode,
-			Total:   int64(len(txMap)),
+			Total:   int64(len(appMap)),
 		})
 
-		iterCtx, iterCancel := context.WithCancel(context.Background())
-		defer iterCancel()
-
-		iterFunc := func(app identifiers.ResolvedId, value *types.SimpleTransaction) error {
+		iterFunc := func(app types.SimpleAppearance, value *types.SimpleTransaction) error {
 			if value.Receipt == nil {
 				value.Receipt = &types.SimpleReceipt{}
 			}
 
-			ts := opts.Conn.GetBlockTimestamp(app.BlockNumber)
-			if logs, err := opts.Conn.GetLogsByNumber(app.BlockNumber, ts); err != nil {
-				errorChan <- fmt.Errorf("block at %d returned an error: %w", app.BlockNumber, err)
+			bn := uint64(app.BlockNumber)
+			ts := opts.Conn.GetBlockTimestamp(bn)
+			if logs, err := opts.Conn.GetLogsByNumber(bn, ts); err != nil {
+				errorChan <- fmt.Errorf("block at %d returned an error: %w", bn, err)
 				return nil
 
 			} else if len(logs) == 0 {
-				errorChan <- fmt.Errorf("block at %d has no logs", app.BlockNumber)
+				errorChan <- fmt.Errorf("block at %d has no logs", bn)
 				return nil
 
 			} else {
@@ -82,11 +84,11 @@ func (opts *BlocksOptions) HandleLogs() error {
 		}
 
 		iterErrorChan := make(chan error)
-		go utils.IterateOverMap(iterCtx, iterErrorChan, txMap, iterFunc)
+		iterCtx, iterCancel := context.WithCancel(context.Background())
+		defer iterCancel()
+		go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
 		for err := range iterErrorChan {
-			// TODO: I don't really want to quit looping here. Just report the error and keep going.
-			// iterCancel()
-			if !opts.Globals.TestMode || nErrors == 0 {
+			if !testMode || nErrors == 0 {
 				errorChan <- err
 				// Reporting more than one error causes tests to fail because they
 				// appear concurrently so sort differently
@@ -95,8 +97,8 @@ func (opts *BlocksOptions) HandleLogs() error {
 		}
 		bar.Finish(true)
 
-		items := make([]types.SimpleLog, 0, len(txMap))
-		for _, tx := range txMap {
+		items := make([]types.SimpleLog, 0, len(appMap))
+		for _, tx := range appMap {
 			tx := tx
 			items = append(items, tx.Receipt.Logs...)
 		}

diff --git a/src/apps/chifra/internal/blocks/handle_show.go b/src/apps/chifra/internal/blocks/handle_show.go
@@ -19,28 +19,27 @@ import (
 
 func (opts *BlocksOptions) HandleShow() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
 	nErrors := 0
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawBlock], errorChan chan error) {
+		// var cnt int
 		var err error
-		var appMap map[identifiers.ResolvedId]*types.SimpleBlock[types.SimpleTransaction]
+		var appMap map[types.SimpleAppearance]*types.SimpleBlock[types.SimpleTransaction]
 		if appMap, _, err = identifiers.AsMap[types.SimpleBlock[types.SimpleTransaction]](chain, opts.BlockIds); err != nil {
 			errorChan <- err
 			cancel()
 		}
 
-		iterCtx, iterCancel := context.WithCancel(context.Background())
-		defer iterCancel()
-
 		bar := logger.NewBar(logger.BarOptions{
 			Type:    logger.Expanding,
 			Enabled: !opts.Globals.TestMode,
 			Total:   int64(len(appMap)),
 		})
 
-		iterFunc := func(app identifiers.ResolvedId, value *types.SimpleBlock[types.SimpleTransaction]) error {
-			if block, err := opts.Conn.GetBlockBodyByNumber(app.BlockNumber); err != nil {
+		iterFunc := func(app types.SimpleAppearance, value *types.SimpleBlock[types.SimpleTransaction]) error {
+			if block, err := opts.Conn.GetBlockBodyByNumber(uint64(app.BlockNumber)); err != nil {
 				errorChan <- err
 				if errors.Is(err, ethereum.NotFound) {
 					errorChan <- errors.New("uncles not found")
@@ -55,9 +54,11 @@ func (opts *BlocksOptions) HandleShow() error {
 		}
 
 		iterErrorChan := make(chan error)
+		iterCtx, iterCancel := context.WithCancel(context.Background())
+		defer iterCancel()
 		go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
 		for err := range iterErrorChan {
-			if !opts.Globals.TestMode || nErrors == 0 {
+			if !testMode || nErrors == 0 {
 				errorChan <- err
 				nErrors++
 			}

diff --git a/src/apps/chifra/internal/blocks/handle_uncles.go b/src/apps/chifra/internal/blocks/handle_uncles.go
@@ -19,29 +19,29 @@ import (
 
 func (opts *BlocksOptions) HandleUncles() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
 	nErrors := 0
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawBlock], errorChan chan error) {
+		// var cnt int
 		var err error
-		var appMap map[identifiers.ResolvedId]*types.SimpleBlock[types.SimpleTransaction]
+		var appMap map[types.SimpleAppearance]*types.SimpleBlock[types.SimpleTransaction]
 		if appMap, _, err = identifiers.AsMap[types.SimpleBlock[types.SimpleTransaction]](chain, opts.BlockIds); err != nil {
 			errorChan <- err
 			cancel()
 		}
 
-		iterCtx, iterCancel := context.WithCancel(context.Background())
-		defer iterCancel()
-
-		uncles := make([]types.SimpleBlock[types.SimpleTransaction], 0, len(appMap))
 		bar := logger.NewBar(logger.BarOptions{
 			Type:    logger.Expanding,
 			Enabled: !opts.Globals.TestMode,
 			Total:   int64(len(appMap)),
 		})
 
-		iterFunc := func(app identifiers.ResolvedId, value *types.SimpleBlock[types.SimpleTransaction]) error {
-			if uncs, err := opts.Conn.GetUncleBodiesByNumber(app.BlockNumber); err != nil {
+		uncles := make([]types.SimpleBlock[types.SimpleTransaction], 0, len(appMap))
+		iterFunc := func(app types.SimpleAppearance, value *types.SimpleBlock[types.SimpleTransaction]) error {
+			bn := uint64(app.BlockNumber)
+			if uncs, err := opts.Conn.GetUncleBodiesByNumber(bn); err != nil {
 				errorChan <- err
 				if errors.Is(err, ethereum.NotFound) {
 					errorChan <- errors.New("uncles not found")
@@ -61,9 +61,11 @@ func (opts *BlocksOptions) HandleUncles() error {
 		}
 
 		iterErrorChan := make(chan error)
+		iterCtx, iterCancel := context.WithCancel(context.Background())
+		defer iterCancel()
 		go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
 		for err := range iterErrorChan {
-			if !opts.Globals.TestMode || nErrors == 0 {
+			if !testMode || nErrors == 0 {
 				errorChan <- err
 				nErrors++
 			}

diff --git a/src/apps/chifra/internal/blocks/handle_uniq.go b/src/apps/chifra/internal/blocks/handle_uniq.go
@@ -20,34 +20,35 @@ import (
 
 func (opts *BlocksOptions) HandleUniq() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
 	nErrors := 0
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawAppearance], errorChan chan error) {
+		// var cnt int
 		var err error
-		var appMap map[identifiers.ResolvedId]*types.SimpleAppearance
+		var appMap map[types.SimpleAppearance]*types.SimpleAppearance
 		if appMap, _, err = identifiers.AsMap[types.SimpleAppearance](chain, opts.BlockIds); err != nil {
 			errorChan <- err
 			cancel()
 		}
 
-		iterCtx, iterCancel := context.WithCancel(context.Background())
-		defer iterCancel()
-
-		apps := make([]types.SimpleAppearance, 0, len(appMap))
 		bar := logger.NewBar(logger.BarOptions{
 			Type:    logger.Expanding,
 			Enabled: !opts.Globals.TestMode,
 			Total:   int64(len(appMap)),
 		})
-		iterFunc := func(app identifiers.ResolvedId, value *types.SimpleAppearance) error {
+
+		apps := make([]types.SimpleAppearance, 0, len(appMap))
+		iterFunc := func(app types.SimpleAppearance, value *types.SimpleAppearance) error {
+			bn := uint64(app.BlockNumber)
 			procFunc := func(s *types.SimpleAppearance) error {
 				bar.Tick()
 				apps = append(apps, *s)
 				return nil
 			}
 
-			if err := uniq.GetUniqAddressesInBlock(chain, opts.Flow, opts.Conn, procFunc, app.BlockNumber); err != nil {
+			if err := uniq.GetUniqAddressesInBlock(chain, opts.Flow, opts.Conn, procFunc, bn); err != nil {
 				errorChan <- err
 				if errors.Is(err, ethereum.NotFound) {
 					return nil
@@ -58,9 +59,11 @@ func (opts *BlocksOptions) HandleUniq() error {
 		}
 
 		iterErrorChan := make(chan error)
+		iterCtx, iterCancel := context.WithCancel(context.Background())
+		defer iterCancel()
 		go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
 		for err := range iterErrorChan {
-			if !opts.Globals.TestMode || nErrors == 0 {
+			if !testMode || nErrors == 0 {
 				errorChan <- err
 				nErrors++
 			}

diff --git a/src/apps/chifra/internal/blocks/handle_withdrawals.go b/src/apps/chifra/internal/blocks/handle_withdrawals.go
@@ -17,28 +17,28 @@ import (
 
 func (opts *BlocksOptions) HandleWithdrawals() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
 	nErrors := 0
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawWithdrawal], errorChan chan error) {
+		// var cnt int
 		var err error
-		var appMap map[identifiers.ResolvedId]*types.SimpleBlock[string]
+		var appMap map[types.SimpleAppearance]*types.SimpleBlock[string]
 		if appMap, _, err = identifiers.AsMap[types.SimpleBlock[string]](chain, opts.BlockIds); err != nil {
 			errorChan <- err
 			cancel()
 		}
 
-		iterCtx, iterCancel := context.WithCancel(context.Background())
-		defer iterCancel()
-
 		bar := logger.NewBar(logger.BarOptions{
 			Type:    logger.Expanding,
 			Enabled: !opts.Globals.TestMode,
 			Total:   int64(len(appMap)),
 		})
 
-		iterFunc := func(app identifiers.ResolvedId, value *types.SimpleBlock[string]) error {
-			if block, err := opts.Conn.GetBlockHeaderByNumber(app.BlockNumber); err != nil {
+		iterFunc := func(app types.SimpleAppearance, value *types.SimpleBlock[string]) error {
+			bn := uint64(app.BlockNumber)
+			if block, err := opts.Conn.GetBlockHeaderByNumber(bn); err != nil {
 				errorChan <- err
 				cancel()
 				return nil
@@ -50,9 +50,11 @@ func (opts *BlocksOptions) HandleWithdrawals() error {
 		}
 
 		iterErrorChan := make(chan error)
+		iterCtx, iterCancel := context.WithCancel(context.Background())
+		defer iterCancel()
 		go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
 		for err := range iterErrorChan {
-			if !opts.Globals.TestMode || nErrors == 0 {
+			if !testMode || nErrors == 0 {
 				errorChan <- err
 				nErrors++
 			}