Remove traces downloader

Makefile

@@ -35,7 +35,6 @@ generate:
 	mockery --dir=storage --name=TransactionIndexer --output=storage/mocks
 	mockery --dir=storage --name=AccountIndexer --output=storage/mocks
 	mockery --dir=storage --name=TraceIndexer --output=storage/mocks
-	mockery --all --dir=services/traces --output=services/traces/mocks
 	mockery --all --dir=services/ingestion --output=services/ingestion/mocks
 	mockery --dir=models --name=Engine --output=models/mocks
 

README.md

@@ -222,14 +222,10 @@ The application can be configured using the following flags at runtime:
 | `stream-limit`                 | `10`                          | Rate-limit for client events sent per second                                             |
 | `rate-limit`                   | `50`                          | Requests per second limit for clients over any protocol (ws/http)                        |
 | `address-header`               | `""`                          | Header for client IP when server is behind a proxy                                       |
-| `heartbeat-interval`           | `100`                         | Interval for AN event subscription heartbeats                                            |
 | `stream-timeout`               | `3`                           | Timeout in seconds for sending events to clients                                         |
 | `force-start-height`           | `0`                           | Force-set starting Cadence height (local/testing use only)                               |
 | `wallet-api-key`               | `""`                          | ECDSA private key for wallet APIs (local/testing use only)                               |
 | `filter-expiry`                | `5m`                          | Expiry time for idle filters                                                             |
-| `traces-gcp-bucket`            | `""`                          | GCP bucket name for transaction traces                                                   |
-| `traces-backfill-start-height` | `0`                           | Start height for backfilling transaction traces                                          |
-| `traces-backfill-end-height`   | `0`                           | End height for backfilling transaction traces                                            |
 | `index-only`                   | `false`                       | Run in index-only mode, allowing state queries and indexing but no transaction sending   |
 | `profiler-enabled`             | `false`                       | Enable the pprof profiler server                                                         |
 | `profiler-host`                | `localhost`                   | Host for the pprof profiler                                                              |

bootstrap/bootstrap.go

@@ -25,7 +25,6 @@ import (
 	"github.com/onflow/flow-evm-gateway/services/ingestion"
 	"github.com/onflow/flow-evm-gateway/services/replayer"
 	"github.com/onflow/flow-evm-gateway/services/requester"
-	"github.com/onflow/flow-evm-gateway/services/traces"
 	"github.com/onflow/flow-evm-gateway/storage"
 	"github.com/onflow/flow-evm-gateway/storage/pebble"
 
@@ -64,7 +63,6 @@ type Bootstrap struct {
 	server     *api.Server
 	metrics    *metrics.Server
 	events     *ingestion.Engine
-	traces     *traces.Engine
 	profiler   *api.ProfileServer
 }
 
@@ -169,71 +167,6 @@ func (b *Bootstrap) StartEventIngestion(ctx context.Context) error {
 	return nil
 }
 
-func (b *Bootstrap) StartTraceDownloader(ctx context.Context) error {
-	l := b.logger.With().Str("component", "bootstrap-traces").Logger()
-	l.Info().Msg("starting engine")
-
-	// create gcp downloader
-	downloader, err := traces.NewGCPDownloader(b.config.TracesBucketName, b.logger)
-	if err != nil {
-		return err
-	}
-
-	// initialize trace downloader engine
-	b.traces = traces.NewTracesIngestionEngine(
-		b.publishers.Block,
-		b.storages.Blocks,
-		b.storages.Traces,
-		downloader,
-		b.logger,
-		b.collector,
-	)
-
-	StartEngine(ctx, b.traces, l)
-
-	if b.config.TracesBackfillStartHeight > 0 {
-		startHeight := b.config.TracesBackfillStartHeight
-		if _, err := b.storages.Blocks.GetByHeight(startHeight); err != nil {
-			return fmt.Errorf("failed to get provided start height %d in db: %w", startHeight, err)
-		}
-
-		cadenceStartHeight, err := b.storages.Blocks.GetCadenceHeight(startHeight)
-		if err != nil {
-			return fmt.Errorf("failed to get cadence height for backfill start height %d: %w", startHeight, err)
-		}
-
-		if cadenceStartHeight < b.config.InitCadenceHeight {
-			b.logger.Warn().
-				Uint64("evm-start-height", startHeight).
-				Uint64("cadence-start-height", cadenceStartHeight).
-				Uint64("init-cadence-height", b.config.InitCadenceHeight).
-				Msg("backfill start height is before initial cadence height. data may be missing from configured traces bucket")
-		}
-
-		endHeight := b.config.TracesBackfillEndHeight
-		if endHeight == 0 {
-			endHeight, err = b.storages.Blocks.LatestEVMHeight()
-			if err != nil {
-				return fmt.Errorf("failed to get latest EVM height: %w", err)
-			}
-		} else if _, err := b.storages.Blocks.GetByHeight(endHeight); err != nil {
-			return fmt.Errorf("failed to get provided end height %d in db: %w", endHeight, err)
-		}
-
-		go b.traces.Backfill(startHeight, endHeight)
-	}
-
-	return nil
-}
-
-func (b *Bootstrap) StopTraceDownloader() {
-	if b.traces == nil {
-		return
-	}
-	b.logger.Warn().Msg("stopping trace downloader engine")
-	b.traces.Stop()
-}
-
 func (b *Bootstrap) StopEventIngestion() {
 	if b.events == nil {
 		return
@@ -336,10 +269,7 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
 		ratelimiter,
 	)
 
-	var debugAPI *api.DebugAPI
-	if b.config.TracesEnabled {
-		debugAPI = api.NewDebugAPI(b.storages.Traces, b.storages.Blocks, b.logger, b.collector)
-	}
+	var debugAPI = api.NewDebugAPI(b.storages.Traces, b.storages.Blocks, b.logger, b.collector)
 
 	var walletAPI *api.WalletAPI
 	if b.config.WalletEnabled {
@@ -562,12 +492,6 @@ func Run(ctx context.Context, cfg *config.Config, ready chan struct{}) error {
 		return err
 	}
 
-	if cfg.TracesEnabled {
-		if err := boot.StartTraceDownloader(ctx); err != nil {
-			return fmt.Errorf("failed to start trace downloader engine: %w", err)
-		}
-	}
-
 	if err := boot.StartEventIngestion(ctx); err != nil {
 		return fmt.Errorf("failed to start event ingestion engine: %w", err)
 	}
@@ -593,7 +517,6 @@ func Run(ctx context.Context, cfg *config.Config, ready chan struct{}) error {
 
 	boot.StopEventIngestion()
 	boot.StopMetricsServer()
-	boot.StopTraceDownloader()
 	boot.StopAPIServer()
 
 	return nil

cmd/run/cmd.go

@@ -72,7 +72,7 @@ func parseConfigFromFlags() error {
 
 	if g, ok := new(big.Int).SetString(gas, 10); ok {
 		cfg.GasPrice = g
-	} else if !ok {
+	} else {
 		return fmt.Errorf("invalid gas price")
 	}
 
@@ -201,12 +201,6 @@ func parseConfigFromFlags() error {
 		cfg.ForceStartCadenceHeight = forceStartHeight
 	}
 
-	cfg.TracesEnabled = cfg.TracesBucketName != ""
-
-	if cfg.TracesBackfillStartHeight > 0 && cfg.TracesBackfillEndHeight > 0 && cfg.TracesBackfillStartHeight > cfg.TracesBackfillEndHeight {
-		return fmt.Errorf("traces backfill start height must be less than the end height")
-	}
-
 	if walletKey != "" {
 		k, err := gethCrypto.HexToECDSA(walletKey)
 		if err != nil {
@@ -272,9 +266,6 @@ func init() {
 	Cmd.Flags().IntVar(&streamTimeout, "stream-timeout", 3, "Defines the timeout in seconds the server waits for the event to be sent to the client")
 	Cmd.Flags().Uint64Var(&forceStartHeight, "force-start-height", 0, "Force set starting Cadence height. WARNING: This should only be used locally or for testing, never in production.")
 	Cmd.Flags().StringVar(&filterExpiry, "filter-expiry", "5m", "Filter defines the time it takes for an idle filter to expire")
-	Cmd.Flags().StringVar(&cfg.TracesBucketName, "traces-gcp-bucket", "", "GCP bucket name where transaction traces are stored")
-	Cmd.Flags().Uint64Var(&cfg.TracesBackfillStartHeight, "traces-backfill-start-height", 0, "evm block height from which to start backfilling missing traces.")
-	Cmd.Flags().Uint64Var(&cfg.TracesBackfillEndHeight, "traces-backfill-end-height", 0, "evm block height until which to backfill missing traces. If 0, backfill until the latest block")
 	Cmd.Flags().StringVar(&cloudKMSProjectID, "coa-cloud-kms-project-id", "", "The project ID containing the KMS keys, e.g. 'flow-evm-gateway'")
 	Cmd.Flags().StringVar(&cloudKMSLocationID, "coa-cloud-kms-location-id", "", "The location ID where the key ring is grouped into, e.g. 'global'")
 	Cmd.Flags().StringVar(&cloudKMSKeyRingID, "coa-cloud-kms-key-ring-id", "", "The key ring ID where the KMS keys exist, e.g. 'tx-signing'")

config/config.go

@@ -74,14 +74,6 @@ type Config struct {
 	FilterExpiry time.Duration
 	// ForceStartCadenceHeight will force set the starting Cadence height, this should be only used for testing or locally.
 	ForceStartCadenceHeight uint64
-	// TracesBucketName sets the GCP bucket name where transaction traces are being stored.
-	TracesBucketName string
-	// TracesEnabled sets whether the node is supporting transaction traces.
-	TracesEnabled bool
-	// TracesBackfillStartHeight sets the starting block height for backfilling missing traces.
-	TracesBackfillStartHeight uint64
-	// TracesBackfillEndHeight sets the ending block height for backfilling missing traces.
-	TracesBackfillEndHeight uint64
 	// WalletEnabled sets whether wallet APIs are enabled
 	WalletEnabled bool
 	// WalletKey used for signing transactions

go.mod

@@ -3,7 +3,6 @@ module github.com/onflow/flow-evm-gateway
 go 1.22
 
 require (
-	cloud.google.com/go/storage v1.36.0
 	github.com/cockroachdb/pebble v1.1.1
 	github.com/goccy/go-json v0.10.2
 	github.com/hashicorp/golang-lru/v2 v2.0.7
@@ -23,7 +22,6 @@ require (
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/sync v0.8.0
-	google.golang.org/api v0.162.0
 	google.golang.org/grpc v1.63.2
 )
 
@@ -33,6 +31,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/kms v1.15.7 // indirect
+	cloud.google.com/go/storage v1.36.0 // indirect
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc // indirect
@@ -200,6 +199,7 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect
+	google.golang.org/api v0.162.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect