From fc36a206b6ed4b8bbac2dc2260e6c6b063dc7fe9 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 14:37:32 +0200 Subject: [PATCH 01/58] add cross-spork client type --- models/cross-spork_client.go | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 models/cross-spork_client.go diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go new file mode 100644 index 000000000..c75906dc1 --- /dev/null +++ b/models/cross-spork_client.go @@ -0,0 +1,44 @@ +package models + +import ( + "fmt" + "github.com/onflow/flow-go-sdk/access" + "math" +) + +// CrossSporkClient is a wrapper around the Flow AN client that can +// access different AN APIs based on the height boundaries of the sporks. +// +// Each spork is defined with the last height included in that spork, +// based on the list we know which AN host to use when requesting the data. +type CrossSporkClient struct { + // this map holds the last height and host for each spork + sporkHosts map[uint64]string + + access.Client +} + +// NewCrossSporkClient creates a new instance of the client, it accepts the +// host to the current spork AN API. +func NewCrossSporkClient(currentSporkHost string) (*CrossSporkClient, error) { + c := &CrossSporkClient{ + sporkHosts: make(map[uint64]string), + } + // add current spork AN host with the highest value for last height + err := c.AddSpork(math.MaxUint64, currentSporkHost) + if err != nil { + return nil, err + } + + return c, nil +} + +func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { + if _, ok := c.sporkHosts[lastHeight]; ok { + return fmt.Errorf("provided last height already exists") + } + + c.sporkHosts[lastHeight] = host + + return nil +} From 0eaa3d434831c4fe06a60bf2b7f6c0a6d9a6e6f9 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 15:03:40 +0200 Subject: [PATCH 02/58] add cross-spork methods --- models/cross-spork_client.go | 90 +++++++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index c75906dc1..506b97542 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -1,19 +1,27 @@ package models import ( + "context" "fmt" + "github.com/onflow/cadence" + "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" - "math" + "github.com/onflow/flow-go-sdk/access/grpc" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) // CrossSporkClient is a wrapper around the Flow AN client that can // access different AN APIs based on the height boundaries of the sporks. // // Each spork is defined with the last height included in that spork, -// based on the list we know which AN host to use when requesting the data. +// based on the list we know which AN client to use when requesting the data. +// +// Any API that supports cross-spork access must have a defined function +// that shadows the original access Client function. type CrossSporkClient struct { - // this map holds the last height and host for each spork - sporkHosts map[uint64]string + // this map holds the last heights and clients for each spork + sporkHosts map[uint64]access.Client access.Client } @@ -21,16 +29,16 @@ type CrossSporkClient struct { // NewCrossSporkClient creates a new instance of the client, it accepts the // host to the current spork AN API. func NewCrossSporkClient(currentSporkHost string) (*CrossSporkClient, error) { - c := &CrossSporkClient{ - sporkHosts: make(map[uint64]string), - } - // add current spork AN host with the highest value for last height - err := c.AddSpork(math.MaxUint64, currentSporkHost) + // add current spork AN host as the default client + client, err := grpc.NewClient(currentSporkHost) if err != nil { return nil, err } - return c, nil + return &CrossSporkClient{ + make(map[uint64]access.Client), + client, + }, nil } func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { @@ -38,7 +46,67 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { return fmt.Errorf("provided last height already exists") } - c.sporkHosts[lastHeight] = host + client, err := grpc.NewClient(host) + if err != nil { + return err + } + + c.sporkHosts[lastHeight] = client return nil } + +// getClientForHeight returns the client for the given height. It starts by using the current spork client, +// then iteratively checks the upper height boundaries in descending order and returns the last client +// that still contains the given height within its upper height limit. If no client is found, it returns +// the current spork client. +// Please note that even if a client for provided height is found we don't guarantee the data being available +// because it still might not have access to the height provided, because there might be other sporks with +// lower height boundaries that we didn't configure for. +// This would result in the error when using the client to access such data. +func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { + heights := maps.Keys(c.sporkHosts) + slices.Sort(heights) // order heights in ascending order + slices.Reverse(heights) // make it descending + + // start by using the current spork client, then iterate all the upper height boundaries + // and find the last client that still contains the height in its upper height limit + client := c.Client + for _, upperBound := range heights { + if upperBound > height { + client = c.sporkHosts[upperBound] + } + } + + return client +} + +func (c *CrossSporkClient) GetBlockByHeight( + ctx context.Context, + height uint64, +) (*flow.Block, error) { + return c. + getClientForHeight(height). + GetBlockByHeight(ctx, height) +} + +func (c *CrossSporkClient) ExecuteScriptAtBlockHeight( + ctx context.Context, + height uint64, + script []byte, + arguments []cadence.Value, +) (cadence.Value, error) { + return c. + getClientForHeight(height). + ExecuteScriptAtBlockHeight(ctx, height, script, arguments) +} + +func (c *CrossSporkClient) SubscribeEventsByBlockHeight( + ctx context.Context, + startHeight uint64, + filter flow.EventFilter, +) (<-chan flow.BlockEvents, <-chan error, error) { + return c. + getClientForHeight(startHeight). + SubscribeEventsByBlockHeight(ctx, startHeight, filter) +} From e47ad92d23303cb61a39106a6bca99f4b84cfc22 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 15:27:52 +0200 Subject: [PATCH 03/58] add support to config for providing the hosts values for previous sporks --- config/config.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index d181d5adf..e70486c77 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,8 @@ import ( "io" "math/big" "os" + "strconv" + "strings" "time" "github.com/goccy/go-json" @@ -29,8 +31,11 @@ const LiveNetworkInitCadenceHeght = uint64(1) type Config struct { // DatabaseDir is where the database should be stored. DatabaseDir string - // AccessNodeGRPCHost defines the Flow network AN host. - AccessNodeGRPCHost string + // AccessNodeHost defines the current spork Flow network AN host. + AccessNodeHost string + // AccessNodePreviousSporkHosts contains a map of latest heights for each spork, + // which can be accessed via the host of the AN provided + AccessNodePreviousSporkHosts map[uint64]string // GRPCPort for the RPC API server RPCPort int // GRPCHost for the RPC API server @@ -68,8 +73,10 @@ type Config struct { } func FromFlags() (*Config, error) { - cfg := &Config{} - var evmNetwork, coinbase, gas, coa, key, keysPath, flowNetwork, logLevel, filterExpiry string + cfg := &Config{ + AccessNodePreviousSporkHosts: make(map[uint64]string), + } + var evmNetwork, coinbase, gas, coa, key, keysPath, flowNetwork, logLevel, filterExpiry, accessSporkHosts string var streamTimeout int var initHeight uint64 @@ -77,7 +84,8 @@ func FromFlags() (*Config, error) { flag.StringVar(&cfg.DatabaseDir, "database-dir", "./db", "Path to the directory for the database") flag.StringVar(&cfg.RPCHost, "rpc-host", "", "Host for the RPC API server") flag.IntVar(&cfg.RPCPort, "rpc-port", 8545, "Port for the RPC API server") - flag.StringVar(&cfg.AccessNodeGRPCHost, "access-node-grpc-host", "localhost:3569", "Host to the flow access node gRPC API") + flag.StringVar(&cfg.AccessNodeHost, "access-node-grpc-host", "localhost:3569", "Host to the flow access node gRPC API") + flag.StringVar(&accessSporkHosts, "access-node-spork-hosts", "", `Previous spork AN hosts, defined following the schema: {latest height}@{host} as comma separated list (e.g. "200@host-1.com,300@host2.com")`) flag.StringVar(&evmNetwork, "evm-network-id", "previewnet", "EVM network ID (previewnet, testnet, mainnet)") flag.StringVar(&flowNetwork, "flow-network-id", "flow-emulator", "Flow network ID (flow-emulator, flow-previewnet)") flag.StringVar(&coinbase, "coinbase", "", "Coinbase address to use for fee collection") @@ -181,6 +189,20 @@ func FromFlags() (*Config, error) { } cfg.FilterExpiry = exp + if accessSporkHosts != "" { + heightHosts := strings.Split(accessSporkHosts, ",") + for _, hh := range heightHosts { + v := strings.Split(hh, "@") + heightVal, host := v[0], v[1] + height, err := strconv.Atoi(heightVal) + if err != nil { + return nil, fmt.Errorf("failed to parse AN host height value for previous sporks, provided with --access-node-spork-hosts flag") + } + + cfg.AccessNodePreviousSporkHosts[uint64(height)] = host + } + } + // todo validate Config values return cfg, nil } From b887c37fe60a6d850c5bb783d3b64372637b114a Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 15:29:29 +0200 Subject: [PATCH 04/58] setup cross-spork client --- bootstrap/bootstrap.go | 11 +++++++++-- tests/helpers.go | 30 +++++++++++++++--------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 9430603dc..83ce610ba 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -99,11 +99,18 @@ func startIngestion( ) error { logger.Info().Msg("starting up event ingestion") - client, err := grpc.NewClient(cfg.AccessNodeGRPCHost) + client, err := models.NewCrossSporkClient(cfg.AccessNodeHost) if err != nil { return err } + // if we provided access node previous spork hosts add them to the client + for height, host := range cfg.AccessNodePreviousSporkHosts { + if err := client.AddSpork(height, host); err != nil { + return fmt.Errorf("failed to add previous spork host to the client: %w", err) + } + } + blk, err := client.GetLatestBlock(context.Background(), false) if err != nil { return fmt.Errorf("failed to get latest cadence block: %w", err) @@ -172,7 +179,7 @@ func startServer( srv := api.NewHTTPServer(l, rpc.DefaultHTTPTimeouts) - client, err := grpc.NewClient(cfg.AccessNodeGRPCHost) + client, err := grpc.NewClient(cfg.AccessNodeHost) if err != nil { return err } diff --git a/tests/helpers.go b/tests/helpers.go index a1d169d5e..8238d254e 100644 --- a/tests/helpers.go +++ b/tests/helpers.go @@ -129,21 +129,21 @@ func servicesSetup(t *testing.T) (emulator.Emulator, func()) { // default config cfg := &config.Config{ - DatabaseDir: t.TempDir(), - AccessNodeGRPCHost: "localhost:3569", // emulator - RPCPort: 8545, - RPCHost: "127.0.0.1", - FlowNetworkID: "flow-emulator", - EVMNetworkID: evmTypes.FlowEVMPreviewNetChainID, - Coinbase: common.HexToAddress(eoaTestAddress), - COAAddress: service.Address, - COAKey: service.PrivateKey, - CreateCOAResource: false, - GasPrice: new(big.Int).SetUint64(0), - LogLevel: zerolog.DebugLevel, - LogWriter: zerolog.NewConsoleWriter(), - StreamTimeout: time.Second * 30, - StreamLimit: 10, + DatabaseDir: t.TempDir(), + AccessNodeHost: "localhost:3569", // emulator + RPCPort: 8545, + RPCHost: "127.0.0.1", + FlowNetworkID: "flow-emulator", + EVMNetworkID: evmTypes.FlowEVMPreviewNetChainID, + Coinbase: common.HexToAddress(eoaTestAddress), + COAAddress: service.Address, + COAKey: service.PrivateKey, + CreateCOAResource: false, + GasPrice: new(big.Int).SetUint64(0), + LogLevel: zerolog.DebugLevel, + LogWriter: zerolog.NewConsoleWriter(), + StreamTimeout: time.Second * 30, + StreamLimit: 10, } if !logOutput { From cf5e6ed7166d772a5b25a57a48847080fee13778 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 15:29:35 +0200 Subject: [PATCH 05/58] change name --- tests/integration_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/integration_test.go b/tests/integration_test.go index 1f25047e5..b41dd2d51 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -55,19 +55,19 @@ func Test_ConcurrentTransactionSubmission(t *testing.T) { require.NoError(t, err) cfg := &config.Config{ - DatabaseDir: t.TempDir(), - AccessNodeGRPCHost: grpcHost, - RPCPort: 8545, - RPCHost: "127.0.0.1", - FlowNetworkID: "flow-emulator", - EVMNetworkID: types.FlowEVMTestNetChainID, - Coinbase: eoaTestAccount, - COAAddress: *createdAddr, - COAKeys: keys, - CreateCOAResource: true, - GasPrice: new(big.Int).SetUint64(0), - LogLevel: zerolog.DebugLevel, - LogWriter: os.Stdout, + DatabaseDir: t.TempDir(), + AccessNodeHost: grpcHost, + RPCPort: 8545, + RPCHost: "127.0.0.1", + FlowNetworkID: "flow-emulator", + EVMNetworkID: types.FlowEVMTestNetChainID, + Coinbase: eoaTestAccount, + COAAddress: *createdAddr, + COAKeys: keys, + CreateCOAResource: true, + GasPrice: new(big.Int).SetUint64(0), + LogLevel: zerolog.DebugLevel, + LogWriter: os.Stdout, } // todo change this test to use ingestion and emulator directly so we can completely remove From 4ca509305bd99e19862fde637d4f4959c6d83dbd Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:28:55 +0200 Subject: [PATCH 06/58] add logging --- models/cross-spork_client.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 506b97542..585b2e2b2 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -7,6 +7,7 @@ import ( "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/rs/zerolog" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -20,6 +21,7 @@ import ( // Any API that supports cross-spork access must have a defined function // that shadows the original access Client function. type CrossSporkClient struct { + logger zerolog.Logger // this map holds the last heights and clients for each spork sporkHosts map[uint64]access.Client @@ -28,7 +30,7 @@ type CrossSporkClient struct { // NewCrossSporkClient creates a new instance of the client, it accepts the // host to the current spork AN API. -func NewCrossSporkClient(currentSporkHost string) (*CrossSporkClient, error) { +func NewCrossSporkClient(currentSporkHost string, logger zerolog.Logger) (*CrossSporkClient, error) { // add current spork AN host as the default client client, err := grpc.NewClient(currentSporkHost) if err != nil { @@ -36,6 +38,7 @@ func NewCrossSporkClient(currentSporkHost string) (*CrossSporkClient, error) { } return &CrossSporkClient{ + logger, make(map[uint64]access.Client), client, }, nil @@ -53,6 +56,11 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { c.sporkHosts[lastHeight] = client + c.logger.Info(). + Uint64("spork-boundary", lastHeight). + Str("host", host). + Msg("added spork specific client") + return nil } @@ -73,8 +81,12 @@ func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { // and find the last client that still contains the height in its upper height limit client := c.Client for _, upperBound := range heights { - if upperBound > height { + if upperBound >= height { client = c.sporkHosts[upperBound] + + c.logger.Debug(). + Uint64("spork-boundary", upperBound). + Msg("using previous spork client") } } From ca26af25e243e6217b3bbec52384a38d89498416 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:29:01 +0200 Subject: [PATCH 07/58] check correct client is used --- models/cross-spork_client_test.go | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 models/cross-spork_client_test.go diff --git a/models/cross-spork_client_test.go b/models/cross-spork_client_test.go new file mode 100644 index 000000000..7caa69921 --- /dev/null +++ b/models/cross-spork_client_test.go @@ -0,0 +1,37 @@ +package models + +import ( + "context" + "github.com/stretchr/testify/require" + "testing" +) + +func TestCrossSporkClient_MultiClient(t *testing.T) { + clientHosts := []string{"test1.com", "test2.com", "test3.com"} + + client, err := NewCrossSporkClient(clientHosts[0]) + require.NoError(t, err) + + err = client.AddSpork(100, clientHosts[1]) + require.NoError(t, err) + + err = client.AddSpork(200, clientHosts[2]) + require.NoError(t, err) + + c := client.getClientForHeight(300) + require.NotNil(t, c) + + ctx := context.Background() + + // this height should use current spork client + _, err = client.GetBlockByHeight(ctx, 300) + require.ErrorContains(t, err, clientHosts[0]) + + // this height should use test2 client + _, err = client.GetBlockByHeight(ctx, 150) + require.ErrorContains(t, err, clientHosts[1]) + + // this height should use test3 client + _, err = client.GetBlockByHeight(ctx, 50) + require.ErrorContains(t, err, clientHosts[2]) +} From b29186899451f9af1f2de52a9beae696ef36e4e6 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:29:05 +0200 Subject: [PATCH 08/58] add logger --- bootstrap/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 83ce610ba..3c1e102bb 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -99,7 +99,7 @@ func startIngestion( ) error { logger.Info().Msg("starting up event ingestion") - client, err := models.NewCrossSporkClient(cfg.AccessNodeHost) + client, err := models.NewCrossSporkClient(cfg.AccessNodeHost, logger) if err != nil { return err } From 21ced97be40bff4b906912bb454304ae286b7986 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:30:04 +0200 Subject: [PATCH 09/58] use test logger --- models/cross-spork_client_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/cross-spork_client_test.go b/models/cross-spork_client_test.go index 7caa69921..cdf609c75 100644 --- a/models/cross-spork_client_test.go +++ b/models/cross-spork_client_test.go @@ -2,6 +2,7 @@ package models import ( "context" + "github.com/rs/zerolog" "github.com/stretchr/testify/require" "testing" ) @@ -9,7 +10,7 @@ import ( func TestCrossSporkClient_MultiClient(t *testing.T) { clientHosts := []string{"test1.com", "test2.com", "test3.com"} - client, err := NewCrossSporkClient(clientHosts[0]) + client, err := NewCrossSporkClient(clientHosts[0], zerolog.Nop()) require.NoError(t, err) err = client.AddSpork(100, clientHosts[1]) From b42c16aa439d3d1b38dbae23986fa072915dd8e2 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:30:44 +0200 Subject: [PATCH 10/58] test boundaries --- models/cross-spork_client_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/cross-spork_client_test.go b/models/cross-spork_client_test.go index cdf609c75..723999bfd 100644 --- a/models/cross-spork_client_test.go +++ b/models/cross-spork_client_test.go @@ -35,4 +35,8 @@ func TestCrossSporkClient_MultiClient(t *testing.T) { // this height should use test3 client _, err = client.GetBlockByHeight(ctx, 50) require.ErrorContains(t, err, clientHosts[2]) + + // test boundaries are inclusive + _, err = client.GetBlockByHeight(ctx, 200) + require.ErrorContains(t, err, clientHosts[2]) } From a7ee4db3cf56d33da08aa86046b392db1a6962f0 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:32:10 +0200 Subject: [PATCH 11/58] fix test --- models/cross-spork_client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/cross-spork_client_test.go b/models/cross-spork_client_test.go index 723999bfd..bd77bc856 100644 --- a/models/cross-spork_client_test.go +++ b/models/cross-spork_client_test.go @@ -30,11 +30,11 @@ func TestCrossSporkClient_MultiClient(t *testing.T) { // this height should use test2 client _, err = client.GetBlockByHeight(ctx, 150) - require.ErrorContains(t, err, clientHosts[1]) + require.ErrorContains(t, err, clientHosts[2]) // this height should use test3 client _, err = client.GetBlockByHeight(ctx, 50) - require.ErrorContains(t, err, clientHosts[2]) + require.ErrorContains(t, err, clientHosts[1]) // test boundaries are inclusive _, err = client.GetBlockByHeight(ctx, 200) From cbb71e4891708315bcd0773cd3ab5350f3c67a26 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:35:38 +0200 Subject: [PATCH 12/58] add existing err test --- models/cross-spork_client_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/models/cross-spork_client_test.go b/models/cross-spork_client_test.go index bd77bc856..e2127ea73 100644 --- a/models/cross-spork_client_test.go +++ b/models/cross-spork_client_test.go @@ -40,3 +40,14 @@ func TestCrossSporkClient_MultiClient(t *testing.T) { _, err = client.GetBlockByHeight(ctx, 200) require.ErrorContains(t, err, clientHosts[2]) } + +func TestCrossSporkClient_ExistingHeight(t *testing.T) { + client, err := NewCrossSporkClient("host1.com", zerolog.Nop()) + require.NoError(t, err) + + err = client.AddSpork(100, "host2.com") + require.NoError(t, err) + + err = client.AddSpork(100, "host3.com") + require.EqualError(t, err, "provided last height already exists") +} From f6fdfbca3a4749e51568b5010c68ac6d5bfbb5fb Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:38:40 +0200 Subject: [PATCH 13/58] update readme --- README.md | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 34d5adcd2..6b17cf349 100644 --- a/README.md +++ b/README.md @@ -90,23 +90,26 @@ it should return: The application can be configured using the following flags at runtime: -| Flag | Default Value | Description | -|---------------------------|------------------|------------------------------------------------------------------------------------------------------------------------| -| `--database-dir` | `./db` | Path to the directory for the database. | -| `--rpc-host` | `localhost` | Host for the JSON RPC API server. | -| `--rpc-port` | `8545` | Port for the JSON RPC API server. | -| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | -| `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | -| `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | -| `--coinbase` | (required) | Coinbase address to use for fee collection. | -| `--gas-price` | `1` | Static gas price used for EVM transactions. | -| `--coa-address` | (required) | Flow address that holds COA account used for submitting transactions. | -| `--coa-key` | (required) | *WARNING*: Do not use this flag in production! Private key value for the COA address used for submitting transactions. | -| `--coa-resource-create` | `false` | Auto-create the COA resource in the Flow COA account provided if one doesn't exist. | -| `--log-level` | `debug` | Define verbosity of the log output ('debug', 'info', 'error') | -| `--stream-limit` | 10 | Rate-limits the events sent to the client within one second | -| `--stream-timeout` | 3sec | Defines the timeout in seconds the server waits for the event to be sent to the client | - +| Flag | Default Value | Description | +|-----------------------------|------------------|------------------------------------------------------------------------------------------------------------------------| +| `--database-dir` | `./db` | Path to the directory for the database. | +| `--rpc-host` | `localhost` | Host for the JSON RPC API server. | +| `--rpc-port` | `8545` | Port for the JSON RPC API server. | +| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | +| `--access-node-spork-hosts` | | Previous spork AN hosts, defined following the schema: {latest height}@{host} as comma separated list | +| `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | +| `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | +| `--coinbase` | (required) | Coinbase address to use for fee collection. | +| `--init-cadence-height` | 0 | Define the Cadence block height at which to start the indexing. | +| `--gas-price` | `1` | Static gas price used for EVM transactions. | +| `--coa-address` | (required) | Flow address that holds COA account used for submitting transactions. | +| `--coa-key` | (required) | *WARNING*: Do not use this flag in production! Private key value for the COA address used for submitting transactions. | +| `--coa-key-file` | | File path that contains JSON array of COA keys used in key-rotation mechanism, this is exclusive with `coa-key` flag. | +| `--coa-resource-create` | `false` | Auto-create the COA resource in the Flow COA account provided if one doesn't exist. | +| `--log-level` | `debug` | Define verbosity of the log output ('debug', 'info', 'error') | +| `--stream-limit` | 10 | Rate-limits the events sent to the client within one second | +| `--stream-timeout` | 3sec | Defines the timeout in seconds the server waits for the event to be sent to the client | +| `--filter-expiry` | `5m` | Filter defines the time it takes for an idle filter to expire | ## Getting Started From aee635cdb0618b691cec9eeb06edeb3ec87167a1 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:40:10 +0200 Subject: [PATCH 14/58] add example --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6b17cf349..236d1832e 100644 --- a/README.md +++ b/README.md @@ -90,26 +90,26 @@ it should return: The application can be configured using the following flags at runtime: -| Flag | Default Value | Description | -|-----------------------------|------------------|------------------------------------------------------------------------------------------------------------------------| -| `--database-dir` | `./db` | Path to the directory for the database. | -| `--rpc-host` | `localhost` | Host for the JSON RPC API server. | -| `--rpc-port` | `8545` | Port for the JSON RPC API server. | -| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | -| `--access-node-spork-hosts` | | Previous spork AN hosts, defined following the schema: {latest height}@{host} as comma separated list | -| `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | -| `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | -| `--coinbase` | (required) | Coinbase address to use for fee collection. | -| `--init-cadence-height` | 0 | Define the Cadence block height at which to start the indexing. | -| `--gas-price` | `1` | Static gas price used for EVM transactions. | -| `--coa-address` | (required) | Flow address that holds COA account used for submitting transactions. | -| `--coa-key` | (required) | *WARNING*: Do not use this flag in production! Private key value for the COA address used for submitting transactions. | -| `--coa-key-file` | | File path that contains JSON array of COA keys used in key-rotation mechanism, this is exclusive with `coa-key` flag. | -| `--coa-resource-create` | `false` | Auto-create the COA resource in the Flow COA account provided if one doesn't exist. | -| `--log-level` | `debug` | Define verbosity of the log output ('debug', 'info', 'error') | -| `--stream-limit` | 10 | Rate-limits the events sent to the client within one second | -| `--stream-timeout` | 3sec | Defines the timeout in seconds the server waits for the event to be sent to the client | -| `--filter-expiry` | `5m` | Filter defines the time it takes for an idle filter to expire | +| Flag | Default Value | Description | +|-----------------------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `--database-dir` | `./db` | Path to the directory for the database. | +| `--rpc-host` | `localhost` | Host for the JSON RPC API server. | +| `--rpc-port` | `8545` | Port for the JSON RPC API server. | +| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | +| `--access-node-spork-hosts` | | Previous spork AN hosts, defined following the schema: `{latest height}@{host}` as comma separated list (e.g. "200@host-1.com,300@host2.com") | +| `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | +| `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | +| `--coinbase` | (required) | Coinbase address to use for fee collection. | +| `--init-cadence-height` | 0 | Define the Cadence block height at which to start the indexing. | +| `--gas-price` | `1` | Static gas price used for EVM transactions. | +| `--coa-address` | (required) | Flow address that holds COA account used for submitting transactions. | +| `--coa-key` | (required) | *WARNING*: Do not use this flag in production! Private key value for the COA address used for submitting transactions. | +| `--coa-key-file` | | File path that contains JSON array of COA keys used in key-rotation mechanism, this is exclusive with `coa-key` flag. | +| `--coa-resource-create` | `false` | Auto-create the COA resource in the Flow COA account provided if one doesn't exist. | +| `--log-level` | `debug` | Define verbosity of the log output ('debug', 'info', 'error') | +| `--stream-limit` | 10 | Rate-limits the events sent to the client within one second | +| `--stream-timeout` | 3sec | Defines the timeout in seconds the server waits for the event to be sent to the client | +| `--filter-expiry` | `5m` | Filter defines the time it takes for an idle filter to expire | ## Getting Started From 8479fcb728af41afc7ea68bfb85b5dbc9060de53 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:41:08 +0200 Subject: [PATCH 15/58] add example --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 236d1832e..5b63deba8 100644 --- a/README.md +++ b/README.md @@ -90,26 +90,26 @@ it should return: The application can be configured using the following flags at runtime: -| Flag | Default Value | Description | -|-----------------------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| `--database-dir` | `./db` | Path to the directory for the database. | -| `--rpc-host` | `localhost` | Host for the JSON RPC API server. | -| `--rpc-port` | `8545` | Port for the JSON RPC API server. | -| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | -| `--access-node-spork-hosts` | | Previous spork AN hosts, defined following the schema: `{latest height}@{host}` as comma separated list (e.g. "200@host-1.com,300@host2.com") | -| `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | -| `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | -| `--coinbase` | (required) | Coinbase address to use for fee collection. | -| `--init-cadence-height` | 0 | Define the Cadence block height at which to start the indexing. | -| `--gas-price` | `1` | Static gas price used for EVM transactions. | -| `--coa-address` | (required) | Flow address that holds COA account used for submitting transactions. | -| `--coa-key` | (required) | *WARNING*: Do not use this flag in production! Private key value for the COA address used for submitting transactions. | -| `--coa-key-file` | | File path that contains JSON array of COA keys used in key-rotation mechanism, this is exclusive with `coa-key` flag. | -| `--coa-resource-create` | `false` | Auto-create the COA resource in the Flow COA account provided if one doesn't exist. | -| `--log-level` | `debug` | Define verbosity of the log output ('debug', 'info', 'error') | -| `--stream-limit` | 10 | Rate-limits the events sent to the client within one second | -| `--stream-timeout` | 3sec | Defines the timeout in seconds the server waits for the event to be sent to the client | -| `--filter-expiry` | `5m` | Filter defines the time it takes for an idle filter to expire | +| Flag | Default Value | Description | +|-----------------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| `--database-dir` | `./db` | Path to the directory for the database. | +| `--rpc-host` | `localhost` | Host for the JSON RPC API server. | +| `--rpc-port` | `8545` | Port for the JSON RPC API server. | +| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | +| `--access-node-spork-hosts` | | Previous spork AN hosts, defined following the schema: `{latest height}@{host}` as comma separated list (e.g. `"200@host-1.com,300@host2.com"`) | +| `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | +| `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | +| `--coinbase` | (required) | Coinbase address to use for fee collection. | +| `--init-cadence-height` | 0 | Define the Cadence block height at which to start the indexing. | +| `--gas-price` | `1` | Static gas price used for EVM transactions. | +| `--coa-address` | (required) | Flow address that holds COA account used for submitting transactions. | +| `--coa-key` | (required) | *WARNING*: Do not use this flag in production! Private key value for the COA address used for submitting transactions. | +| `--coa-key-file` | | File path that contains JSON array of COA keys used in key-rotation mechanism, this is exclusive with `coa-key` flag. | +| `--coa-resource-create` | `false` | Auto-create the COA resource in the Flow COA account provided if one doesn't exist. | +| `--log-level` | `debug` | Define verbosity of the log output ('debug', 'info', 'error') | +| `--stream-limit` | 10 | Rate-limits the events sent to the client within one second | +| `--stream-timeout` | 3sec | Defines the timeout in seconds the server waits for the event to be sent to the client | +| `--filter-expiry` | `5m` | Filter defines the time it takes for an idle filter to expire | ## Getting Started From d068c593dca747acfa7f684a83d4e2dc15ceaed2 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 6 May 2024 18:41:51 +0200 Subject: [PATCH 16/58] improve docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b63deba8..348e574a7 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ The application can be configured using the following flags at runtime: | `--database-dir` | `./db` | Path to the directory for the database. | | `--rpc-host` | `localhost` | Host for the JSON RPC API server. | | `--rpc-port` | `8545` | Port for the JSON RPC API server. | -| `--access-node-grpc-host` | `localhost:3569` | Host to the Flow access node (AN) gRPC API. | +| `--access-node-grpc-host` | `localhost:3569` | Host to the current spork Flow access node (AN) gRPC API. | | `--access-node-spork-hosts` | | Previous spork AN hosts, defined following the schema: `{latest height}@{host}` as comma separated list (e.g. `"200@host-1.com,300@host2.com"`) | | `--evm-network-id` | `testnet` | EVM network ID (options: `testnet`, `mainnet`). | | `--flow-network-id` | `emulator` | Flow network ID (options: `emulator`, `previewnet`). | From 1874e826b1769308668b697999fc21d98fb782f2 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 13:08:10 +0200 Subject: [PATCH 17/58] add past spork method --- models/cross-spork_client.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 585b2e2b2..c632119b0 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -44,6 +44,7 @@ func NewCrossSporkClient(currentSporkHost string, logger zerolog.Logger) (*Cross }, nil } +// AddSpork will add a new spork host defined by the last height boundary in that spork. func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { if _, ok := c.sporkHosts[lastHeight]; ok { return fmt.Errorf("provided last height already exists") @@ -64,6 +65,11 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { return nil } +// IsPastSpork will check if the provided height is contained in the previous sporks. +func (c *CrossSporkClient) IsPastSpork(height uint64) bool { + return height <= c.getSporkBoundariesDesc()[0] +} + // getClientForHeight returns the client for the given height. It starts by using the current spork client, // then iteratively checks the upper height boundaries in descending order and returns the last client // that still contains the given height within its upper height limit. If no client is found, it returns @@ -73,14 +79,11 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { // lower height boundaries that we didn't configure for. // This would result in the error when using the client to access such data. func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { - heights := maps.Keys(c.sporkHosts) - slices.Sort(heights) // order heights in ascending order - slices.Reverse(heights) // make it descending // start by using the current spork client, then iterate all the upper height boundaries // and find the last client that still contains the height in its upper height limit client := c.Client - for _, upperBound := range heights { + for _, upperBound := range c.getSporkBoundariesDesc() { if upperBound >= height { client = c.sporkHosts[upperBound] @@ -93,6 +96,14 @@ func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { return client } +// getSporkBoundaries will return descending order of spork height boundaries +func (c *CrossSporkClient) getSporkBoundariesDesc() []uint64 { + heights := maps.Keys(c.sporkHosts) + slices.Sort(heights) // order heights in ascending order + slices.Reverse(heights) // make it descending + return heights +} + func (c *CrossSporkClient) GetBlockByHeight( ctx context.Context, height uint64, From 156026d312f5d4641689fadb55e8db0a8add18c8 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 15:50:13 +0200 Subject: [PATCH 18/58] change the client to base client with options --- models/cross-spork_client.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index c632119b0..a993d45db 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -5,13 +5,16 @@ import ( "fmt" "github.com/onflow/cadence" "github.com/onflow/flow-go-sdk" - "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-go-sdk/access/grpc" "github.com/rs/zerolog" "golang.org/x/exp/maps" "golang.org/x/exp/slices" + gogrpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) +var dialOpts = gogrpc.WithTransportCredentials(insecure.NewCredentials()) + // CrossSporkClient is a wrapper around the Flow AN client that can // access different AN APIs based on the height boundaries of the sporks. // @@ -23,23 +26,23 @@ import ( type CrossSporkClient struct { logger zerolog.Logger // this map holds the last heights and clients for each spork - sporkHosts map[uint64]access.Client + sporkHosts map[uint64]*grpc.BaseClient - access.Client + *grpc.BaseClient } // NewCrossSporkClient creates a new instance of the client, it accepts the // host to the current spork AN API. func NewCrossSporkClient(currentSporkHost string, logger zerolog.Logger) (*CrossSporkClient, error) { // add current spork AN host as the default client - client, err := grpc.NewClient(currentSporkHost) + client, err := grpc.NewBaseClient(currentSporkHost, dialOpts) if err != nil { return nil, err } return &CrossSporkClient{ logger, - make(map[uint64]access.Client), + make(map[uint64]*grpc.BaseClient), client, }, nil } @@ -50,7 +53,7 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { return fmt.Errorf("provided last height already exists") } - client, err := grpc.NewClient(host) + client, err := grpc.NewBaseClient(host, dialOpts) if err != nil { return err } @@ -78,11 +81,11 @@ func (c *CrossSporkClient) IsPastSpork(height uint64) bool { // because it still might not have access to the height provided, because there might be other sporks with // lower height boundaries that we didn't configure for. // This would result in the error when using the client to access such data. -func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { +func (c *CrossSporkClient) getClientForHeight(height uint64) *grpc.BaseClient { // start by using the current spork client, then iterate all the upper height boundaries // and find the last client that still contains the height in its upper height limit - client := c.Client + client := c.BaseClient for _, upperBound := range c.getSporkBoundariesDesc() { if upperBound >= height { client = c.sporkHosts[upperBound] @@ -128,8 +131,9 @@ func (c *CrossSporkClient) SubscribeEventsByBlockHeight( ctx context.Context, startHeight uint64, filter flow.EventFilter, + opts ...grpc.SubscribeOption, ) (<-chan flow.BlockEvents, <-chan error, error) { return c. getClientForHeight(startHeight). - SubscribeEventsByBlockHeight(ctx, startHeight, filter) + SubscribeEventsByBlockHeight(ctx, startHeight, filter, opts...) } From 5d7a81e73ad94cbf78518af3a731394b1fb13378 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 15:50:21 +0200 Subject: [PATCH 19/58] define new block event type --- models/events.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/events.go b/models/events.go index 942d4bb45..c87d7e4d7 100644 --- a/models/events.go +++ b/models/events.go @@ -107,3 +107,10 @@ func (c *CadenceEvents) CadenceHeight() uint64 { func (c *CadenceEvents) Length() int { return len(c.events.Events) } + +// BlockEvents is a wrapper around events streamed, and it also contains an error +type BlockEvents struct { + Events *flow.BlockEvents + Err error + Done bool +} From 44078d85a084d8eb060bf17db80371ffa491e683 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 15:56:03 +0200 Subject: [PATCH 20/58] introduce backfilling ability --- services/ingestion/subscriber.go | 198 +++++++++++++++++++++++++++---- 1 file changed, 173 insertions(+), 25 deletions(-) diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 088452d29..cb99519ea 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -2,55 +2,203 @@ package ingestion import ( "context" + "errors" "fmt" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/flow-evm-gateway/models" "github.com/onflow/flow-go-sdk" - "github.com/onflow/flow-go-sdk/access" + "github.com/onflow/flow-go-sdk/access/grpc" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/fvm/systemcontracts" flowGo "github.com/onflow/flow-go/model/flow" + "github.com/rs/zerolog" ) type EventSubscriber interface { - // Subscribe to relevant events from the provided block height. - // Returns a channel with block events and errors, - // if subscription fails returns an error as the third value. - Subscribe(ctx context.Context, height uint64) (<-chan flow.BlockEvents, <-chan error, error) + // Subscribe to EVM events from the provided height, and return a chanel with the events. + // + // The BlockEvents type will contain an optional error in case + // the error happens, the consumer of the chanel should handle it. + Subscribe(ctx context.Context, height uint64) <-chan models.BlockEvents } +var _ EventSubscriber = &RPCSubscriber{} + type RPCSubscriber struct { - client access.Client + client *models.CrossSporkClient chain flowGo.ChainID + logger zerolog.Logger } -func NewRPCSubscriber(client access.Client, chainID flowGo.ChainID) *RPCSubscriber { - return &RPCSubscriber{client: client, chain: chainID} +func NewRPCSubscriber(client *models.CrossSporkClient, chainID flowGo.ChainID, logger zerolog.Logger) *RPCSubscriber { + return &RPCSubscriber{client: client, chain: chainID, logger: logger} } -func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) (<-chan flow.BlockEvents, <-chan error, error) { - // define events we subscribe to: A.{evm}.EVM.BlockExecuted and A.{evm}.EVM.TransactionExecuted, - // where {evm} is EVM deployed contract address, which depends on the chain ID we configure - evmAddress := common.Address(systemcontracts.SystemContractsForChain(r.chain).EVMContract.Address) - blockExecutedEvent := common.NewAddressLocation(nil, evmAddress, string(types.EventTypeBlockExecuted)).ID() - transactionExecutedEvent := common.NewAddressLocation(nil, evmAddress, string(types.EventTypeTransactionExecuted)).ID() +// Subscribe will retrieve all the events from the provided height. If the height is from previous +// sporks, it will first backfill all the events in all the previous sporks, and then continue +// to listen all new events in the current spork. +// +// If error is encountered during backfill the subscription will end and the response chanel will be closed. +func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan models.BlockEvents { + events := make(chan models.BlockEvents) - filter := flow.EventFilter{ - EventTypes: []string{ - blockExecutedEvent, - transactionExecutedEvent, - }, - } + go func() { + defer func() { + close(events) + }() + + // if the height is from the previous spork, backfill all the events from previous sporks first + if r.client.IsPastSpork(height) { + for ev := range r.backfill(ctx, height) { + // keep updating height, so after we are done back-filling it will be at the + // first height in the current spork + if ev.Events != nil { + height = ev.Events.CadenceHeight() + } + + events <- ev + + if ev.Err != nil { + return + } + } + + // after back-filling is done, increment height by one, + // so we start with the height in the current spork + height = height + 1 + } + + // subscribe in the current spork + for ev := range r.subscribe(ctx, height) { + events <- ev + } + }() + + return events +} + +// subscribe to events by the provided height and handle any errors. +func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...grpc.SubscribeOption) <-chan models.BlockEvents { + events := make(chan models.BlockEvents) _, err := r.client.GetBlockByHeight(ctx, height) if err != nil { - return nil, nil, fmt.Errorf("failed to subscribe for events, the block height %d doesn't exist: %w", height, err) + err = fmt.Errorf("failed to subscribe for events, the block height %d doesn't exist: %w", height, err) + events <- models.NewBlockEventsError(err) + return events } - // todo revisit if we should use custom heartbeat interval grpc.WithHeartbeatInterval(1) - evs, errs, err := r.client.SubscribeEventsByBlockHeight(ctx, height, filter) + evs, errs, err := r.client.SubscribeEventsByBlockHeight(ctx, height, r.blocksFilter(), opts...) if err != nil { - return nil, nil, fmt.Errorf("failed to subscribe to events by block height: %w", err) + events <- models.NewBlockEventsError(fmt.Errorf("failed to subscribe to events by block height: %w", err)) + return events } - return evs, errs, nil + go func() { + defer func() { + close(events) + }() + + for { + select { + case <-ctx.Done(): + r.logger.Info().Msg("event ingestion received done signal") + return + + case blockEvents, ok := <-evs: + if !ok { + var err error + err = models.ErrDisconnected + if ctx.Err() != nil { + err = ctx.Err() + } + events <- models.NewBlockEventsError(err) + return + } + + events <- models.NewBlockEvents(blockEvents) + + case err, ok := <-errs: + if !ok { + var err error + err = models.ErrDisconnected + if ctx.Err() != nil { + err = ctx.Err() + } + events <- models.NewBlockEventsError(err) + return + } + + events <- models.NewBlockEventsError(errors.Join(err, models.ErrDisconnected)) + return + } + } + }() + + return events +} + +// backfill will use the provided height and with the client for the provided spork will start backfilling +// events. Before subscribing, it will check what is the latest block in the current spork (defined by height) +// and check for each event it receives whether we reached the end, if we reach the end it will increase +// the height by one (next height), and check if we are still in previous sporks, if so repeat everything, +// otherwise return. +func (r *RPCSubscriber) backfill(ctx context.Context, height uint64) <-chan models.BlockEvents { + events := make(chan models.BlockEvents) + + go func() { + defer func() { + close(events) + }() + + for { + // check if the current height is still in past sporks, and if not return since we are done with backfilling + if !r.client.IsPastSpork(height) { + return + } + + latestBlock, err := r.client.GetLatestBlock(ctx, true) + if err != nil { + events <- models.NewBlockEventsError(err) + return + } + + for ev := range r.subscribe(ctx, height, grpc.WithHeartbeatInterval(1)) { + events <- ev + + if ev.Events.CadenceHeight() == latestBlock.Height { + height = ev.Events.CadenceHeight() + 1 // go to next height in the next spork + break + } + } + } + }() + + return events +} + +// blockFilter define events we subscribe to: +// A.{evm}.EVM.BlockExecuted and A.{evm}.EVM.TransactionExecuted, +// where {evm} is EVM deployed contract address, which depends on the chain ID we configure. +func (r *RPCSubscriber) blocksFilter() flow.EventFilter { + evmAddress := common.Address(systemcontracts.SystemContractsForChain(r.chain).EVMContract.Address) + + blockExecutedEvent := common.NewAddressLocation( + nil, + evmAddress, + string(types.EventTypeBlockExecuted), + ).ID() + + transactionExecutedEvent := common.NewAddressLocation( + nil, + evmAddress, + string(types.EventTypeTransactionExecuted), + ).ID() + + return flow.EventFilter{ + EventTypes: []string{ + blockExecutedEvent, + transactionExecutedEvent, + }, + } } From 819a84dbcf09046ac3b87113ecc5235eb256e050 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 15:56:20 +0200 Subject: [PATCH 21/58] add factories for block events --- models/events.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/models/events.go b/models/events.go index c87d7e4d7..48b843076 100644 --- a/models/events.go +++ b/models/events.go @@ -110,7 +110,18 @@ func (c *CadenceEvents) Length() int { // BlockEvents is a wrapper around events streamed, and it also contains an error type BlockEvents struct { - Events *flow.BlockEvents + Events *CadenceEvents Err error - Done bool +} + +func NewBlockEvents(events flow.BlockEvents) BlockEvents { + return BlockEvents{ + Events: NewCadenceEvents(events), + } +} + +func NewBlockEventsError(err error) BlockEvents { + return BlockEvents{ + Err: err, + } } From 969e9b5735055e9df9e0ce35fca204d7903a8b16 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:07:21 +0200 Subject: [PATCH 22/58] add logging --- services/ingestion/subscriber.go | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index cb99519ea..09c198ce0 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -31,6 +31,7 @@ type RPCSubscriber struct { } func NewRPCSubscriber(client *models.CrossSporkClient, chainID flowGo.ChainID, logger zerolog.Logger) *RPCSubscriber { + logger = logger.With().Str("component", "subscriber").Logger() return &RPCSubscriber{client: client, chain: chainID, logger: logger} } @@ -49,6 +50,10 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod // if the height is from the previous spork, backfill all the events from previous sporks first if r.client.IsPastSpork(height) { + r.logger.Info(). + Uint64("height", height). + Msg("height found in previous spork, starting to backfill") + for ev := range r.backfill(ctx, height) { // keep updating height, so after we are done back-filling it will be at the // first height in the current spork @@ -68,6 +73,10 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod height = height + 1 } + r.logger.Info(). + Uint64("next-height", height). + Msg("backfilling done, subscribe for live data") + // subscribe in the current spork for ev := range r.subscribe(ctx, height) { events <- ev @@ -154,20 +163,36 @@ func (r *RPCSubscriber) backfill(ctx context.Context, height uint64) <-chan mode for { // check if the current height is still in past sporks, and if not return since we are done with backfilling if !r.client.IsPastSpork(height) { + r.logger.Info(). + Uint64("height", height). + Msg("completed backfilling") + return } - latestBlock, err := r.client.GetLatestBlock(ctx, true) + latestHeight, err := r.client.GetLatestHeightForSpork(ctx, height) if err != nil { events <- models.NewBlockEventsError(err) return } + r.logger.Info(). + Uint64("start-height", height). + Uint64("last-spork-height", latestHeight). + Msg("backfilling spork") + for ev := range r.subscribe(ctx, height, grpc.WithHeartbeatInterval(1)) { events <- ev - if ev.Events.CadenceHeight() == latestBlock.Height { + r.logger.Debug().Msg(fmt.Sprintf("backfilling [%d / %d]...", ev.Events.CadenceHeight(), latestHeight)) + + if ev.Events.CadenceHeight() == latestHeight { height = ev.Events.CadenceHeight() + 1 // go to next height in the next spork + + r.logger.Info(). + Uint64("next-height", height). + Msg("reached the end of spork, checking next spork") + break } } From 0e08f32f1289d5ac148f4e365d89705cf2b49348 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:07:30 +0200 Subject: [PATCH 23/58] get latest height in the spork --- models/cross-spork_client.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index a993d45db..72c857410 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -107,6 +107,19 @@ func (c *CrossSporkClient) getSporkBoundariesDesc() []uint64 { return heights } +// GetLatestHeightForSpork will determine the spork client in which the provided height is contained +// and then find the latest height in that spork. +func (c *CrossSporkClient) GetLatestHeightForSpork(ctx context.Context, height uint64) (uint64, error) { + block, err := c. + getClientForHeight(height). + GetLatestBlock(ctx, true) + if err != nil { + return 0, err + } + + return block.Height, nil +} + func (c *CrossSporkClient) GetBlockByHeight( ctx context.Context, height uint64, From ccfb9e4cd21e4627dd35ae1c637e94cfba86e7a7 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:07:35 +0200 Subject: [PATCH 24/58] add logger --- bootstrap/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 3c1e102bb..25359218a 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -133,7 +133,7 @@ func startIngestion( Uint64("missed-heights", blk.Height-latestCadenceHeight). Msg("indexing cadence height information") - subscriber := ingestion.NewRPCSubscriber(client, cfg.FlowNetworkID) + subscriber := ingestion.NewRPCSubscriber(client, cfg.FlowNetworkID, logger) engine := ingestion.NewEventIngestionEngine( subscriber, blocks, From 8dba9c25e84271a6a167186b036ac962a727848c Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:12:33 +0200 Subject: [PATCH 25/58] update ingestion engine with subscriber api change --- services/ingestion/engine.go | 56 +++++++++--------------------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/services/ingestion/engine.go b/services/ingestion/engine.go index 6af993057..57f285b59 100644 --- a/services/ingestion/engine.go +++ b/services/ingestion/engine.go @@ -2,12 +2,10 @@ package ingestion import ( "context" - "errors" "fmt" "github.com/onflow/flow-evm-gateway/models" "github.com/onflow/flow-evm-gateway/storage" - "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/fvm/evm/types" gethTypes "github.com/onflow/go-ethereum/core/types" @@ -96,45 +94,21 @@ func (e *Engine) Run(ctx context.Context) error { e.log.Info().Uint64("start-cadence-height", latestCadence).Msg("starting ingestion") - events, errs, err := e.subscriber.Subscribe(ctx, latestCadence) - if err != nil { - return fmt.Errorf("failed to subscribe to events: %w", err) - } - e.status.MarkReady() - for { - select { - case <-ctx.Done(): - e.log.Info().Msg("event ingestion received done signal") - return nil - - case blockEvents, ok := <-events: - if !ok { - if ctx.Err() != nil { - return ctx.Err() - } - return models.ErrDisconnected - } - - err = e.processEvents(blockEvents) - if err != nil { - e.log.Error().Err(err).Msg("failed to process EVM events") - return err - } - - case err, ok := <-errs: - if !ok { - if ctx.Err() != nil { - return ctx.Err() - } - - return models.ErrDisconnected - } - - return errors.Join(err, models.ErrDisconnected) + for events := range e.subscriber.Subscribe(ctx, latestCadence) { + if events.Err != nil { + return fmt.Errorf("failure in event subscription: %w", events.Err) + } + + err = e.processEvents(events.Events) + if err != nil { + e.log.Error().Err(err).Msg("failed to process EVM events") + return err } } + + return nil } // processEvents converts the events to block and transactions and indexes them. @@ -149,14 +123,12 @@ func (e *Engine) Run(ctx context.Context) error { // https://github.com/onflow/flow-go/blob/master/fvm/evm/types/events.go // // Any error is unexpected and fatal. -func (e *Engine) processEvents(blockEvents flow.BlockEvents) error { +func (e *Engine) processEvents(events *models.CadenceEvents) error { e.log.Debug(). - Uint64("cadence-height", blockEvents.Height). - Int("cadence-event-length", len(blockEvents.Events)). + Uint64("cadence-height", events.CadenceHeight()). + Int("cadence-event-length", events.Length()). Msg("received new cadence evm events") - events := models.NewCadenceEvents(blockEvents) - // if heartbeat interval with no data still update the cadence height if events.Empty() { if err := e.blocks.SetLatestCadenceHeight(events.CadenceHeight()); err != nil { From b6a680a5477b40515acb55dfd7665895546248d8 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:16:38 +0200 Subject: [PATCH 26/58] improve error handling --- config/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.go b/config/config.go index e70486c77..d54c3d4ba 100644 --- a/config/config.go +++ b/config/config.go @@ -193,6 +193,9 @@ func FromFlags() (*Config, error) { heightHosts := strings.Split(accessSporkHosts, ",") for _, hh := range heightHosts { v := strings.Split(hh, "@") + if len(v) != 2 { + return nil, fmt.Errorf("failed to parse AN spork value provided with --access-node-spork-hosts flag") + } heightVal, host := v[0], v[1] height, err := strconv.Atoi(heightVal) if err != nil { From b182d07a4ddc73cd813845cc0c709d85afe9c4b9 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:57:51 +0200 Subject: [PATCH 27/58] improve err handling --- models/cross-spork_client.go | 1 - services/ingestion/subscriber.go | 16 +++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 72c857410..2029435dd 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -27,7 +27,6 @@ type CrossSporkClient struct { logger zerolog.Logger // this map holds the last heights and clients for each spork sporkHosts map[uint64]*grpc.BaseClient - *grpc.BaseClient } diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 09c198ce0..714d489e3 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -55,17 +55,15 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod Msg("height found in previous spork, starting to backfill") for ev := range r.backfill(ctx, height) { - // keep updating height, so after we are done back-filling it will be at the - // first height in the current spork - if ev.Events != nil { - height = ev.Events.CadenceHeight() - } - events <- ev if ev.Err != nil { return } + + // keep updating height, so after we are done back-filling + // it will be at the first height in the current spork + height = ev.Events.CadenceHeight() } // after back-filling is done, increment height by one, @@ -184,9 +182,13 @@ func (r *RPCSubscriber) backfill(ctx context.Context, height uint64) <-chan mode for ev := range r.subscribe(ctx, height, grpc.WithHeartbeatInterval(1)) { events <- ev + if ev.Err != nil { + return + } + r.logger.Debug().Msg(fmt.Sprintf("backfilling [%d / %d]...", ev.Events.CadenceHeight(), latestHeight)) - if ev.Events.CadenceHeight() == latestHeight { + if ev.Events != nil && ev.Events.CadenceHeight() == latestHeight { height = ev.Events.CadenceHeight() + 1 // go to next height in the next spork r.logger.Info(). From d55ac591fafe3637d3be4717bb82b414cd66a761 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 16:59:09 +0200 Subject: [PATCH 28/58] add log --- services/ingestion/subscriber.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 714d489e3..65d5d3b26 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -79,6 +79,8 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod for ev := range r.subscribe(ctx, height) { events <- ev } + + r.logger.Warn().Msg("ended subscription for events") }() return events From 1556554dd26037fdf0359541cea56034b072aa7e Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:02:00 +0200 Subject: [PATCH 29/58] update mocks --- go.mod | 6 ++++ go.sum | 9 ++++++ models/mocks/Engine.go | 23 +++++++++++---- storage/mocks/AccountIndexer.go | 23 +++++++++++---- storage/mocks/BlockIndexer.go | 43 +++++++++++++++++++++++++---- storage/mocks/ReceiptIndexer.go | 27 ++++++++++++++---- storage/mocks/TransactionIndexer.go | 19 +++++++++---- 7 files changed, 120 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 860923b9e..c0fb4e4d3 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chigopher/pathlib v0.12.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.3 // indirect @@ -84,6 +85,7 @@ require ( github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/goprocess v0.1.4 // indirect + github.com/jinzhu/copier v0.3.5 // indirect github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/kevinburke/go-bindata v3.24.0+incompatible // indirect github.com/klauspost/compress v1.17.4 // indirect @@ -99,6 +101,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -155,6 +158,7 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect + github.com/vektra/mockery/v2 v2.21.4 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v4 v4.3.11 // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect @@ -174,6 +178,7 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect @@ -185,6 +190,7 @@ require ( google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 203ba42ec..48fd07bae 100644 --- a/go.sum +++ b/go.sum @@ -1078,6 +1078,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chigopher/pathlib v0.12.0 h1:1GM7fN/IwXXmOHbd1jkMqHD2wUhYqUvafgxTwmLT/q8= +github.com/chigopher/pathlib v0.12.0/go.mod h1:EJ5UtJ/sK8Nt6q3VWN+EwZLZ3g0afJiG8NegYiQQ/gQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -1627,6 +1629,8 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -1806,6 +1810,7 @@ github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dz github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -2051,6 +2056,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.4.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= @@ -2137,6 +2143,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vektra/mockery/v2 v2.21.4 h1:QInaL4ma4Bz/cVipKeBLggB5XyPenpksiz8sJpgBZSE= +github.com/vektra/mockery/v2 v2.21.4/go.mod h1:le9nnD50TkeV8QGwrkza7v/xd3DA7YbSZYBEyiuAlsI= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.11 h1:Q47CePddpNGNhk4GCnAx9DDtASi2rasatE0cd26cZoE= @@ -2624,6 +2632,7 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/models/mocks/Engine.go b/models/mocks/Engine.go index 2954041b2..2127b2941 100644 --- a/models/mocks/Engine.go +++ b/models/mocks/Engine.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks @@ -17,6 +17,10 @@ type Engine struct { func (_m *Engine) Done() <-chan struct{} { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Done") + } + var r0 <-chan struct{} if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { r0 = rf() @@ -33,6 +37,10 @@ func (_m *Engine) Done() <-chan struct{} { func (_m *Engine) Ready() <-chan struct{} { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Ready") + } + var r0 <-chan struct{} if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { r0 = rf() @@ -49,6 +57,10 @@ func (_m *Engine) Ready() <-chan struct{} { func (_m *Engine) Run(ctx context.Context) error { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for Run") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) @@ -64,13 +76,12 @@ func (_m *Engine) Stop() { _m.Called() } -type mockConstructorTestingTNewEngine interface { +// NewEngine creates a new instance of Engine. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEngine(t interface { mock.TestingT Cleanup(func()) -} - -// NewEngine creates a new instance of Engine. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewEngine(t mockConstructorTestingTNewEngine) *Engine { +}) *Engine { mock := &Engine{} mock.Mock.Test(t) diff --git a/storage/mocks/AccountIndexer.go b/storage/mocks/AccountIndexer.go index fe2419141..e949b1ee9 100644 --- a/storage/mocks/AccountIndexer.go +++ b/storage/mocks/AccountIndexer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks @@ -22,6 +22,10 @@ type AccountIndexer struct { func (_m *AccountIndexer) GetBalance(address *common.Address) (*big.Int, error) { ret := _m.Called(address) + if len(ret) == 0 { + panic("no return value specified for GetBalance") + } + var r0 *big.Int var r1 error if rf, ok := ret.Get(0).(func(*common.Address) (*big.Int, error)); ok { @@ -48,6 +52,10 @@ func (_m *AccountIndexer) GetBalance(address *common.Address) (*big.Int, error) func (_m *AccountIndexer) GetNonce(address *common.Address) (uint64, error) { ret := _m.Called(address) + if len(ret) == 0 { + panic("no return value specified for GetNonce") + } + var r0 uint64 var r1 error if rf, ok := ret.Get(0).(func(*common.Address) (uint64, error)); ok { @@ -72,6 +80,10 @@ func (_m *AccountIndexer) GetNonce(address *common.Address) (uint64, error) { func (_m *AccountIndexer) Update(tx models.Transaction, receipt *types.Receipt) error { ret := _m.Called(tx, receipt) + if len(ret) == 0 { + panic("no return value specified for Update") + } + var r0 error if rf, ok := ret.Get(0).(func(models.Transaction, *types.Receipt) error); ok { r0 = rf(tx, receipt) @@ -82,13 +94,12 @@ func (_m *AccountIndexer) Update(tx models.Transaction, receipt *types.Receipt) return r0 } -type mockConstructorTestingTNewAccountIndexer interface { +// NewAccountIndexer creates a new instance of AccountIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAccountIndexer(t interface { mock.TestingT Cleanup(func()) -} - -// NewAccountIndexer creates a new instance of AccountIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewAccountIndexer(t mockConstructorTestingTNewAccountIndexer) *AccountIndexer { +}) *AccountIndexer { mock := &AccountIndexer{} mock.Mock.Test(t) diff --git a/storage/mocks/BlockIndexer.go b/storage/mocks/BlockIndexer.go index ee82d7cde..9a77ee5f6 100644 --- a/storage/mocks/BlockIndexer.go +++ b/storage/mocks/BlockIndexer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks @@ -18,6 +18,10 @@ type BlockIndexer struct { func (_m *BlockIndexer) GetByHeight(height uint64) (*types.Block, error) { ret := _m.Called(height) + if len(ret) == 0 { + panic("no return value specified for GetByHeight") + } + var r0 *types.Block var r1 error if rf, ok := ret.Get(0).(func(uint64) (*types.Block, error)); ok { @@ -44,6 +48,10 @@ func (_m *BlockIndexer) GetByHeight(height uint64) (*types.Block, error) { func (_m *BlockIndexer) GetByID(ID common.Hash) (*types.Block, error) { ret := _m.Called(ID) + if len(ret) == 0 { + panic("no return value specified for GetByID") + } + var r0 *types.Block var r1 error if rf, ok := ret.Get(0).(func(common.Hash) (*types.Block, error)); ok { @@ -70,6 +78,10 @@ func (_m *BlockIndexer) GetByID(ID common.Hash) (*types.Block, error) { func (_m *BlockIndexer) GetCadenceHeight(evmHeight uint64) (uint64, error) { ret := _m.Called(evmHeight) + if len(ret) == 0 { + panic("no return value specified for GetCadenceHeight") + } + var r0 uint64 var r1 error if rf, ok := ret.Get(0).(func(uint64) (uint64, error)); ok { @@ -94,6 +106,10 @@ func (_m *BlockIndexer) GetCadenceHeight(evmHeight uint64) (uint64, error) { func (_m *BlockIndexer) GetHeightByID(ID common.Hash) (uint64, error) { ret := _m.Called(ID) + if len(ret) == 0 { + panic("no return value specified for GetHeightByID") + } + var r0 uint64 var r1 error if rf, ok := ret.Get(0).(func(common.Hash) (uint64, error)); ok { @@ -118,6 +134,10 @@ func (_m *BlockIndexer) GetHeightByID(ID common.Hash) (uint64, error) { func (_m *BlockIndexer) LatestCadenceHeight() (uint64, error) { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for LatestCadenceHeight") + } + var r0 uint64 var r1 error if rf, ok := ret.Get(0).(func() (uint64, error)); ok { @@ -142,6 +162,10 @@ func (_m *BlockIndexer) LatestCadenceHeight() (uint64, error) { func (_m *BlockIndexer) LatestEVMHeight() (uint64, error) { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for LatestEVMHeight") + } + var r0 uint64 var r1 error if rf, ok := ret.Get(0).(func() (uint64, error)); ok { @@ -166,6 +190,10 @@ func (_m *BlockIndexer) LatestEVMHeight() (uint64, error) { func (_m *BlockIndexer) SetLatestCadenceHeight(height uint64) error { ret := _m.Called(height) + if len(ret) == 0 { + panic("no return value specified for SetLatestCadenceHeight") + } + var r0 error if rf, ok := ret.Get(0).(func(uint64) error); ok { r0 = rf(height) @@ -180,6 +208,10 @@ func (_m *BlockIndexer) SetLatestCadenceHeight(height uint64) error { func (_m *BlockIndexer) Store(cadenceHeight uint64, block *types.Block) error { ret := _m.Called(cadenceHeight, block) + if len(ret) == 0 { + panic("no return value specified for Store") + } + var r0 error if rf, ok := ret.Get(0).(func(uint64, *types.Block) error); ok { r0 = rf(cadenceHeight, block) @@ -190,13 +222,12 @@ func (_m *BlockIndexer) Store(cadenceHeight uint64, block *types.Block) error { return r0 } -type mockConstructorTestingTNewBlockIndexer interface { +// NewBlockIndexer creates a new instance of BlockIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewBlockIndexer(t interface { mock.TestingT Cleanup(func()) -} - -// NewBlockIndexer creates a new instance of BlockIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewBlockIndexer(t mockConstructorTestingTNewBlockIndexer) *BlockIndexer { +}) *BlockIndexer { mock := &BlockIndexer{} mock.Mock.Test(t) diff --git a/storage/mocks/ReceiptIndexer.go b/storage/mocks/ReceiptIndexer.go index 638b369b9..1f94595f1 100644 --- a/storage/mocks/ReceiptIndexer.go +++ b/storage/mocks/ReceiptIndexer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks @@ -20,6 +20,10 @@ type ReceiptIndexer struct { func (_m *ReceiptIndexer) BloomsForBlockRange(start *big.Int, end *big.Int) ([]*types.Bloom, []*big.Int, error) { ret := _m.Called(start, end) + if len(ret) == 0 { + panic("no return value specified for BloomsForBlockRange") + } + var r0 []*types.Bloom var r1 []*big.Int var r2 error @@ -55,6 +59,10 @@ func (_m *ReceiptIndexer) BloomsForBlockRange(start *big.Int, end *big.Int) ([]* func (_m *ReceiptIndexer) GetByBlockHeight(height *big.Int) ([]*types.Receipt, error) { ret := _m.Called(height) + if len(ret) == 0 { + panic("no return value specified for GetByBlockHeight") + } + var r0 []*types.Receipt var r1 error if rf, ok := ret.Get(0).(func(*big.Int) ([]*types.Receipt, error)); ok { @@ -81,6 +89,10 @@ func (_m *ReceiptIndexer) GetByBlockHeight(height *big.Int) ([]*types.Receipt, e func (_m *ReceiptIndexer) GetByTransactionID(ID common.Hash) (*types.Receipt, error) { ret := _m.Called(ID) + if len(ret) == 0 { + panic("no return value specified for GetByTransactionID") + } + var r0 *types.Receipt var r1 error if rf, ok := ret.Get(0).(func(common.Hash) (*types.Receipt, error)); ok { @@ -107,6 +119,10 @@ func (_m *ReceiptIndexer) GetByTransactionID(ID common.Hash) (*types.Receipt, er func (_m *ReceiptIndexer) Store(receipt *types.Receipt) error { ret := _m.Called(receipt) + if len(ret) == 0 { + panic("no return value specified for Store") + } + var r0 error if rf, ok := ret.Get(0).(func(*types.Receipt) error); ok { r0 = rf(receipt) @@ -117,13 +133,12 @@ func (_m *ReceiptIndexer) Store(receipt *types.Receipt) error { return r0 } -type mockConstructorTestingTNewReceiptIndexer interface { +// NewReceiptIndexer creates a new instance of ReceiptIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewReceiptIndexer(t interface { mock.TestingT Cleanup(func()) -} - -// NewReceiptIndexer creates a new instance of ReceiptIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewReceiptIndexer(t mockConstructorTestingTNewReceiptIndexer) *ReceiptIndexer { +}) *ReceiptIndexer { mock := &ReceiptIndexer{} mock.Mock.Test(t) diff --git a/storage/mocks/TransactionIndexer.go b/storage/mocks/TransactionIndexer.go index 0273b005c..f3d23e8a4 100644 --- a/storage/mocks/TransactionIndexer.go +++ b/storage/mocks/TransactionIndexer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks @@ -18,6 +18,10 @@ type TransactionIndexer struct { func (_m *TransactionIndexer) Get(ID common.Hash) (models.Transaction, error) { ret := _m.Called(ID) + if len(ret) == 0 { + panic("no return value specified for Get") + } + var r0 models.Transaction var r1 error if rf, ok := ret.Get(0).(func(common.Hash) (models.Transaction, error)); ok { @@ -44,6 +48,10 @@ func (_m *TransactionIndexer) Get(ID common.Hash) (models.Transaction, error) { func (_m *TransactionIndexer) Store(tx models.Transaction) error { ret := _m.Called(tx) + if len(ret) == 0 { + panic("no return value specified for Store") + } + var r0 error if rf, ok := ret.Get(0).(func(models.Transaction) error); ok { r0 = rf(tx) @@ -54,13 +62,12 @@ func (_m *TransactionIndexer) Store(tx models.Transaction) error { return r0 } -type mockConstructorTestingTNewTransactionIndexer interface { +// NewTransactionIndexer creates a new instance of TransactionIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionIndexer(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionIndexer creates a new instance of TransactionIndexer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionIndexer(t mockConstructorTestingTNewTransactionIndexer) *TransactionIndexer { +}) *TransactionIndexer { mock := &TransactionIndexer{} mock.Mock.Test(t) From 183496aee8270c775b93bf7360460c9ef216d8b7 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:03:11 +0200 Subject: [PATCH 30/58] update mocks --- services/ingestion/mocks/Subscriber.go | 66 -------------------------- 1 file changed, 66 deletions(-) delete mode 100644 services/ingestion/mocks/Subscriber.go diff --git a/services/ingestion/mocks/Subscriber.go b/services/ingestion/mocks/Subscriber.go deleted file mode 100644 index 03f31cdd5..000000000 --- a/services/ingestion/mocks/Subscriber.go +++ /dev/null @@ -1,66 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - context "context" - - flow "github.com/onflow/flow-go-sdk" - - mock "github.com/stretchr/testify/mock" -) - -// Subscriber is an autogenerated mock type for the Subscriber type -type Subscriber struct { - mock.Mock -} - -// Subscribe provides a mock function with given fields: ctx, height -func (_m *Subscriber) Subscribe(ctx context.Context, height uint64) (<-chan flow.BlockEvents, <-chan error, error) { - ret := _m.Called(ctx, height) - - var r0 <-chan flow.BlockEvents - var r1 <-chan error - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, uint64) (<-chan flow.BlockEvents, <-chan error, error)); ok { - return rf(ctx, height) - } - if rf, ok := ret.Get(0).(func(context.Context, uint64) <-chan flow.BlockEvents); ok { - r0 = rf(ctx, height) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan flow.BlockEvents) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, uint64) <-chan error); ok { - r1 = rf(ctx, height) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(<-chan error) - } - } - - if rf, ok := ret.Get(2).(func(context.Context, uint64) error); ok { - r2 = rf(ctx, height) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -type mockConstructorTestingTNewSubscriber interface { - mock.TestingT - Cleanup(func()) -} - -// NewSubscriber creates a new instance of Subscriber. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewSubscriber(t mockConstructorTestingTNewSubscriber) *Subscriber { - mock := &Subscriber{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} From c37a9425648710d104892ee35bb0fe3c6a2854f8 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:12:12 +0200 Subject: [PATCH 31/58] fix make file --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4040009f7..6061e7aa0 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ generate: mockery --dir=storage --name=ReceiptIndexer --output=storage/mocks mockery --dir=storage --name=TransactionIndexer --output=storage/mocks mockery --dir=storage --name=AccountIndexer --output=storage/mocks - mockery --all --dir=services/events --output=services/events/mocks + mockery --all --dir=services/ingestion --output=services/ingestion/mocks mockery --dir=models --name=Engine --output=models/mocks .PHONY: ci From 375b36c968417aaf28382e1ee0fe19577e5b9039 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:15:12 +0200 Subject: [PATCH 32/58] fix test api change --- services/ingestion/engine_test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/services/ingestion/engine_test.go b/services/ingestion/engine_test.go index 1ec0d99e6..148cc8dba 100644 --- a/services/ingestion/engine_test.go +++ b/services/ingestion/engine_test.go @@ -3,13 +3,14 @@ package ingestion import ( "context" "encoding/hex" + "github.com/onflow/flow-evm-gateway/services/ingestion/mocks" "math/big" "testing" "github.com/onflow/cadence" "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-evm-gateway/models" - "github.com/onflow/flow-evm-gateway/services/ingestion/mocks" + storageMock "github.com/onflow/flow-evm-gateway/storage/mocks" "github.com/onflow/flow-go-sdk" broadcast "github.com/onflow/flow-go/engine" @@ -41,12 +42,13 @@ func TestSerialBlockIngestion(t *testing.T) { On("Update"). Return(func() error { return nil }) - eventsChan := make(chan flow.BlockEvents) - subscriber := &mocks.Subscriber{} + eventsChan := make(chan models.BlockEvents) + + subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). - Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { - return eventsChan, make(<-chan error), nil + Return(func(ctx context.Context, latest uint64) <-chan models.BlockEvents { + return eventsChan }) engine := NewEventIngestionEngine( @@ -85,13 +87,13 @@ func TestSerialBlockIngestion(t *testing.T) { }). Once() - eventsChan <- flow.BlockEvents{ + eventsChan <- models.NewBlockEvents(flow.BlockEvents{ Events: []flow.Event{{ Type: string(blockEvent.Etype), Value: blockCdc, }}, Height: cadenceHeight, - } + }) } close(eventsChan) @@ -118,7 +120,7 @@ func TestSerialBlockIngestion(t *testing.T) { Return(func(t models.TransactionCall, r *gethTypes.Receipt) error { return nil }) eventsChan := make(chan flow.BlockEvents) - subscriber := &mocks.Subscriber{} + subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { @@ -214,7 +216,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { Return(func(tx models.Transaction, receipt *gethTypes.Receipt) error { return nil }) eventsChan := make(chan flow.BlockEvents) - subscriber := &mocks.Subscriber{} + subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { @@ -311,7 +313,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { Return(func(tx models.Transaction, receipt *gethTypes.Receipt) error { return nil }) eventsChan := make(chan flow.BlockEvents) - subscriber := &mocks.Subscriber{} + subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { @@ -405,7 +407,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { Return(func(t models.Transaction, r *gethTypes.Receipt) error { return nil }) eventsChan := make(chan flow.BlockEvents) - subscriber := &mocks.Subscriber{} + subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { From 15ebaffb27a88747a4707230ae0412df1ffb144d Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:30:40 +0200 Subject: [PATCH 33/58] fix test api change --- services/ingestion/engine_test.go | 34 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/services/ingestion/engine_test.go b/services/ingestion/engine_test.go index 148cc8dba..f8405d378 100644 --- a/services/ingestion/engine_test.go +++ b/services/ingestion/engine_test.go @@ -119,12 +119,12 @@ func TestSerialBlockIngestion(t *testing.T) { On("Update", mock.Anything, mock.Anything). Return(func(t models.TransactionCall, r *gethTypes.Receipt) error { return nil }) - eventsChan := make(chan flow.BlockEvents) + eventsChan := make(chan models.BlockEvents) subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). - Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { - return eventsChan, make(<-chan error), nil + Return(func(ctx context.Context, latest uint64) <-chan models.BlockEvents { + return eventsChan }) engine := NewEventIngestionEngine( @@ -162,24 +162,28 @@ func TestSerialBlockIngestion(t *testing.T) { }). Once() // this should only be called for first valid block - eventsChan <- flow.BlockEvents{ - Events: []flow.Event{{ - Type: string(blockEvent.Etype), - Value: blockCdc, - }}, - Height: cadenceHeight, + eventsChan <- models.BlockEvents{ + Events: models.NewCadenceEvents(flow.BlockEvents{ + Events: []flow.Event{{ + Type: string(blockEvent.Etype), + Value: blockCdc, + }}, + Height: cadenceHeight, + }), } // fail with next block height being incorrect blockCdc, _, blockEvent, err = newBlock(latestHeight + 10) // not sequential next block height require.NoError(t, err) - eventsChan <- flow.BlockEvents{ - Events: []flow.Event{{ - Type: string(blockEvent.Etype), - Value: blockCdc, - }}, - Height: cadenceHeight + 1, + eventsChan <- models.BlockEvents{ + Events: models.NewCadenceEvents(flow.BlockEvents{ + Events: []flow.Event{{ + Type: string(blockEvent.Etype), + Value: blockCdc, + }}, + Height: cadenceHeight + 1, + }), } close(eventsChan) From 4ea7f9ad0c61811b99f18c8e4f1932c6efc14a01 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:32:29 +0200 Subject: [PATCH 34/58] fix test api change --- services/ingestion/engine_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/ingestion/engine_test.go b/services/ingestion/engine_test.go index f8405d378..8c8f4fe9b 100644 --- a/services/ingestion/engine_test.go +++ b/services/ingestion/engine_test.go @@ -219,12 +219,12 @@ func TestBlockAndTransactionIngestion(t *testing.T) { On("Update", mock.AnythingOfType("models.TransactionCall"), mock.AnythingOfType("*types.Receipt")). Return(func(tx models.Transaction, receipt *gethTypes.Receipt) error { return nil }) - eventsChan := make(chan flow.BlockEvents) + eventsChan := make(chan models.BlockEvents) subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). - Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { - return eventsChan, make(<-chan error), nil + Return(func(ctx context.Context, latest uint64) <-chan models.BlockEvents { + return eventsChan }) txCdc, txEvent, transaction, result, err := newTransaction() @@ -281,7 +281,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { }). Once() - eventsChan <- flow.BlockEvents{ + eventsChan <- models.NewBlockEvents(flow.BlockEvents{ Events: []flow.Event{{ Type: string(blockEvent.Etype), Value: blockCdc, @@ -290,7 +290,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { Value: txCdc, }}, Height: nextHeight, - } + }) close(eventsChan) <-done From d207e5bbcb52396de5ef4bee3f57a818e93cdc54 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:33:55 +0200 Subject: [PATCH 35/58] fix test api change --- services/ingestion/engine_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/services/ingestion/engine_test.go b/services/ingestion/engine_test.go index 8c8f4fe9b..7dc63b88e 100644 --- a/services/ingestion/engine_test.go +++ b/services/ingestion/engine_test.go @@ -316,12 +316,12 @@ func TestBlockAndTransactionIngestion(t *testing.T) { On("Update", mock.AnythingOfType("models.TransactionCall"), mock.AnythingOfType("*types.Receipt")). Return(func(tx models.Transaction, receipt *gethTypes.Receipt) error { return nil }) - eventsChan := make(chan flow.BlockEvents) + eventsChan := make(chan models.BlockEvents) subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). - Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { - return eventsChan, make(<-chan error), nil + Return(func(ctx context.Context, latest uint64) <-chan models.BlockEvents { + return eventsChan }) txCdc, txEvent, _, _, err := newTransaction() @@ -373,7 +373,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { }). Once() - eventsChan <- flow.BlockEvents{ + eventsChan <- models.NewBlockEvents(flow.BlockEvents{ Events: []flow.Event{ // first transaction { @@ -386,7 +386,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { Value: blockCdc, }}, Height: nextHeight, - } + }) close(eventsChan) <-done @@ -410,13 +410,13 @@ func TestBlockAndTransactionIngestion(t *testing.T) { On("Update", mock.Anything, mock.AnythingOfType("*types.Receipt")). Return(func(t models.Transaction, r *gethTypes.Receipt) error { return nil }) - eventsChan := make(chan flow.BlockEvents) + eventsChan := make(chan models.BlockEvents) subscriber := &mocks.EventSubscriber{} subscriber. On("Subscribe", mock.Anything, mock.AnythingOfType("uint64")). - Return(func(ctx context.Context, latest uint64) (<-chan flow.BlockEvents, <-chan error, error) { + Return(func(ctx context.Context, latest uint64) <-chan models.BlockEvents { assert.Equal(t, latestCadenceHeight, latest) - return eventsChan, make(<-chan error), nil + return eventsChan }). Once() @@ -502,10 +502,10 @@ func TestBlockAndTransactionIngestion(t *testing.T) { // and it will make the first block be swapped with second block out-of-order events[1], events[2] = events[2], events[1] - eventsChan <- flow.BlockEvents{ + eventsChan <- models.NewBlockEvents(flow.BlockEvents{ Events: events, Height: latestCadenceHeight + 1, - } + }) close(eventsChan) <-done From 56d6d194e6c0a587a4cc7d6fc7b5fec17b722040 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:36:12 +0200 Subject: [PATCH 36/58] fix test api change --- services/ingestion/engine_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/ingestion/engine_test.go b/services/ingestion/engine_test.go index 7dc63b88e..23273663b 100644 --- a/services/ingestion/engine_test.go +++ b/services/ingestion/engine_test.go @@ -66,7 +66,7 @@ func TestSerialBlockIngestion(t *testing.T) { done := make(chan struct{}) go func() { err := engine.Run(context.Background()) - assert.ErrorIs(t, err, models.ErrDisconnected) // we disconnect at the end + assert.NoError(t, err) close(done) }() @@ -247,7 +247,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { done := make(chan struct{}) go func() { err := engine.Run(context.Background()) - assert.ErrorIs(t, err, models.ErrDisconnected) // we disconnect at the end + assert.NoError(t, err) close(done) }() @@ -344,7 +344,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { done := make(chan struct{}) go func() { err := engine.Run(context.Background()) - assert.ErrorIs(t, err, models.ErrDisconnected) // we disconnect at the end + assert.NoError(t, err) close(done) }() @@ -435,7 +435,7 @@ func TestBlockAndTransactionIngestion(t *testing.T) { done := make(chan struct{}) go func() { err := engine.Run(context.Background()) - assert.ErrorIs(t, err, models.ErrDisconnected) // we disconnect at the end + assert.NoError(t, err) close(done) }() From fad7b9e3f341606724e70edd04b753834684a85c Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:37:25 +0200 Subject: [PATCH 37/58] fix spork check --- models/cross-spork_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 2029435dd..ba1fc7337 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -69,7 +69,7 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { // IsPastSpork will check if the provided height is contained in the previous sporks. func (c *CrossSporkClient) IsPastSpork(height uint64) bool { - return height <= c.getSporkBoundariesDesc()[0] + return len(c.getSporkBoundariesDesc()) > 0 && height <= c.getSporkBoundariesDesc()[0] } // getClientForHeight returns the client for the given height. It starts by using the current spork client, From 9a3ce7486330caeaa973aba6d53543605aacc4d1 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:39:11 +0200 Subject: [PATCH 38/58] add mock --- services/ingestion/mocks/EventSubscriber.go | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 services/ingestion/mocks/EventSubscriber.go diff --git a/services/ingestion/mocks/EventSubscriber.go b/services/ingestion/mocks/EventSubscriber.go new file mode 100644 index 000000000..6a42a2e7f --- /dev/null +++ b/services/ingestion/mocks/EventSubscriber.go @@ -0,0 +1,50 @@ +// Code generated by mockery v2.38.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + models "github.com/onflow/flow-evm-gateway/models" +) + +// EventSubscriber is an autogenerated mock type for the EventSubscriber type +type EventSubscriber struct { + mock.Mock +} + +// Subscribe provides a mock function with given fields: ctx, height +func (_m *EventSubscriber) Subscribe(ctx context.Context, height uint64) <-chan models.BlockEvents { + ret := _m.Called(ctx, height) + + if len(ret) == 0 { + panic("no return value specified for Subscribe") + } + + var r0 <-chan models.BlockEvents + if rf, ok := ret.Get(0).(func(context.Context, uint64) <-chan models.BlockEvents); ok { + r0 = rf(ctx, height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan models.BlockEvents) + } + } + + return r0 +} + +// NewEventSubscriber creates a new instance of EventSubscriber. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEventSubscriber(t interface { + mock.TestingT + Cleanup(func()) +}) *EventSubscriber { + mock := &EventSubscriber{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From 67aceefd6b4a9ef20d3c1548a091fa6828c0e9c4 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Tue, 7 May 2024 17:53:50 +0200 Subject: [PATCH 39/58] mod tidy --- go.mod | 8 +------- go.sum | 9 --------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/go.mod b/go.mod index c0fb4e4d3..24bcd00b4 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 + google.golang.org/grpc v1.63.2 ) require ( @@ -29,7 +30,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chigopher/pathlib v0.12.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.3 // indirect @@ -85,7 +85,6 @@ require ( github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/jinzhu/copier v0.3.5 // indirect github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/kevinburke/go-bindata v3.24.0+incompatible // indirect github.com/klauspost/compress v1.17.4 // indirect @@ -101,7 +100,6 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -158,7 +156,6 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect - github.com/vektra/mockery/v2 v2.21.4 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v4 v4.3.11 // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect @@ -178,7 +175,6 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect @@ -187,10 +183,8 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 48fd07bae..203ba42ec 100644 --- a/go.sum +++ b/go.sum @@ -1078,8 +1078,6 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chigopher/pathlib v0.12.0 h1:1GM7fN/IwXXmOHbd1jkMqHD2wUhYqUvafgxTwmLT/q8= -github.com/chigopher/pathlib v0.12.0/go.mod h1:EJ5UtJ/sK8Nt6q3VWN+EwZLZ3g0afJiG8NegYiQQ/gQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -1629,8 +1627,6 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E= -github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -1810,7 +1806,6 @@ github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dz github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -2056,7 +2051,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.4.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= @@ -2143,8 +2137,6 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vektra/mockery/v2 v2.21.4 h1:QInaL4ma4Bz/cVipKeBLggB5XyPenpksiz8sJpgBZSE= -github.com/vektra/mockery/v2 v2.21.4/go.mod h1:le9nnD50TkeV8QGwrkza7v/xd3DA7YbSZYBEyiuAlsI= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.11 h1:Q47CePddpNGNhk4GCnAx9DDtASi2rasatE0cd26cZoE= @@ -2632,7 +2624,6 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 873113f2d50d09cd181985eba0b61a76d5079764 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 14:32:25 +0200 Subject: [PATCH 40/58] change to getting a block header only --- bootstrap/bootstrap.go | 2 +- models/cross-spork_client.go | 8 ++++---- models/cross-spork_client_test.go | 8 ++++---- services/ingestion/subscriber.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 25359218a..895d05642 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -122,7 +122,7 @@ func startIngestion( } // make sure the provided block to start the indexing can be loaded - _, err = client.GetBlockByHeight(context.Background(), latestCadenceHeight) + _, err = client.GetBlockHeaderByHeight(context.Background(), latestCadenceHeight) if err != nil { return fmt.Errorf("failed to get provided cadence height: %w", err) } diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index ba1fc7337..25e4904a0 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -111,7 +111,7 @@ func (c *CrossSporkClient) getSporkBoundariesDesc() []uint64 { func (c *CrossSporkClient) GetLatestHeightForSpork(ctx context.Context, height uint64) (uint64, error) { block, err := c. getClientForHeight(height). - GetLatestBlock(ctx, true) + GetLatestBlockHeader(ctx, true) if err != nil { return 0, err } @@ -119,13 +119,13 @@ func (c *CrossSporkClient) GetLatestHeightForSpork(ctx context.Context, height u return block.Height, nil } -func (c *CrossSporkClient) GetBlockByHeight( +func (c *CrossSporkClient) GetBlockHeaderByHeight( ctx context.Context, height uint64, -) (*flow.Block, error) { +) (*flow.BlockHeader, error) { return c. getClientForHeight(height). - GetBlockByHeight(ctx, height) + GetBlockHeaderByHeight(ctx, height) } func (c *CrossSporkClient) ExecuteScriptAtBlockHeight( diff --git a/models/cross-spork_client_test.go b/models/cross-spork_client_test.go index e2127ea73..57d449c46 100644 --- a/models/cross-spork_client_test.go +++ b/models/cross-spork_client_test.go @@ -25,19 +25,19 @@ func TestCrossSporkClient_MultiClient(t *testing.T) { ctx := context.Background() // this height should use current spork client - _, err = client.GetBlockByHeight(ctx, 300) + _, err = client.GetBlockHeaderByHeight(ctx, 300) require.ErrorContains(t, err, clientHosts[0]) // this height should use test2 client - _, err = client.GetBlockByHeight(ctx, 150) + _, err = client.GetBlockHeaderByHeight(ctx, 150) require.ErrorContains(t, err, clientHosts[2]) // this height should use test3 client - _, err = client.GetBlockByHeight(ctx, 50) + _, err = client.GetBlockHeaderByHeight(ctx, 50) require.ErrorContains(t, err, clientHosts[1]) // test boundaries are inclusive - _, err = client.GetBlockByHeight(ctx, 200) + _, err = client.GetBlockHeaderByHeight(ctx, 200) require.ErrorContains(t, err, clientHosts[2]) } diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 65d5d3b26..0a038c099 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -90,7 +90,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...grpc.SubscribeOption) <-chan models.BlockEvents { events := make(chan models.BlockEvents) - _, err := r.client.GetBlockByHeight(ctx, height) + _, err := r.client.GetBlockHeaderByHeight(ctx, height) if err != nil { err = fmt.Errorf("failed to subscribe for events, the block height %d doesn't exist: %w", height, err) events <- models.NewBlockEventsError(err) From 10094b2eebf2c866417040ce6284dd638085d13f Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 14:34:22 +0200 Subject: [PATCH 41/58] add commments --- services/ingestion/subscriber.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 0a038c099..2d27eacaf 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -54,6 +54,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod Uint64("height", height). Msg("height found in previous spork, starting to backfill") + // backfill all the missed events, handling of context cancelation is done by the producer for ev := range r.backfill(ctx, height) { events <- ev @@ -75,7 +76,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod Uint64("next-height", height). Msg("backfilling done, subscribe for live data") - // subscribe in the current spork + // subscribe in the current spork, handling of context cancelation is done by the producer for ev := range r.subscribe(ctx, height) { events <- ev } @@ -87,6 +88,9 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod } // subscribe to events by the provided height and handle any errors. +// +// Subscribing to EVM specific events and handle any disconnection errors +// as well as context cancelations. func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...grpc.SubscribeOption) <-chan models.BlockEvents { events := make(chan models.BlockEvents) From ffac9c36c580035ff41efded448172d7258d7fce Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 14:35:04 +0200 Subject: [PATCH 42/58] add while context error is nil --- services/ingestion/subscriber.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 2d27eacaf..b4fdc6b7f 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -112,7 +112,7 @@ func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...gr close(events) }() - for { + for ctx.Err() == nil { select { case <-ctx.Done(): r.logger.Info().Msg("event ingestion received done signal") From 0949ca4c1f37fd86b299baf4a1bb9485a65dee3f Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 14:38:59 +0200 Subject: [PATCH 43/58] typo --- services/ingestion/subscriber.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index b4fdc6b7f..cd63a4f08 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -54,7 +54,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod Uint64("height", height). Msg("height found in previous spork, starting to backfill") - // backfill all the missed events, handling of context cancelation is done by the producer + // backfill all the missed events, handling of context cancellation is done by the producer for ev := range r.backfill(ctx, height) { events <- ev @@ -76,7 +76,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod Uint64("next-height", height). Msg("backfilling done, subscribe for live data") - // subscribe in the current spork, handling of context cancelation is done by the producer + // subscribe in the current spork, handling of context cancellation is done by the producer for ev := range r.subscribe(ctx, height) { events <- ev } @@ -90,7 +90,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod // subscribe to events by the provided height and handle any errors. // // Subscribing to EVM specific events and handle any disconnection errors -// as well as context cancelations. +// as well as context cancellations. func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...grpc.SubscribeOption) <-chan models.BlockEvents { events := make(chan models.BlockEvents) From 45e49dc1400a944d51d2ceb11e8f76fdafbbfa09 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 15:56:37 +0200 Subject: [PATCH 44/58] update go-sdk to specific version --- go.mod | 4 ++-- go.sum | 8 ++++---- tests/go.mod | 4 ++-- tests/go.sum | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 24bcd00b4..f492fa2de 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/goccy/go-json v0.10.2 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/onflow/cadence v1.0.0-preview.25 + github.com/onflow/cadence v1.0.0-preview.26 github.com/onflow/flow-go v0.34.0-crescendo-preview.18 - github.com/onflow/flow-go-sdk v1.0.0-preview.25 + github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 github.com/onflow/go-ethereum v1.13.4 github.com/rs/cors v1.8.0 github.com/rs/zerolog v1.31.0 diff --git a/go.sum b/go.sum index 203ba42ec..6c8ad64de 100644 --- a/go.sum +++ b/go.sum @@ -1867,8 +1867,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.7.0-rc.1 h1:g2DFhC3JeaA+L7/HZOp4NwE+OfxvfJ8nibymHHw7i3g= github.com/onflow/atree v0.7.0-rc.1/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.25 h1:kSmWjxmg9PS+bsk8C3j1NUTkFAl/jNrintVhlh6miM0= -github.com/onflow/cadence v1.0.0-preview.25/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= +github.com/onflow/cadence v1.0.0-preview.26 h1:FcinbrV7s7SIPzgxEAtA0wWqqf7QrvcSFvMVaKEedv4= +github.com/onflow/cadence v1.0.0-preview.26/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -1883,8 +1883,8 @@ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/ github.com/onflow/flow-go v0.34.0-crescendo-preview.18 h1:Bre7uU/n1PjOEcIkTtaJDo4T5tngjKcr/cAOvxr3se4= github.com/onflow/flow-go v0.34.0-crescendo-preview.18/go.mod h1:bwjzi2kSev1emRVN685FqYfCLYcZ6t2A5z5ztYXfvH8= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.25 h1:wL/+cK7oxSww31qSqTpt1Yfr26c8hJ8YHh9nIdq6PlI= -github.com/onflow/flow-go-sdk v1.0.0-preview.25/go.mod h1:Px1fQdB7WFC0yhYsEM3rhKzuE+Zi8GpBjR4qVuDAwMA= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 h1:+QQq2UOsm/BTpj3Vc42XkJpDqhBao8Eh7rEutc8Out4= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357/go.mod h1:tYb83v2vHO+ew1g0VkPKf7w0geNuTqKOV9zyzqQxN5M= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= diff --git a/tests/go.mod b/tests/go.mod index 765d1370b..84e53d95e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -4,11 +4,11 @@ go 1.20 require ( github.com/goccy/go-json v0.10.2 - github.com/onflow/cadence v1.0.0-preview.25 + github.com/onflow/cadence v1.0.0-preview.26 github.com/onflow/flow-emulator v1.0.0-preview.22 github.com/onflow/flow-evm-gateway v0.0.0-20240201154855-4d4d3d3f19c7 github.com/onflow/flow-go v0.34.0-crescendo-preview.18 - github.com/onflow/flow-go-sdk v1.0.0-preview.25 + github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 github.com/onflow/go-ethereum v1.13.4 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.9.0 diff --git a/tests/go.sum b/tests/go.sum index cfc33f15c..55669d0d8 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -2008,8 +2008,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.7.0-rc.1 h1:g2DFhC3JeaA+L7/HZOp4NwE+OfxvfJ8nibymHHw7i3g= github.com/onflow/atree v0.7.0-rc.1/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.25 h1:kSmWjxmg9PS+bsk8C3j1NUTkFAl/jNrintVhlh6miM0= -github.com/onflow/cadence v1.0.0-preview.25/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= +github.com/onflow/cadence v1.0.0-preview.26 h1:FcinbrV7s7SIPzgxEAtA0wWqqf7QrvcSFvMVaKEedv4= +github.com/onflow/cadence v1.0.0-preview.26/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -2026,8 +2026,8 @@ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/ github.com/onflow/flow-go v0.34.0-crescendo-preview.18 h1:Bre7uU/n1PjOEcIkTtaJDo4T5tngjKcr/cAOvxr3se4= github.com/onflow/flow-go v0.34.0-crescendo-preview.18/go.mod h1:bwjzi2kSev1emRVN685FqYfCLYcZ6t2A5z5ztYXfvH8= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.25 h1:wL/+cK7oxSww31qSqTpt1Yfr26c8hJ8YHh9nIdq6PlI= -github.com/onflow/flow-go-sdk v1.0.0-preview.25/go.mod h1:Px1fQdB7WFC0yhYsEM3rhKzuE+Zi8GpBjR4qVuDAwMA= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 h1:+QQq2UOsm/BTpj3Vc42XkJpDqhBao8Eh7rEutc8Out4= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357/go.mod h1:tYb83v2vHO+ew1g0VkPKf7w0geNuTqKOV9zyzqQxN5M= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= From 689797ceb7927ed2bccdfe82ed66363157aa6a70 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 16:13:59 +0200 Subject: [PATCH 45/58] change to client interface --- models/cross-spork_client.go | 21 +++++++++------------ services/ingestion/subscriber.go | 6 +++--- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 25e4904a0..79654831e 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -5,16 +5,13 @@ import ( "fmt" "github.com/onflow/cadence" "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-go-sdk/access/grpc" "github.com/rs/zerolog" "golang.org/x/exp/maps" "golang.org/x/exp/slices" - gogrpc "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) -var dialOpts = gogrpc.WithTransportCredentials(insecure.NewCredentials()) - // CrossSporkClient is a wrapper around the Flow AN client that can // access different AN APIs based on the height boundaries of the sporks. // @@ -26,22 +23,22 @@ var dialOpts = gogrpc.WithTransportCredentials(insecure.NewCredentials()) type CrossSporkClient struct { logger zerolog.Logger // this map holds the last heights and clients for each spork - sporkHosts map[uint64]*grpc.BaseClient - *grpc.BaseClient + sporkHosts map[uint64]access.Client + access.Client } // NewCrossSporkClient creates a new instance of the client, it accepts the // host to the current spork AN API. func NewCrossSporkClient(currentSporkHost string, logger zerolog.Logger) (*CrossSporkClient, error) { // add current spork AN host as the default client - client, err := grpc.NewBaseClient(currentSporkHost, dialOpts) + client, err := grpc.NewClient(currentSporkHost) if err != nil { return nil, err } return &CrossSporkClient{ logger, - make(map[uint64]*grpc.BaseClient), + make(map[uint64]access.Client), client, }, nil } @@ -52,7 +49,7 @@ func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { return fmt.Errorf("provided last height already exists") } - client, err := grpc.NewBaseClient(host, dialOpts) + client, err := grpc.NewClient(host) if err != nil { return err } @@ -80,11 +77,11 @@ func (c *CrossSporkClient) IsPastSpork(height uint64) bool { // because it still might not have access to the height provided, because there might be other sporks with // lower height boundaries that we didn't configure for. // This would result in the error when using the client to access such data. -func (c *CrossSporkClient) getClientForHeight(height uint64) *grpc.BaseClient { +func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { // start by using the current spork client, then iterate all the upper height boundaries // and find the last client that still contains the height in its upper height limit - client := c.BaseClient + client := c.Client for _, upperBound := range c.getSporkBoundariesDesc() { if upperBound >= height { client = c.sporkHosts[upperBound] @@ -143,7 +140,7 @@ func (c *CrossSporkClient) SubscribeEventsByBlockHeight( ctx context.Context, startHeight uint64, filter flow.EventFilter, - opts ...grpc.SubscribeOption, + opts ...access.SubscribeOption, ) (<-chan flow.BlockEvents, <-chan error, error) { return c. getClientForHeight(startHeight). diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index cd63a4f08..9115d323a 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -7,7 +7,7 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-evm-gateway/models" "github.com/onflow/flow-go-sdk" - "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/fvm/systemcontracts" flowGo "github.com/onflow/flow-go/model/flow" @@ -91,7 +91,7 @@ func (r *RPCSubscriber) Subscribe(ctx context.Context, height uint64) <-chan mod // // Subscribing to EVM specific events and handle any disconnection errors // as well as context cancellations. -func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...grpc.SubscribeOption) <-chan models.BlockEvents { +func (r *RPCSubscriber) subscribe(ctx context.Context, height uint64, opts ...access.SubscribeOption) <-chan models.BlockEvents { events := make(chan models.BlockEvents) _, err := r.client.GetBlockHeaderByHeight(ctx, height) @@ -185,7 +185,7 @@ func (r *RPCSubscriber) backfill(ctx context.Context, height uint64) <-chan mode Uint64("last-spork-height", latestHeight). Msg("backfilling spork") - for ev := range r.subscribe(ctx, height, grpc.WithHeartbeatInterval(1)) { + for ev := range r.subscribe(ctx, height, access.WithHeartbeatInterval(1)) { events <- ev if ev.Err != nil { From 6e4dca5ec4b2ecf909899c59775a8e9a8f72617b Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 16:25:12 +0200 Subject: [PATCH 46/58] change client not requiring height provided, also change client being passed in --- bootstrap/bootstrap.go | 16 +++++++++++++--- config/config.go | 24 +++++------------------- models/cross-spork_client.go | 26 ++++++++++---------------- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 895d05642..791c2433d 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -99,14 +99,24 @@ func startIngestion( ) error { logger.Info().Msg("starting up event ingestion") - client, err := models.NewCrossSporkClient(cfg.AccessNodeHost, logger) + grpcClient, err := grpc.NewClient(cfg.AccessNodeHost) + if err != nil { + return fmt.Errorf("failed to create client connection for host: %s, with error: %w", cfg.AccessNodeHost, err) + } + + client, err := models.NewCrossSporkClient(grpcClient, logger) if err != nil { return err } // if we provided access node previous spork hosts add them to the client - for height, host := range cfg.AccessNodePreviousSporkHosts { - if err := client.AddSpork(height, host); err != nil { + for _, host := range cfg.AccessNodePreviousSporkHosts { + grpcClient, err := grpc.NewClient(host) + if err != nil { + return fmt.Errorf("failed to create client connection for host: %s, with error: %w", host, err) + } + + if err := client.AddSpork(grpcClient); err != nil { return fmt.Errorf("failed to add previous spork host to the client: %w", err) } } diff --git a/config/config.go b/config/config.go index d54c3d4ba..994e9c45d 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "io" "math/big" "os" - "strconv" "strings" "time" @@ -33,9 +32,8 @@ type Config struct { DatabaseDir string // AccessNodeHost defines the current spork Flow network AN host. AccessNodeHost string - // AccessNodePreviousSporkHosts contains a map of latest heights for each spork, - // which can be accessed via the host of the AN provided - AccessNodePreviousSporkHosts map[uint64]string + // AccessNodePreviousSporkHosts contains a list of the ANs hosts for each spork + AccessNodePreviousSporkHosts []string // GRPCPort for the RPC API server RPCPort int // GRPCHost for the RPC API server @@ -73,9 +71,7 @@ type Config struct { } func FromFlags() (*Config, error) { - cfg := &Config{ - AccessNodePreviousSporkHosts: make(map[uint64]string), - } + cfg := &Config{} var evmNetwork, coinbase, gas, coa, key, keysPath, flowNetwork, logLevel, filterExpiry, accessSporkHosts string var streamTimeout int var initHeight uint64 @@ -85,7 +81,7 @@ func FromFlags() (*Config, error) { flag.StringVar(&cfg.RPCHost, "rpc-host", "", "Host for the RPC API server") flag.IntVar(&cfg.RPCPort, "rpc-port", 8545, "Port for the RPC API server") flag.StringVar(&cfg.AccessNodeHost, "access-node-grpc-host", "localhost:3569", "Host to the flow access node gRPC API") - flag.StringVar(&accessSporkHosts, "access-node-spork-hosts", "", `Previous spork AN hosts, defined following the schema: {latest height}@{host} as comma separated list (e.g. "200@host-1.com,300@host2.com")`) + flag.StringVar(&accessSporkHosts, "access-node-spork-hosts", "", `Previous spork AN hosts, defined following the schema: {host1},{host2} as a comma separated list (e.g. "host-1.com,host2.com")`) flag.StringVar(&evmNetwork, "evm-network-id", "previewnet", "EVM network ID (previewnet, testnet, mainnet)") flag.StringVar(&flowNetwork, "flow-network-id", "flow-emulator", "Flow network ID (flow-emulator, flow-previewnet)") flag.StringVar(&coinbase, "coinbase", "", "Coinbase address to use for fee collection") @@ -192,17 +188,7 @@ func FromFlags() (*Config, error) { if accessSporkHosts != "" { heightHosts := strings.Split(accessSporkHosts, ",") for _, hh := range heightHosts { - v := strings.Split(hh, "@") - if len(v) != 2 { - return nil, fmt.Errorf("failed to parse AN spork value provided with --access-node-spork-hosts flag") - } - heightVal, host := v[0], v[1] - height, err := strconv.Atoi(heightVal) - if err != nil { - return nil, fmt.Errorf("failed to parse AN host height value for previous sporks, provided with --access-node-spork-hosts flag") - } - - cfg.AccessNodePreviousSporkHosts[uint64(height)] = host + cfg.AccessNodePreviousSporkHosts = append(cfg.AccessNodePreviousSporkHosts, hh) } } diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 79654831e..44b1136ee 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -6,7 +6,6 @@ import ( "github.com/onflow/cadence" "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" - "github.com/onflow/flow-go-sdk/access/grpc" "github.com/rs/zerolog" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -29,36 +28,31 @@ type CrossSporkClient struct { // NewCrossSporkClient creates a new instance of the client, it accepts the // host to the current spork AN API. -func NewCrossSporkClient(currentSporkHost string, logger zerolog.Logger) (*CrossSporkClient, error) { - // add current spork AN host as the default client - client, err := grpc.NewClient(currentSporkHost) - if err != nil { - return nil, err - } - +func NewCrossSporkClient(currentSporkClient access.Client, logger zerolog.Logger) (*CrossSporkClient, error) { return &CrossSporkClient{ logger, make(map[uint64]access.Client), - client, + currentSporkClient, }, nil } // AddSpork will add a new spork host defined by the last height boundary in that spork. -func (c *CrossSporkClient) AddSpork(lastHeight uint64, host string) error { - if _, ok := c.sporkHosts[lastHeight]; ok { - return fmt.Errorf("provided last height already exists") +func (c *CrossSporkClient) AddSpork(client access.Client) error { + header, err := client.GetLatestBlockHeader(context.Background(), true) + if err != nil { + return fmt.Errorf("could not get latest height using the spork client: %w", err) } - client, err := grpc.NewClient(host) - if err != nil { - return err + lastHeight := header.Height + + if _, ok := c.sporkHosts[lastHeight]; ok { + return fmt.Errorf("provided last height already exists") } c.sporkHosts[lastHeight] = client c.logger.Info(). Uint64("spork-boundary", lastHeight). - Str("host", host). Msg("added spork specific client") return nil From 02d85ad8c6e18bcb889109c6631f34219dde851e Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 18:18:59 +0200 Subject: [PATCH 47/58] add test for subscriber --- go.mod | 2 +- go.sum | 2 + services/ingestion/subscriber_test.go | 95 +++++++++++++++++++++++++++ tests/go.mod | 2 +- tests/go.sum | 2 + 5 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 services/ingestion/subscriber_test.go diff --git a/go.mod b/go.mod index f492fa2de..4f24730b7 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/onflow/cadence v1.0.0-preview.26 github.com/onflow/flow-go v0.34.0-crescendo-preview.18 - github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 + github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 github.com/onflow/go-ethereum v1.13.4 github.com/rs/cors v1.8.0 github.com/rs/zerolog v1.31.0 diff --git a/go.sum b/go.sum index 6c8ad64de..c8d2b8de9 100644 --- a/go.sum +++ b/go.sum @@ -1885,6 +1885,8 @@ github.com/onflow/flow-go v0.34.0-crescendo-preview.18/go.mod h1:bwjzi2kSev1emRV github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 h1:+QQq2UOsm/BTpj3Vc42XkJpDqhBao8Eh7rEutc8Out4= github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357/go.mod h1:tYb83v2vHO+ew1g0VkPKf7w0geNuTqKOV9zyzqQxN5M= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 h1:2Grkjw42ENEUNXv3UEWdOGWdp3SANfH1qTdC4YTCPXk= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530/go.mod h1:ls7R+yg7N17BlzqQ+G1AElQSn9StenSoHeakerrPZ0s= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= diff --git a/services/ingestion/subscriber_test.go b/services/ingestion/subscriber_test.go new file mode 100644 index 000000000..1edd88d9c --- /dev/null +++ b/services/ingestion/subscriber_test.go @@ -0,0 +1,95 @@ +package ingestion + +import ( + "context" + "testing" + + "github.com/onflow/flow-evm-gateway/models" + + flowSDK "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go-sdk/access/mocks" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func setupClient(client *mocks.Client, startHeight uint64, endHeight uint64) { + client. + On("GetLatestBlockHeader", mock.Anything, mock.AnythingOfType("bool")). + Return(func(ctx context.Context, sealed bool) (*flowSDK.BlockHeader, error) { + return &flowSDK.BlockHeader{ + Height: endHeight, + }, nil + }) + + client. + On("GetBlockHeaderByHeight", mock.Anything, mock.AnythingOfType("uint64")). + Return(func(ctx context.Context, height uint64) (*flowSDK.BlockHeader, error) { + if height < startHeight || height > endHeight { + return nil, storage.ErrNotFound + } + + return &flowSDK.BlockHeader{ + Height: height, + }, nil + }) + + client. + On("SubscribeEventsByBlockHeight", mock.Anything, mock.AnythingOfType("uint64"), mock.Anything). + Return(func() (<-chan flow.BlockEvents, <-chan error, error) { + events := make(chan flow.BlockEvents) + + for i := startHeight; i <= endHeight; i++ { + events <- flow.BlockEvents{ + BlockHeight: i, + } + } + + return events, make(chan error), nil + }) +} + +// this test simulates two previous sporks and current spork +// the subscriber should start with spork1Client then proceed to +// spork2Client and end with currentClient. +// All event heights should be emitted in sequence. +func Test_Subscribing(t *testing.T) { + + currentClient := &mocks.Client{} + spork1Client := &mocks.Client{} + spork2Client := &mocks.Client{} + + const endHeight = 50 + setupClient(spork1Client, 1, 10) + setupClient(spork2Client, 11, 20) + setupClient(currentClient, 21, endHeight) + + client, err := models.NewCrossSporkClient(currentClient, zerolog.Nop()) + require.NoError(t, err) + + err = client.AddSpork(spork2Client) + require.NoError(t, err) + + err = client.AddSpork(spork1Client) + require.NoError(t, err) + + subscriber := NewRPCSubscriber(client, flow.Emulator, zerolog.Nop()) + + events := subscriber.Subscribe(context.Background(), 1) + + var prevHeight uint64 + + for ev := range events { + require.NoError(t, ev.Err) + + // this makes sure all the event heights are sequential + eventHeight := ev.Events.CadenceHeight() + require.Equal(t, prevHeight+1, eventHeight) + prevHeight = eventHeight + } + + // this makes sure we indexed all the events + require.Equal(t, endHeight, prevHeight) +} diff --git a/tests/go.mod b/tests/go.mod index 84e53d95e..4d300519c 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -8,7 +8,7 @@ require ( github.com/onflow/flow-emulator v1.0.0-preview.22 github.com/onflow/flow-evm-gateway v0.0.0-20240201154855-4d4d3d3f19c7 github.com/onflow/flow-go v0.34.0-crescendo-preview.18 - github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 + github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 github.com/onflow/go-ethereum v1.13.4 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.9.0 diff --git a/tests/go.sum b/tests/go.sum index 55669d0d8..c688914f0 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -2028,6 +2028,8 @@ github.com/onflow/flow-go v0.34.0-crescendo-preview.18/go.mod h1:bwjzi2kSev1emRV github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 h1:+QQq2UOsm/BTpj3Vc42XkJpDqhBao8Eh7rEutc8Out4= github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357/go.mod h1:tYb83v2vHO+ew1g0VkPKf7w0geNuTqKOV9zyzqQxN5M= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 h1:2Grkjw42ENEUNXv3UEWdOGWdp3SANfH1qTdC4YTCPXk= +github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530/go.mod h1:ls7R+yg7N17BlzqQ+G1AElQSn9StenSoHeakerrPZ0s= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= From db9f8d748206aed52d46b6e3e89cc3d091bf5f88 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 8 May 2024 18:56:55 +0200 Subject: [PATCH 48/58] change type --- services/ingestion/subscriber_test.go | 30 +++++++++++++-------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/services/ingestion/subscriber_test.go b/services/ingestion/subscriber_test.go index 1edd88d9c..62783a972 100644 --- a/services/ingestion/subscriber_test.go +++ b/services/ingestion/subscriber_test.go @@ -6,9 +6,9 @@ import ( "github.com/onflow/flow-evm-gateway/models" - flowSDK "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access/mocks" - "github.com/onflow/flow-go/model/flow" + flowGo "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/storage" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" @@ -17,32 +17,30 @@ import ( func setupClient(client *mocks.Client, startHeight uint64, endHeight uint64) { client. - On("GetLatestBlockHeader", mock.Anything, mock.AnythingOfType("bool")). - Return(func(ctx context.Context, sealed bool) (*flowSDK.BlockHeader, error) { - return &flowSDK.BlockHeader{ - Height: endHeight, - }, nil - }) + On("GetLatestBlockHeader", mock.Anything, mock.Anything). + Return(&flow.BlockHeader{ + Height: endHeight, + }, nil) client. - On("GetBlockHeaderByHeight", mock.Anything, mock.AnythingOfType("uint64")). - Return(func(ctx context.Context, height uint64) (*flowSDK.BlockHeader, error) { + On("GetBlockHeaderByHeight", mock.Anything, mock.Anything). + Return(func(ctx context.Context, height uint64) (*flow.BlockHeader, error) { if height < startHeight || height > endHeight { return nil, storage.ErrNotFound } - return &flowSDK.BlockHeader{ + return &flow.BlockHeader{ Height: height, }, nil }) client. - On("SubscribeEventsByBlockHeight", mock.Anything, mock.AnythingOfType("uint64"), mock.Anything). - Return(func() (<-chan flow.BlockEvents, <-chan error, error) { - events := make(chan flow.BlockEvents) + On("SubscribeEventsByBlockHeight", mock.Anything, mock.Anything, mock.Anything). + Return(func() (<-chan flowGo.BlockEvents, <-chan error, error) { + events := make(chan flowGo.BlockEvents) for i := startHeight; i <= endHeight; i++ { - events <- flow.BlockEvents{ + events <- flowGo.BlockEvents{ BlockHeight: i, } } @@ -75,7 +73,7 @@ func Test_Subscribing(t *testing.T) { err = client.AddSpork(spork1Client) require.NoError(t, err) - subscriber := NewRPCSubscriber(client, flow.Emulator, zerolog.Nop()) + subscriber := NewRPCSubscriber(client, flowGo.Emulator, zerolog.Nop()) events := subscriber.Subscribe(context.Background(), 1) From de058449169e3b5704d76aa82072aaeeb4bd02ed Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Thu, 9 May 2024 14:05:12 +0200 Subject: [PATCH 49/58] implement typed mock --- services/ingestion/subscriber_test.go | 75 +++++++++++++++++---------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/services/ingestion/subscriber_test.go b/services/ingestion/subscriber_test.go index 62783a972..53301e443 100644 --- a/services/ingestion/subscriber_test.go +++ b/services/ingestion/subscriber_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "github.com/onflow/flow-go-sdk/access" + "github.com/onflow/flow-evm-gateway/models" "github.com/onflow/flow-go-sdk" @@ -11,20 +13,42 @@ import ( flowGo "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/storage" "github.com/rs/zerolog" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func setupClient(client *mocks.Client, startHeight uint64, endHeight uint64) { - client. - On("GetLatestBlockHeader", mock.Anything, mock.Anything). - Return(&flow.BlockHeader{ - Height: endHeight, - }, nil) +type mockClient struct { + *mocks.Client + getLatestBlockHeaderFunc func(context.Context, bool) (*flow.BlockHeader, error) + getBlockHeaderByHeightFunc func(context.Context, uint64) (*flow.BlockHeader, error) + subscribeEventsByBlockHeightFunc func(context.Context, uint64, flow.EventFilter, ...access.SubscribeOption) (<-chan flow.BlockEvents, <-chan error, error) +} + +func (c *mockClient) GetBlockHeaderByHeight(ctx context.Context, height uint64) (*flow.BlockHeader, error) { + return c.getBlockHeaderByHeightFunc(ctx, height) +} - client. - On("GetBlockHeaderByHeight", mock.Anything, mock.Anything). - Return(func(ctx context.Context, height uint64) (*flow.BlockHeader, error) { +func (c *mockClient) GetLatestBlockHeader(ctx context.Context, sealed bool) (*flow.BlockHeader, error) { + return c.getLatestBlockHeaderFunc(ctx, sealed) +} + +func (c *mockClient) SubscribeEventsByBlockHeight( + ctx context.Context, + startHeight uint64, + filter flow.EventFilter, + opts ...access.SubscribeOption, +) (<-chan flow.BlockEvents, <-chan error, error) { + return c.subscribeEventsByBlockHeightFunc(ctx, startHeight, filter, opts...) +} + +func setupClient(startHeight uint64, endHeight uint64) access.Client { + return &mockClient{ + Client: &mocks.Client{}, + getLatestBlockHeaderFunc: func(ctx context.Context, sealed bool) (*flow.BlockHeader, error) { + return &flow.BlockHeader{ + Height: endHeight, + }, nil + }, + getBlockHeaderByHeightFunc: func(ctx context.Context, height uint64) (*flow.BlockHeader, error) { if height < startHeight || height > endHeight { return nil, storage.ErrNotFound } @@ -32,21 +56,24 @@ func setupClient(client *mocks.Client, startHeight uint64, endHeight uint64) { return &flow.BlockHeader{ Height: height, }, nil - }) - - client. - On("SubscribeEventsByBlockHeight", mock.Anything, mock.Anything, mock.Anything). - Return(func() (<-chan flowGo.BlockEvents, <-chan error, error) { - events := make(chan flowGo.BlockEvents) + }, + subscribeEventsByBlockHeightFunc: func( + ctx context.Context, + startHeight uint64, + filter flow.EventFilter, + opts ...access.SubscribeOption, + ) (<-chan flow.BlockEvents, <-chan error, error) { + events := make(chan flow.BlockEvents) for i := startHeight; i <= endHeight; i++ { - events <- flowGo.BlockEvents{ - BlockHeight: i, + events <- flow.BlockEvents{ + Height: i, } } return events, make(chan error), nil - }) + }, + } } // this test simulates two previous sporks and current spork @@ -55,14 +82,10 @@ func setupClient(client *mocks.Client, startHeight uint64, endHeight uint64) { // All event heights should be emitted in sequence. func Test_Subscribing(t *testing.T) { - currentClient := &mocks.Client{} - spork1Client := &mocks.Client{} - spork2Client := &mocks.Client{} - const endHeight = 50 - setupClient(spork1Client, 1, 10) - setupClient(spork2Client, 11, 20) - setupClient(currentClient, 21, endHeight) + spork1Client := setupClient(1, 10) + spork2Client := setupClient(11, 20) + currentClient := setupClient(21, endHeight) client, err := models.NewCrossSporkClient(currentClient, zerolog.Nop()) require.NoError(t, err) From 3f910aa3b93c4ddeadd88e2985df618439cad1a2 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Thu, 9 May 2024 14:12:39 +0200 Subject: [PATCH 50/58] test order of blocks --- services/ingestion/subscriber_test.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/services/ingestion/subscriber_test.go b/services/ingestion/subscriber_test.go index 53301e443..b18be9a2f 100644 --- a/services/ingestion/subscriber_test.go +++ b/services/ingestion/subscriber_test.go @@ -65,11 +65,15 @@ func setupClient(startHeight uint64, endHeight uint64) access.Client { ) (<-chan flow.BlockEvents, <-chan error, error) { events := make(chan flow.BlockEvents) - for i := startHeight; i <= endHeight; i++ { - events <- flow.BlockEvents{ - Height: i, + go func() { + defer close(events) + + for i := startHeight; i <= endHeight; i++ { + events <- flow.BlockEvents{ + Height: i, + } } - } + }() return events, make(chan error), nil }, @@ -90,10 +94,10 @@ func Test_Subscribing(t *testing.T) { client, err := models.NewCrossSporkClient(currentClient, zerolog.Nop()) require.NoError(t, err) - err = client.AddSpork(spork2Client) + err = client.AddSpork(spork1Client) require.NoError(t, err) - err = client.AddSpork(spork1Client) + err = client.AddSpork(spork2Client) require.NoError(t, err) subscriber := NewRPCSubscriber(client, flowGo.Emulator, zerolog.Nop()) @@ -103,6 +107,11 @@ func Test_Subscribing(t *testing.T) { var prevHeight uint64 for ev := range events { + if prevHeight == endHeight { + require.ErrorIs(t, ev.Err, models.ErrDisconnected) + break + } + require.NoError(t, ev.Err) // this makes sure all the event heights are sequential @@ -112,5 +121,5 @@ func Test_Subscribing(t *testing.T) { } // this makes sure we indexed all the events - require.Equal(t, endHeight, prevHeight) + require.Equal(t, uint64(endHeight), prevHeight) } From 03b4666c3c27d720a32f161db70c38771c21f9e3 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 15 May 2024 17:38:22 +0200 Subject: [PATCH 51/58] change to accept spork clients as part of the factory --- bootstrap/bootstrap.go | 31 ++++++++++----------- models/cross-spork_client.go | 39 ++++++++++++++++++--------- services/ingestion/subscriber_test.go | 14 ++++------ 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 791c2433d..a67853c94 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -5,6 +5,13 @@ import ( "errors" "fmt" + "github.com/onflow/flow-go-sdk/access" + "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/onflow/flow-go-sdk/crypto" + broadcast "github.com/onflow/flow-go/engine" + "github.com/onflow/go-ethereum/rpc" + "github.com/rs/zerolog" + "github.com/onflow/flow-evm-gateway/api" "github.com/onflow/flow-evm-gateway/config" "github.com/onflow/flow-evm-gateway/models" @@ -13,11 +20,6 @@ import ( "github.com/onflow/flow-evm-gateway/storage" storageErrs "github.com/onflow/flow-evm-gateway/storage/errors" "github.com/onflow/flow-evm-gateway/storage/pebble" - "github.com/onflow/flow-go-sdk/access/grpc" - "github.com/onflow/flow-go-sdk/crypto" - broadcast "github.com/onflow/flow-go/engine" - "github.com/onflow/go-ethereum/rpc" - "github.com/rs/zerolog" ) func Start(ctx context.Context, cfg *config.Config) error { @@ -99,26 +101,25 @@ func startIngestion( ) error { logger.Info().Msg("starting up event ingestion") - grpcClient, err := grpc.NewClient(cfg.AccessNodeHost) + currentSporkClient, err := grpc.NewClient(cfg.AccessNodeHost) if err != nil { return fmt.Errorf("failed to create client connection for host: %s, with error: %w", cfg.AccessNodeHost, err) } - client, err := models.NewCrossSporkClient(grpcClient, logger) - if err != nil { - return err - } - // if we provided access node previous spork hosts add them to the client - for _, host := range cfg.AccessNodePreviousSporkHosts { + pastSporkClients := make([]access.Client, len(cfg.AccessNodePreviousSporkHosts)) + for i, host := range cfg.AccessNodePreviousSporkHosts { grpcClient, err := grpc.NewClient(host) if err != nil { return fmt.Errorf("failed to create client connection for host: %s, with error: %w", host, err) } - if err := client.AddSpork(grpcClient); err != nil { - return fmt.Errorf("failed to add previous spork host to the client: %w", err) - } + pastSporkClients[i] = grpcClient + } + + client, err := models.NewCrossSporkClient(currentSporkClient, pastSporkClients, logger) + if err != nil { + return err } blk, err := client.GetLatestBlock(context.Background(), false) diff --git a/models/cross-spork_client.go b/models/cross-spork_client.go index 44b1136ee..dec887cdb 100644 --- a/models/cross-spork_client.go +++ b/models/cross-spork_client.go @@ -3,6 +3,7 @@ package models import ( "context" "fmt" + "github.com/onflow/cadence" "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" @@ -22,22 +23,34 @@ import ( type CrossSporkClient struct { logger zerolog.Logger // this map holds the last heights and clients for each spork - sporkHosts map[uint64]access.Client + sporkClients map[uint64]access.Client access.Client } -// NewCrossSporkClient creates a new instance of the client, it accepts the -// host to the current spork AN API. -func NewCrossSporkClient(currentSporkClient access.Client, logger zerolog.Logger) (*CrossSporkClient, error) { - return &CrossSporkClient{ +// NewCrossSporkClient creates a new instance of the multi-spork client. It requires +// the current spork client and a slice of past spork clients. +func NewCrossSporkClient( + currentSpork access.Client, + pastSporks []access.Client, + logger zerolog.Logger, +) (*CrossSporkClient, error) { + client := &CrossSporkClient{ logger, make(map[uint64]access.Client), - currentSporkClient, - }, nil + currentSpork, + } + + for _, sporkClient := range pastSporks { + if err := client.addSpork(sporkClient); err != nil { + return nil, err + } + } + + return client, nil } -// AddSpork will add a new spork host defined by the last height boundary in that spork. -func (c *CrossSporkClient) AddSpork(client access.Client) error { +// addSpork will add a new spork host defined by the last height boundary in that spork. +func (c *CrossSporkClient) addSpork(client access.Client) error { header, err := client.GetLatestBlockHeader(context.Background(), true) if err != nil { return fmt.Errorf("could not get latest height using the spork client: %w", err) @@ -45,11 +58,11 @@ func (c *CrossSporkClient) AddSpork(client access.Client) error { lastHeight := header.Height - if _, ok := c.sporkHosts[lastHeight]; ok { + if _, ok := c.sporkClients[lastHeight]; ok { return fmt.Errorf("provided last height already exists") } - c.sporkHosts[lastHeight] = client + c.sporkClients[lastHeight] = client c.logger.Info(). Uint64("spork-boundary", lastHeight). @@ -78,7 +91,7 @@ func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { client := c.Client for _, upperBound := range c.getSporkBoundariesDesc() { if upperBound >= height { - client = c.sporkHosts[upperBound] + client = c.sporkClients[upperBound] c.logger.Debug(). Uint64("spork-boundary", upperBound). @@ -91,7 +104,7 @@ func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { // getSporkBoundaries will return descending order of spork height boundaries func (c *CrossSporkClient) getSporkBoundariesDesc() []uint64 { - heights := maps.Keys(c.sporkHosts) + heights := maps.Keys(c.sporkClients) slices.Sort(heights) // order heights in ascending order slices.Reverse(heights) // make it descending return heights diff --git a/services/ingestion/subscriber_test.go b/services/ingestion/subscriber_test.go index b18be9a2f..a7a99f68d 100644 --- a/services/ingestion/subscriber_test.go +++ b/services/ingestion/subscriber_test.go @@ -87,17 +87,13 @@ func setupClient(startHeight uint64, endHeight uint64) access.Client { func Test_Subscribing(t *testing.T) { const endHeight = 50 - spork1Client := setupClient(1, 10) - spork2Client := setupClient(11, 20) + sporkClients := []access.Client{ + setupClient(1, 10), + setupClient(11, 20), + } currentClient := setupClient(21, endHeight) - client, err := models.NewCrossSporkClient(currentClient, zerolog.Nop()) - require.NoError(t, err) - - err = client.AddSpork(spork1Client) - require.NoError(t, err) - - err = client.AddSpork(spork2Client) + client, err := models.NewCrossSporkClient(currentClient, sporkClients, zerolog.Nop()) require.NoError(t, err) subscriber := NewRPCSubscriber(client, flowGo.Emulator, zerolog.Nop()) From 752af318ba2eebb22b8892fae4ea277de2e4816f Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 15 May 2024 17:39:16 +0200 Subject: [PATCH 52/58] move spork client to requester package --- bootstrap/bootstrap.go | 2 +- services/ingestion/subscriber.go | 8 ++++++-- services/ingestion/subscriber_test.go | 3 ++- .../requester}/cross-spork_client.go | 2 +- .../requester}/cross-spork_client_test.go | 18 ++++++++++++------ 5 files changed, 22 insertions(+), 11 deletions(-) rename {models => services/requester}/cross-spork_client.go (99%) rename {models => services/requester}/cross-spork_client_test.go (79%) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index a67853c94..a01504bb3 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -117,7 +117,7 @@ func startIngestion( pastSporkClients[i] = grpcClient } - client, err := models.NewCrossSporkClient(currentSporkClient, pastSporkClients, logger) + client, err := requester.NewCrossSporkClient(currentSporkClient, pastSporkClients, logger) if err != nil { return err } diff --git a/services/ingestion/subscriber.go b/services/ingestion/subscriber.go index 9115d323a..48fb2d325 100644 --- a/services/ingestion/subscriber.go +++ b/services/ingestion/subscriber.go @@ -4,8 +4,12 @@ import ( "context" "errors" "fmt" + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/flow-evm-gateway/models" + "github.com/onflow/flow-evm-gateway/services/requester" + "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-go/fvm/evm/types" @@ -25,12 +29,12 @@ type EventSubscriber interface { var _ EventSubscriber = &RPCSubscriber{} type RPCSubscriber struct { - client *models.CrossSporkClient + client *requester.CrossSporkClient chain flowGo.ChainID logger zerolog.Logger } -func NewRPCSubscriber(client *models.CrossSporkClient, chainID flowGo.ChainID, logger zerolog.Logger) *RPCSubscriber { +func NewRPCSubscriber(client *requester.CrossSporkClient, chainID flowGo.ChainID, logger zerolog.Logger) *RPCSubscriber { logger = logger.With().Str("component", "subscriber").Logger() return &RPCSubscriber{client: client, chain: chainID, logger: logger} } diff --git a/services/ingestion/subscriber_test.go b/services/ingestion/subscriber_test.go index a7a99f68d..c55163748 100644 --- a/services/ingestion/subscriber_test.go +++ b/services/ingestion/subscriber_test.go @@ -7,6 +7,7 @@ import ( "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-evm-gateway/models" + "github.com/onflow/flow-evm-gateway/services/requester" "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access/mocks" @@ -93,7 +94,7 @@ func Test_Subscribing(t *testing.T) { } currentClient := setupClient(21, endHeight) - client, err := models.NewCrossSporkClient(currentClient, sporkClients, zerolog.Nop()) + client, err := requester.NewCrossSporkClient(currentClient, sporkClients, zerolog.Nop()) require.NoError(t, err) subscriber := NewRPCSubscriber(client, flowGo.Emulator, zerolog.Nop()) diff --git a/models/cross-spork_client.go b/services/requester/cross-spork_client.go similarity index 99% rename from models/cross-spork_client.go rename to services/requester/cross-spork_client.go index dec887cdb..24a388cbb 100644 --- a/models/cross-spork_client.go +++ b/services/requester/cross-spork_client.go @@ -1,4 +1,4 @@ -package models +package requester import ( "context" diff --git a/models/cross-spork_client_test.go b/services/requester/cross-spork_client_test.go similarity index 79% rename from models/cross-spork_client_test.go rename to services/requester/cross-spork_client_test.go index 57d449c46..bd559a993 100644 --- a/models/cross-spork_client_test.go +++ b/services/requester/cross-spork_client_test.go @@ -1,22 +1,28 @@ -package models +package requester import ( "context" + "testing" + + "github.com/onflow/flow-go-sdk/access" + "github.com/onflow/flow-go-sdk/access/grpc" "github.com/rs/zerolog" "github.com/stretchr/testify/require" - "testing" ) func TestCrossSporkClient_MultiClient(t *testing.T) { + sporkClients := []access.Client{ + grpc.NewClient("test1.com"), + } clientHosts := []string{"test1.com", "test2.com", "test3.com"} client, err := NewCrossSporkClient(clientHosts[0], zerolog.Nop()) require.NoError(t, err) - err = client.AddSpork(100, clientHosts[1]) + err = client.addSpork(clientHosts[1]) require.NoError(t, err) - err = client.AddSpork(200, clientHosts[2]) + err = client.addSpork(clientHosts[2]) require.NoError(t, err) c := client.getClientForHeight(300) @@ -45,9 +51,9 @@ func TestCrossSporkClient_ExistingHeight(t *testing.T) { client, err := NewCrossSporkClient("host1.com", zerolog.Nop()) require.NoError(t, err) - err = client.AddSpork(100, "host2.com") + err = client.addSpork(100, "host2.com") require.NoError(t, err) - err = client.AddSpork(100, "host3.com") + err = client.addSpork(100, "host3.com") require.EqualError(t, err, "provided last height already exists") } From ac66503aa577fcddd58f80776471dcf72ded994d Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Wed, 15 May 2024 17:40:32 +0200 Subject: [PATCH 53/58] remove test that overlap with subscription test --- services/requester/cross-spork_client_test.go | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 services/requester/cross-spork_client_test.go diff --git a/services/requester/cross-spork_client_test.go b/services/requester/cross-spork_client_test.go deleted file mode 100644 index bd559a993..000000000 --- a/services/requester/cross-spork_client_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package requester - -import ( - "context" - "testing" - - "github.com/onflow/flow-go-sdk/access" - "github.com/onflow/flow-go-sdk/access/grpc" - "github.com/rs/zerolog" - "github.com/stretchr/testify/require" -) - -func TestCrossSporkClient_MultiClient(t *testing.T) { - sporkClients := []access.Client{ - grpc.NewClient("test1.com"), - } - clientHosts := []string{"test1.com", "test2.com", "test3.com"} - - client, err := NewCrossSporkClient(clientHosts[0], zerolog.Nop()) - require.NoError(t, err) - - err = client.addSpork(clientHosts[1]) - require.NoError(t, err) - - err = client.addSpork(clientHosts[2]) - require.NoError(t, err) - - c := client.getClientForHeight(300) - require.NotNil(t, c) - - ctx := context.Background() - - // this height should use current spork client - _, err = client.GetBlockHeaderByHeight(ctx, 300) - require.ErrorContains(t, err, clientHosts[0]) - - // this height should use test2 client - _, err = client.GetBlockHeaderByHeight(ctx, 150) - require.ErrorContains(t, err, clientHosts[2]) - - // this height should use test3 client - _, err = client.GetBlockHeaderByHeight(ctx, 50) - require.ErrorContains(t, err, clientHosts[1]) - - // test boundaries are inclusive - _, err = client.GetBlockHeaderByHeight(ctx, 200) - require.ErrorContains(t, err, clientHosts[2]) -} - -func TestCrossSporkClient_ExistingHeight(t *testing.T) { - client, err := NewCrossSporkClient("host1.com", zerolog.Nop()) - require.NoError(t, err) - - err = client.addSpork(100, "host2.com") - require.NoError(t, err) - - err = client.addSpork(100, "host3.com") - require.EqualError(t, err, "provided last height already exists") -} From e2b9adbecf76900439225000bbf4dffab867d2df Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Thu, 16 May 2024 13:11:46 +0200 Subject: [PATCH 54/58] update go sdk --- go.mod | 8 ++++---- go.sum | 14 ++++++-------- tests/go.mod | 6 +++--- tests/go.sum | 16 ++++++---------- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 90ed94332..8a1f0c207 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,15 @@ require ( github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/goccy/go-json v0.10.2 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/onflow/cadence v1.0.0-preview.25.0.20240503144130-39b3fd879e0d + github.com/onflow/cadence v1.0.0-preview.27 github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 - github.com/onflow/flow-go-sdk v1.0.0-preview.25 + github.com/onflow/flow-go-sdk v1.0.0-preview.28 github.com/onflow/go-ethereum v1.13.4 github.com/rs/cors v1.8.0 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 - google.golang.org/grpc v1.63.2 ) require ( @@ -112,7 +111,7 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.7.0-rc.1 // indirect + github.com/onflow/atree v0.7.0-rc.2 // indirect github.com/onflow/crypto v0.25.1 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 // indirect @@ -183,6 +182,7 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7abeb046c..6db3fd370 100644 --- a/go.sum +++ b/go.sum @@ -1864,11 +1864,11 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.7.0-rc.1 h1:g2DFhC3JeaA+L7/HZOp4NwE+OfxvfJ8nibymHHw7i3g= -github.com/onflow/atree v0.7.0-rc.1/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= +github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.25.0.20240503144130-39b3fd879e0d h1:htl4rq4IpP4tfP+W6BWVmUhfK9XI3Pj0nCd3saQ27CQ= -github.com/onflow/cadence v1.0.0-preview.25.0.20240503144130-39b3fd879e0d/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= +github.com/onflow/cadence v1.0.0-preview.27 h1:6aNxnNlOLsRSuCgyOzaJysJogiB71raJlnV1Q6QFehM= +github.com/onflow/cadence v1.0.0-preview.27/go.mod h1:3LM1VgE9HkJ815whY/F0LYWULwJa8p2nJiKyIIxpGAE= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -1883,10 +1883,8 @@ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/ github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 h1:IG4mKNwnasPeGuVkHCgwlBGhij3mZZqIfy4KrLz5GEs= github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3/go.mod h1:FhP2hv+ykEDL5UUi7sn3DUxV787WNyX84CNCltX1hhM= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 h1:+QQq2UOsm/BTpj3Vc42XkJpDqhBao8Eh7rEutc8Out4= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357/go.mod h1:tYb83v2vHO+ew1g0VkPKf7w0geNuTqKOV9zyzqQxN5M= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 h1:2Grkjw42ENEUNXv3UEWdOGWdp3SANfH1qTdC4YTCPXk= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530/go.mod h1:ls7R+yg7N17BlzqQ+G1AElQSn9StenSoHeakerrPZ0s= +github.com/onflow/flow-go-sdk v1.0.0-preview.28 h1:y/HUyOfzo+f4WyRRoWWn4KfBzX/0iAzf5O99Xq8fwjI= +github.com/onflow/flow-go-sdk v1.0.0-preview.28/go.mod h1:6CyisRZsC3KfOIzonTEt13202U0MyaVIEXZmPBfYQF8= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= diff --git a/tests/go.mod b/tests/go.mod index 40ac3a438..7c36fd6c3 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -4,11 +4,11 @@ go 1.20 require ( github.com/goccy/go-json v0.10.2 - github.com/onflow/cadence v1.0.0-preview.26 + github.com/onflow/cadence v1.0.0-preview.27 github.com/onflow/flow-emulator v1.0.0-preview.22 github.com/onflow/flow-evm-gateway v0.0.0-20240201154855-4d4d3d3f19c7 github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 - github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 + github.com/onflow/flow-go-sdk v1.0.0-preview.28 github.com/onflow/go-ethereum v1.13.4 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.9.0 @@ -129,7 +129,7 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.7.0-rc.1 // indirect + github.com/onflow/atree v0.7.0-rc.2 // indirect github.com/onflow/crypto v0.25.1 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 // indirect diff --git a/tests/go.sum b/tests/go.sum index abe1cfbe6..830bde2ee 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -2005,13 +2005,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.7.0-rc.1 h1:g2DFhC3JeaA+L7/HZOp4NwE+OfxvfJ8nibymHHw7i3g= -github.com/onflow/atree v0.7.0-rc.1/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= +github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.26 h1:FcinbrV7s7SIPzgxEAtA0wWqqf7QrvcSFvMVaKEedv4= -github.com/onflow/cadence v1.0.0-preview.26/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= -github.com/onflow/cadence v1.0.0-preview.25.0.20240503144130-39b3fd879e0d h1:htl4rq4IpP4tfP+W6BWVmUhfK9XI3Pj0nCd3saQ27CQ= -github.com/onflow/cadence v1.0.0-preview.25.0.20240503144130-39b3fd879e0d/go.mod h1:fGhLBbuEmv5rh48qv0ZS0tUz53gxWsHpB4dPsF09h6E= +github.com/onflow/cadence v1.0.0-preview.27 h1:6aNxnNlOLsRSuCgyOzaJysJogiB71raJlnV1Q6QFehM= +github.com/onflow/cadence v1.0.0-preview.27/go.mod h1:3LM1VgE9HkJ815whY/F0LYWULwJa8p2nJiKyIIxpGAE= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= @@ -2028,10 +2026,8 @@ github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/ github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 h1:IG4mKNwnasPeGuVkHCgwlBGhij3mZZqIfy4KrLz5GEs= github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3/go.mod h1:FhP2hv+ykEDL5UUi7sn3DUxV787WNyX84CNCltX1hhM= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357 h1:+QQq2UOsm/BTpj3Vc42XkJpDqhBao8Eh7rEutc8Out4= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508134612-8ffbdd39d357/go.mod h1:tYb83v2vHO+ew1g0VkPKf7w0geNuTqKOV9zyzqQxN5M= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530 h1:2Grkjw42ENEUNXv3UEWdOGWdp3SANfH1qTdC4YTCPXk= -github.com/onflow/flow-go-sdk v1.0.0-preview.26.0.20240508143638-d03c30a27530/go.mod h1:ls7R+yg7N17BlzqQ+G1AElQSn9StenSoHeakerrPZ0s= +github.com/onflow/flow-go-sdk v1.0.0-preview.28 h1:y/HUyOfzo+f4WyRRoWWn4KfBzX/0iAzf5O99Xq8fwjI= +github.com/onflow/flow-go-sdk v1.0.0-preview.28/go.mod h1:6CyisRZsC3KfOIzonTEt13202U0MyaVIEXZmPBfYQF8= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= From 3cadf70ac1006afe52bf8b9bf27135f2bc6eb6d7 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Fri, 17 May 2024 18:28:13 +0200 Subject: [PATCH 55/58] cache spork boundaries --- services/requester/cross-spork_client.go | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/services/requester/cross-spork_client.go b/services/requester/cross-spork_client.go index 24a388cbb..f33093247 100644 --- a/services/requester/cross-spork_client.go +++ b/services/requester/cross-spork_client.go @@ -23,7 +23,8 @@ import ( type CrossSporkClient struct { logger zerolog.Logger // this map holds the last heights and clients for each spork - sporkClients map[uint64]access.Client + sporkClients map[uint64]access.Client + sporkBoundaries []uint64 access.Client } @@ -35,9 +36,9 @@ func NewCrossSporkClient( logger zerolog.Logger, ) (*CrossSporkClient, error) { client := &CrossSporkClient{ - logger, - make(map[uint64]access.Client), - currentSpork, + logger: logger, + sporkClients: make(map[uint64]access.Client), + Client: currentSpork, } for _, sporkClient := range pastSporks { @@ -46,6 +47,13 @@ func NewCrossSporkClient( } } + // create a descending list of block heights that represent boundaries + // of each spork, after crossing each height, we use a different client + heights := maps.Keys(client.sporkClients) + slices.Sort(heights) + slices.Reverse(heights) // make it descending + client.sporkBoundaries = heights + return client, nil } @@ -73,7 +81,7 @@ func (c *CrossSporkClient) addSpork(client access.Client) error { // IsPastSpork will check if the provided height is contained in the previous sporks. func (c *CrossSporkClient) IsPastSpork(height uint64) bool { - return len(c.getSporkBoundariesDesc()) > 0 && height <= c.getSporkBoundariesDesc()[0] + return len(c.sporkBoundaries) > 0 && height <= c.sporkBoundaries[0] } // getClientForHeight returns the client for the given height. It starts by using the current spork client, @@ -89,7 +97,7 @@ func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { // start by using the current spork client, then iterate all the upper height boundaries // and find the last client that still contains the height in its upper height limit client := c.Client - for _, upperBound := range c.getSporkBoundariesDesc() { + for _, upperBound := range c.sporkBoundaries { if upperBound >= height { client = c.sporkClients[upperBound] @@ -102,14 +110,6 @@ func (c *CrossSporkClient) getClientForHeight(height uint64) access.Client { return client } -// getSporkBoundaries will return descending order of spork height boundaries -func (c *CrossSporkClient) getSporkBoundariesDesc() []uint64 { - heights := maps.Keys(c.sporkClients) - slices.Sort(heights) // order heights in ascending order - slices.Reverse(heights) // make it descending - return heights -} - // GetLatestHeightForSpork will determine the spork client in which the provided height is contained // and then find the latest height in that spork. func (c *CrossSporkClient) GetLatestHeightForSpork(ctx context.Context, height uint64) (uint64, error) { From 68d9dff182e43a69e3d3803cbf76b0dd0943c058 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Fri, 17 May 2024 18:32:30 +0200 Subject: [PATCH 56/58] update flow-go v0.35.6-crescendo-preview.22-atree-inlining --- go.mod | 20 ++++++++++---------- go.sum | 44 +++++++++++++++++++++++--------------------- tests/go.mod | 22 +++++++++++----------- tests/go.sum | 11 +++++++++++ 4 files changed, 55 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 8a1f0c207..fdae28224 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/goccy/go-json v0.10.2 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/onflow/cadence v1.0.0-preview.27 - github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 + github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27 + github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining github.com/onflow/flow-go-sdk v1.0.0-preview.28 github.com/onflow/go-ethereum v1.13.4 github.com/rs/cors v1.8.0 @@ -111,15 +111,15 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.7.0-rc.2 // indirect + github.com/onflow/atree v0.8.0-rc.2 // indirect github.com/onflow/crypto v0.25.1 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e // indirect - github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e // indirect - github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 // indirect - github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 // indirect - github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0 // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 // indirect + github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect + github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.2.0 // indirect + github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect + github.com/onflow/flow/protobuf/go/flow v0.4.3 // indirect github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.18.1 // indirect diff --git a/go.sum b/go.sum index 6db3fd370..4ab49ed72 100644 --- a/go.sum +++ b/go.sum @@ -1566,6 +1566,8 @@ github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZm github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-clone v1.6.0 h1:HMo5uvg4wgfiy5FoGOqlFLQED/VGRm2D9Pi8g1FXPGc= +github.com/huandu/go-clone/generic v1.7.2 h1:47pQphxs1Xc9cVADjOHN+Bm5D0hNagwH9UXErbxgVKA= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= @@ -1864,39 +1866,39 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= -github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.8.0-rc.2 h1:7XYaOiiYJqLadzmyLyju2ztoqyTw/ikzcI0HI2LA+bI= +github.com/onflow/atree v0.8.0-rc.2/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.27 h1:6aNxnNlOLsRSuCgyOzaJysJogiB71raJlnV1Q6QFehM= -github.com/onflow/cadence v1.0.0-preview.27/go.mod h1:3LM1VgE9HkJ815whY/F0LYWULwJa8p2nJiKyIIxpGAE= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27 h1:J2fp33mZcGI9EIQcKw6nImXhGn9LnEmHakslPwTNlnw= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27/go.mod h1:KclJlSGWG4USgPK4CsI3V/YtCHYOwPpjyzb6iEfWlbM= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 h1:by3a+8p2kUUjnxfbRYRd78bDEeXAc3PK2LzyBEQqkV4= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5/go.mod h1:+4JWLclBOT+emyBh6NAZSEbqEwzHcWHpIbfsXmRASgY= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 h1:6Cg0h+8Iyy/Nnefk5j0gdeVoMTNpUooAMjyV8sk6zoA= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5/go.mod h1:0oTx6Nkc+LdOXaZe3PRtV1cY+J5z5ig08alR8d+OPHs= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e h1:2LO6Rtmz2PVfH+ZXnMwvTwVeIz3PCy0fs3lQraqog14= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e h1:jl7SYAui/gYRmBofrY//Ln8ixRJwvLzvwLstNfRKmWY= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 h1:IG4mKNwnasPeGuVkHCgwlBGhij3mZZqIfy4KrLz5GEs= -github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3/go.mod h1:FhP2hv+ykEDL5UUi7sn3DUxV787WNyX84CNCltX1hhM= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0 h1:tpyP/qMOub/ImWWCbf1PQjBLB9UKAzPvMCIRbGb/GYg= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0/go.mod h1:ACen33wxzi3AsptCw/1rQ4McV5LumEhK0cC17RFG4o0= +github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 h1:za6bxPPW4JIsthhasUDTa1ruKjIO8DIhun9INQfj61Y= +github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= +github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0= +github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= +github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= +github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= +github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining h1:c9shcs72P+kYpeJFY2k/x2ZaxldvqhE5oT3mKOyQcxM= +github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining/go.mod h1:yF/1pAW00n7xePu54RMnKq3l9xwB5N6wrsrH9X+y7Ro= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.28 h1:y/HUyOfzo+f4WyRRoWWn4KfBzX/0iAzf5O99Xq8fwjI= github.com/onflow/flow-go-sdk v1.0.0-preview.28/go.mod h1:6CyisRZsC3KfOIzonTEt13202U0MyaVIEXZmPBfYQF8= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= +github.com/onflow/flow-nft/lib/go/contracts v1.2.0 h1:TFH7GCJKcGi0+x14SCvF7Gbnxp68gJOGNV8PSIAM2J0= +github.com/onflow/flow-nft/lib/go/contracts v1.2.0/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= +github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= +github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030 h1:I+aosSiJny88O4p3nPbCiUcp/UqN6AepvO6uj82bjH0= -github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.3 h1:gdY7Ftto8dtU+0wI+6ZgW4oE+z0DSDUMIDwVx8mqae8= +github.com/onflow/flow/protobuf/go/flow v0.4.3/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ethereum v1.13.4 h1:iNO86fm8RbBbhZ87ZulblInqCdHnAQVY8okBrNsTevc= github.com/onflow/go-ethereum v1.13.4/go.mod h1:cE/gEUkAffhwbVmMJYz+t1dAfVNHNwZCgc3BWtZxBGY= github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba h1:rIehuhO6bj4FkwE4VzwEjX7MoAlOhUJENBJLqDqVxAo= github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= -github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b h1:6O/BEmA99PDT5QVjoJgrYlGsWnpxGJTAMmsC+V9gyds= +github.com/onflow/wal v1.0.2 h1:5bgsJVf2O3cfMNK12fiiTyYZ8cOrUiELt3heBJfHOhc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= diff --git a/tests/go.mod b/tests/go.mod index 7c36fd6c3..2059a10d6 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -4,10 +4,10 @@ go 1.20 require ( github.com/goccy/go-json v0.10.2 - github.com/onflow/cadence v1.0.0-preview.27 + github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27 github.com/onflow/flow-emulator v1.0.0-preview.22 github.com/onflow/flow-evm-gateway v0.0.0-20240201154855-4d4d3d3f19c7 - github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 + github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining github.com/onflow/flow-go-sdk v1.0.0-preview.28 github.com/onflow/go-ethereum v1.13.4 github.com/rs/zerolog v1.31.0 @@ -129,17 +129,17 @@ require ( github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onflow/atree v0.7.0-rc.2 // indirect + github.com/onflow/atree v0.8.0-rc.2 // indirect github.com/onflow/crypto v0.25.1 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e // indirect - github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e // indirect - github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 // indirect - github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 // indirect - github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0 // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 // indirect + github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect + github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.2.0 // indirect + github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect + github.com/onflow/flow/protobuf/go/flow v0.4.3 // indirect github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba // indirect - github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b // indirect + github.com/onflow/wal v1.0.2 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect diff --git a/tests/go.sum b/tests/go.sum index 830bde2ee..b3994a436 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -2007,40 +2007,51 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.8.0-rc.2/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= github.com/onflow/cadence v1.0.0-preview.27 h1:6aNxnNlOLsRSuCgyOzaJysJogiB71raJlnV1Q6QFehM= github.com/onflow/cadence v1.0.0-preview.27/go.mod h1:3LM1VgE9HkJ815whY/F0LYWULwJa8p2nJiKyIIxpGAE= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27/go.mod h1:KclJlSGWG4USgPK4CsI3V/YtCHYOwPpjyzb6iEfWlbM= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 h1:by3a+8p2kUUjnxfbRYRd78bDEeXAc3PK2LzyBEQqkV4= github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5/go.mod h1:+4JWLclBOT+emyBh6NAZSEbqEwzHcWHpIbfsXmRASgY= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0/go.mod h1:ACen33wxzi3AsptCw/1rQ4McV5LumEhK0cC17RFG4o0= github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 h1:6Cg0h+8Iyy/Nnefk5j0gdeVoMTNpUooAMjyV8sk6zoA= github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5/go.mod h1:0oTx6Nkc+LdOXaZe3PRtV1cY+J5z5ig08alR8d+OPHs= +github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= github.com/onflow/flow-emulator v1.0.0-preview.22 h1:l0BPXlDvK0gZDqQhY3Kc7pBm38pE/FbfefiZe7j6Ngg= github.com/onflow/flow-emulator v1.0.0-preview.22/go.mod h1:60M4QPVpdpEhEdz6NGOtLule+jcRLLKID1gYv2ehLvw= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e h1:2LO6Rtmz2PVfH+ZXnMwvTwVeIz3PCy0fs3lQraqog14= github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= +github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e h1:jl7SYAui/gYRmBofrY//Ln8ixRJwvLzvwLstNfRKmWY= github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= +github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 h1:IG4mKNwnasPeGuVkHCgwlBGhij3mZZqIfy4KrLz5GEs= github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3/go.mod h1:FhP2hv+ykEDL5UUi7sn3DUxV787WNyX84CNCltX1hhM= +github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining/go.mod h1:yF/1pAW00n7xePu54RMnKq3l9xwB5N6wrsrH9X+y7Ro= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.28 h1:y/HUyOfzo+f4WyRRoWWn4KfBzX/0iAzf5O99Xq8fwjI= github.com/onflow/flow-go-sdk v1.0.0-preview.28/go.mod h1:6CyisRZsC3KfOIzonTEt13202U0MyaVIEXZmPBfYQF8= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= +github.com/onflow/flow-nft/lib/go/contracts v1.2.0/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= +github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030 h1:I+aosSiJny88O4p3nPbCiUcp/UqN6AepvO6uj82bjH0= github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.3/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ethereum v1.13.4 h1:iNO86fm8RbBbhZ87ZulblInqCdHnAQVY8okBrNsTevc= github.com/onflow/go-ethereum v1.13.4/go.mod h1:cE/gEUkAffhwbVmMJYz+t1dAfVNHNwZCgc3BWtZxBGY= github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba h1:rIehuhO6bj4FkwE4VzwEjX7MoAlOhUJENBJLqDqVxAo= github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b h1:6O/BEmA99PDT5QVjoJgrYlGsWnpxGJTAMmsC+V9gyds= github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= +github.com/onflow/wal v1.0.2/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From f1a1df58f4ca912910f72cffb659c06a528bf1f4 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Fri, 17 May 2024 19:22:47 +0200 Subject: [PATCH 57/58] mod tidy --- tests/go.mod | 2 ++ tests/go.sum | 39 +++++++++++++++++---------------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/tests/go.mod b/tests/go.mod index 2059a10d6..24785f12a 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -84,6 +84,8 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.2.4 // indirect + github.com/huandu/go-clone v1.6.0 // indirect + github.com/huandu/go-clone/generic v1.7.2 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index b3994a436..712db3006 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -1667,6 +1667,12 @@ github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZm github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/go-clone v1.6.0 h1:HMo5uvg4wgfiy5FoGOqlFLQED/VGRm2D9Pi8g1FXPGc= +github.com/huandu/go-clone v1.6.0/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= +github.com/huandu/go-clone/generic v1.7.2 h1:47pQphxs1Xc9cVADjOHN+Bm5D0hNagwH9UXErbxgVKA= +github.com/huandu/go-clone/generic v1.7.2/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= @@ -2005,52 +2011,41 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= -github.com/onflow/atree v0.7.0-rc.2 h1:mZmVrl/zPlfI44EjV3FdR2QwIqT8nz1sCONUBFcML/U= -github.com/onflow/atree v0.7.0-rc.2/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= +github.com/onflow/atree v0.8.0-rc.2 h1:7XYaOiiYJqLadzmyLyju2ztoqyTw/ikzcI0HI2LA+bI= github.com/onflow/atree v0.8.0-rc.2/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.27 h1:6aNxnNlOLsRSuCgyOzaJysJogiB71raJlnV1Q6QFehM= -github.com/onflow/cadence v1.0.0-preview.27/go.mod h1:3LM1VgE9HkJ815whY/F0LYWULwJa8p2nJiKyIIxpGAE= +github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27 h1:J2fp33mZcGI9EIQcKw6nImXhGn9LnEmHakslPwTNlnw= github.com/onflow/cadence v1.0.0-preview-atree-register-inlining.27/go.mod h1:KclJlSGWG4USgPK4CsI3V/YtCHYOwPpjyzb6iEfWlbM= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A= github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5 h1:by3a+8p2kUUjnxfbRYRd78bDEeXAc3PK2LzyBEQqkV4= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.15.2-0.20240429192223-e696a8e439b5/go.mod h1:+4JWLclBOT+emyBh6NAZSEbqEwzHcWHpIbfsXmRASgY= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0 h1:tpyP/qMOub/ImWWCbf1PQjBLB9UKAzPvMCIRbGb/GYg= github.com/onflow/flow-core-contracts/lib/go/contracts v1.0.0/go.mod h1:ACen33wxzi3AsptCw/1rQ4McV5LumEhK0cC17RFG4o0= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5 h1:6Cg0h+8Iyy/Nnefk5j0gdeVoMTNpUooAMjyV8sk6zoA= -github.com/onflow/flow-core-contracts/lib/go/templates v0.15.2-0.20240429192223-e696a8e439b5/go.mod h1:0oTx6Nkc+LdOXaZe3PRtV1cY+J5z5ig08alR8d+OPHs= +github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0 h1:za6bxPPW4JIsthhasUDTa1ruKjIO8DIhun9INQfj61Y= github.com/onflow/flow-core-contracts/lib/go/templates v1.0.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30= github.com/onflow/flow-emulator v1.0.0-preview.22 h1:l0BPXlDvK0gZDqQhY3Kc7pBm38pE/FbfefiZe7j6Ngg= github.com/onflow/flow-emulator v1.0.0-preview.22/go.mod h1:60M4QPVpdpEhEdz6NGOtLule+jcRLLKID1gYv2ehLvw= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e h1:2LO6Rtmz2PVfH+ZXnMwvTwVeIz3PCy0fs3lQraqog14= -github.com/onflow/flow-ft/lib/go/contracts v0.7.1-0.20240424211859-3ff4c0fe2a1e/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= +github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0= github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e h1:jl7SYAui/gYRmBofrY//Ln8ixRJwvLzvwLstNfRKmWY= -github.com/onflow/flow-ft/lib/go/templates v0.7.1-0.20240424211859-3ff4c0fe2a1e/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= +github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3 h1:IG4mKNwnasPeGuVkHCgwlBGhij3mZZqIfy4KrLz5GEs= -github.com/onflow/flow-go v0.35.4-crescendo-preview.20-evm-fix3/go.mod h1:FhP2hv+ykEDL5UUi7sn3DUxV787WNyX84CNCltX1hhM= +github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining h1:c9shcs72P+kYpeJFY2k/x2ZaxldvqhE5oT3mKOyQcxM= github.com/onflow/flow-go v0.35.6-crescendo-preview.22-atree-inlining/go.mod h1:yF/1pAW00n7xePu54RMnKq3l9xwB5N6wrsrH9X+y7Ro= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.28 h1:y/HUyOfzo+f4WyRRoWWn4KfBzX/0iAzf5O99Xq8fwjI= github.com/onflow/flow-go-sdk v1.0.0-preview.28/go.mod h1:6CyisRZsC3KfOIzonTEt13202U0MyaVIEXZmPBfYQF8= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140 h1:oTj4RGgfuJSSBE1aDVrlh6avxKBMraucpNtRg0K+yhg= -github.com/onflow/flow-nft/lib/go/contracts v1.1.1-0.20240429184308-40c3de711140/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= +github.com/onflow/flow-nft/lib/go/contracts v1.2.0 h1:TFH7GCJKcGi0+x14SCvF7Gbnxp68gJOGNV8PSIAM2J0= github.com/onflow/flow-nft/lib/go/contracts v1.2.0/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140 h1:7NwSIG4SEdm+96gr+Aaqx291jZ/Bqkxk/ghVnens9CU= -github.com/onflow/flow-nft/lib/go/templates v0.0.0-20240429184308-40c3de711140/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= +github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030 h1:I+aosSiJny88O4p3nPbCiUcp/UqN6AepvO6uj82bjH0= -github.com/onflow/flow/protobuf/go/flow v0.4.1-0.20240412170550-911321113030/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.3 h1:gdY7Ftto8dtU+0wI+6ZgW4oE+z0DSDUMIDwVx8mqae8= github.com/onflow/flow/protobuf/go/flow v0.4.3/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ethereum v1.13.4 h1:iNO86fm8RbBbhZ87ZulblInqCdHnAQVY8okBrNsTevc= github.com/onflow/go-ethereum v1.13.4/go.mod h1:cE/gEUkAffhwbVmMJYz+t1dAfVNHNwZCgc3BWtZxBGY= github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba h1:rIehuhO6bj4FkwE4VzwEjX7MoAlOhUJENBJLqDqVxAo= github.com/onflow/sdks v0.5.1-0.20230912225508-b35402f12bba/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= -github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b h1:6O/BEmA99PDT5QVjoJgrYlGsWnpxGJTAMmsC+V9gyds= -github.com/onflow/wal v0.0.0-20240208022732-d756cd497d3b/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= +github.com/onflow/wal v1.0.2 h1:5bgsJVf2O3cfMNK12fiiTyYZ8cOrUiELt3heBJfHOhc= github.com/onflow/wal v1.0.2/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From 14d972f0ef5429bfa5fd5420448c7dc30fd7a464 Mon Sep 17 00:00:00 2001 From: sideninja <75445744+sideninja@users.noreply.github.com> Date: Mon, 20 May 2024 13:43:28 +0200 Subject: [PATCH 58/58] replace for loop --- config/config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index 3cac9abe4..43e6ad5d3 100644 --- a/config/config.go +++ b/config/config.go @@ -190,9 +190,7 @@ func FromFlags() (*Config, error) { if accessSporkHosts != "" { heightHosts := strings.Split(accessSporkHosts, ",") - for _, hh := range heightHosts { - cfg.AccessNodePreviousSporkHosts = append(cfg.AccessNodePreviousSporkHosts, hh) - } + cfg.AccessNodePreviousSporkHosts = append(cfg.AccessNodePreviousSporkHosts, heightHosts...) } if forceStartHeight != 0 {