From 1ecb0646f01f3ba0c5372c2f1a51fc680647cb9f Mon Sep 17 00:00:00 2001 From: Adrian Sutton Date: Wed, 16 Oct 2024 15:22:30 +1000 Subject: [PATCH] op-dispute-mon: Use game data from previous update cycle if update fails --- op-dispute-mon/metrics/metrics.go | 15 +++- op-dispute-mon/metrics/noop.go | 2 + op-dispute-mon/mon/extract/extractor.go | 22 ++++-- op-dispute-mon/mon/extract/extractor_test.go | 78 ++++++++++++++++---- op-dispute-mon/mon/monitor.go | 35 +++------ op-dispute-mon/mon/monitor_test.go | 75 +++++-------------- op-dispute-mon/mon/service.go | 17 ++--- op-dispute-mon/mon/types/types.go | 1 + op-dispute-mon/mon/update_times.go | 34 +++++++++ op-dispute-mon/mon/update_times_test.go | 43 +++++++++++ 10 files changed, 208 insertions(+), 114 deletions(-) create mode 100644 op-dispute-mon/mon/update_times.go create mode 100644 op-dispute-mon/mon/update_times_test.go diff --git a/op-dispute-mon/metrics/metrics.go b/op-dispute-mon/metrics/metrics.go index b76c23c4ca26..aa8cd9947905 100644 --- a/op-dispute-mon/metrics/metrics.go +++ b/op-dispute-mon/metrics/metrics.go @@ -183,6 +183,8 @@ type Metricer interface { RecordL2Challenges(agreement bool, count int) + RecordOldestGameUpdateTime(t time.Time) + caching.Metrics contractMetrics.ContractMetricer } @@ -215,7 +217,8 @@ type Metrics struct { credits prometheus.GaugeVec honestWithdrawableAmounts prometheus.GaugeVec - lastOutputFetch prometheus.Gauge + lastOutputFetch prometheus.Gauge + oldestGameUpdateTime prometheus.Gauge gamesAgreement prometheus.GaugeVec latestValidProposalL2Block prometheus.Gauge @@ -269,6 +272,12 @@ func NewMetrics() *Metrics { Name: "last_output_fetch", Help: "Timestamp of the last output fetch", }), + oldestGameUpdateTime: factory.NewGauge(prometheus.GaugeOpts{ + Namespace: Namespace, + Name: "oldest_game_update_time", + Help: "Timestamp the least recently updated game " + + "or the time of the last update cycle if there were no games in the monitoring window", + }), honestActorClaims: *factory.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Name: "honest_actor_claims", @@ -499,6 +508,10 @@ func (m *Metrics) RecordOutputFetchTime(timestamp float64) { m.lastOutputFetch.Set(timestamp) } +func (m *Metrics) RecordOldestGameUpdateTime(t time.Time) { + m.oldestGameUpdateTime.Set(float64(t.Unix())) +} + func (m *Metrics) RecordGameAgreement(status GameAgreementStatus, count int) { m.gamesAgreement.WithLabelValues(labelValuesFor(status)...).Set(float64(count)) } diff --git a/op-dispute-mon/metrics/noop.go b/op-dispute-mon/metrics/noop.go index 6b4982ff26ba..4459fdd9c1fb 100644 --- a/op-dispute-mon/metrics/noop.go +++ b/op-dispute-mon/metrics/noop.go @@ -36,6 +36,8 @@ func (*NoopMetricsImpl) RecordWithdrawalRequests(_ common.Address, _ bool, _ int func (*NoopMetricsImpl) RecordOutputFetchTime(_ float64) {} +func (*NoopMetricsImpl) RecordOldestGameUpdateTime(_ time.Time) {} + func (*NoopMetricsImpl) RecordGameAgreement(_ GameAgreementStatus, _ int) {} func (*NoopMetricsImpl) RecordLatestValidProposalL2Block(_ uint64) {} diff --git a/op-dispute-mon/mon/extract/extractor.go b/op-dispute-mon/mon/extract/extractor.go index d19cab340b66..c40b31ccb6c5 100644 --- a/op-dispute-mon/mon/extract/extractor.go +++ b/op-dispute-mon/mon/extract/extractor.go @@ -9,9 +9,11 @@ import ( gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types" monTypes "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types" + "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/sources/batching/rpcblock" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "golang.org/x/exp/maps" ) var ( @@ -29,20 +31,23 @@ type Enricher interface { type Extractor struct { logger log.Logger + clock clock.Clock createContract CreateGameCaller fetchGames FactoryGameFetcher maxConcurrency int enrichers []Enricher ignoredGames map[common.Address]bool + latestGameData map[common.Address]*monTypes.EnrichedGameData } -func NewExtractor(logger log.Logger, creator CreateGameCaller, fetchGames FactoryGameFetcher, ignoredGames []common.Address, maxConcurrency uint, enrichers ...Enricher) *Extractor { +func NewExtractor(logger log.Logger, cl clock.Clock, creator CreateGameCaller, fetchGames FactoryGameFetcher, ignoredGames []common.Address, maxConcurrency uint, enrichers ...Enricher) *Extractor { ignored := make(map[common.Address]bool) for _, game := range ignoredGames { ignored[game] = true } return &Extractor{ logger: logger, + clock: cl, createContract: creator, fetchGames: fetchGames, maxConcurrency: int(maxConcurrency), @@ -61,7 +66,6 @@ func (e *Extractor) Extract(ctx context.Context, blockHash common.Hash, minTimes } func (e *Extractor) enrichGames(ctx context.Context, blockHash common.Hash, games []gameTypes.GameMetadata) ([]*monTypes.EnrichedGameData, int, int) { - var enrichedGames []*monTypes.EnrichedGameData var ignored atomic.Int32 var failed atomic.Int32 @@ -101,8 +105,14 @@ func (e *Extractor) enrichGames(ctx context.Context, blockHash common.Hash, game }() } - // Push each game into the channel + // Create a new store for game data. This ensures any games no longer in the monitoring set are dropped. + updatedGameData := make(map[common.Address]*monTypes.EnrichedGameData) + // Push each game into the channel and store the latest cached game data as a default if fetching fails for _, game := range games { + previousData := e.latestGameData[game.Proxy] + if previousData != nil { + updatedGameData[game.Proxy] = previousData + } gameCh <- game } close(gameCh) @@ -112,9 +122,10 @@ func (e *Extractor) enrichGames(ctx context.Context, blockHash common.Hash, game // Read the results for enrichedGame := range enrichedCh { - enrichedGames = append(enrichedGames, enrichedGame) + updatedGameData[enrichedGame.Proxy] = enrichedGame } - return enrichedGames, int(ignored.Load()), int(failed.Load()) + e.latestGameData = updatedGameData + return maps.Values(updatedGameData), int(ignored.Load()), int(failed.Load()) } func (e *Extractor) enrichGame(ctx context.Context, blockHash common.Hash, game gameTypes.GameMetadata) (*monTypes.EnrichedGameData, error) { @@ -138,6 +149,7 @@ func (e *Extractor) enrichGame(ctx context.Context, blockHash common.Hash, game enrichedClaims[i] = monTypes.EnrichedClaim{Claim: claim} } enrichedGame := &monTypes.EnrichedGameData{ + LastUpdateTime: e.clock.Now(), GameMetadata: game, L1Head: meta.L1Head, L2BlockNumber: meta.L2BlockNum, diff --git a/op-dispute-mon/mon/extract/extractor_test.go b/op-dispute-mon/mon/extract/extractor_test.go index 361aeb57420e..ee4b8a63ee25 100644 --- a/op-dispute-mon/mon/extract/extractor_test.go +++ b/op-dispute-mon/mon/extract/extractor_test.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts" monTypes "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types" + "github.com/ethereum-optimism/optimism/op-service/clock" "github.com/ethereum-optimism/optimism/op-service/sources/batching/rpcblock" "github.com/stretchr/testify/require" @@ -26,7 +27,7 @@ var ( func TestExtractor_Extract(t *testing.T) { t.Run("FetchGamesError", func(t *testing.T) { - extractor, _, games, _ := setupExtractorTest(t) + extractor, _, games, _, _ := setupExtractorTest(t) games.err = errors.New("boom") _, _, _, err := extractor.Extract(context.Background(), common.Hash{}, 0) require.ErrorIs(t, err, games.err) @@ -34,7 +35,7 @@ func TestExtractor_Extract(t *testing.T) { }) t.Run("CreateGameErrorLog", func(t *testing.T) { - extractor, creator, games, logs := setupExtractorTest(t) + extractor, creator, games, logs, _ := setupExtractorTest(t) games.games = []gameTypes.GameMetadata{{}} creator.err = errors.New("boom") enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) @@ -50,7 +51,7 @@ func TestExtractor_Extract(t *testing.T) { }) t.Run("MetadataFetchErrorLog", func(t *testing.T) { - extractor, creator, games, logs := setupExtractorTest(t) + extractor, creator, games, logs, _ := setupExtractorTest(t) games.games = []gameTypes.GameMetadata{{}} creator.caller.metadataErr = errors.New("boom") enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) @@ -66,7 +67,7 @@ func TestExtractor_Extract(t *testing.T) { }) t.Run("ClaimsFetchErrorLog", func(t *testing.T) { - extractor, creator, games, logs := setupExtractorTest(t) + extractor, creator, games, logs, _ := setupExtractorTest(t) games.games = []gameTypes.GameMetadata{{}} creator.caller.claimsErr = errors.New("boom") enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) @@ -82,7 +83,7 @@ func TestExtractor_Extract(t *testing.T) { }) t.Run("Success", func(t *testing.T) { - extractor, creator, games, _ := setupExtractorTest(t) + extractor, creator, games, _, _ := setupExtractorTest(t) games.games = []gameTypes.GameMetadata{{}} enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) require.NoError(t, err) @@ -97,7 +98,7 @@ func TestExtractor_Extract(t *testing.T) { t.Run("EnricherFails", func(t *testing.T) { enricher := &mockEnricher{err: errors.New("whoops")} - extractor, _, games, logs := setupExtractorTest(t, enricher) + extractor, _, games, logs, _ := setupExtractorTest(t, enricher) games.games = []gameTypes.GameMetadata{{}} enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) require.NoError(t, err) @@ -110,7 +111,7 @@ func TestExtractor_Extract(t *testing.T) { t.Run("EnricherSuccess", func(t *testing.T) { enricher := &mockEnricher{} - extractor, _, games, _ := setupExtractorTest(t, enricher) + extractor, _, games, _, _ := setupExtractorTest(t, enricher) games.games = []gameTypes.GameMetadata{{}} enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) require.NoError(t, err) @@ -123,8 +124,8 @@ func TestExtractor_Extract(t *testing.T) { t.Run("MultipleEnrichersMultipleGames", func(t *testing.T) { enricher1 := &mockEnricher{} enricher2 := &mockEnricher{} - extractor, _, games, _ := setupExtractorTest(t, enricher1, enricher2) - games.games = []gameTypes.GameMetadata{{}, {}} + extractor, _, games, _, _ := setupExtractorTest(t, enricher1, enricher2) + games.games = []gameTypes.GameMetadata{{Proxy: common.Address{0xaa}}, {Proxy: common.Address{0xbb}}} enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) require.NoError(t, err) require.Zero(t, ignored) @@ -136,7 +137,7 @@ func TestExtractor_Extract(t *testing.T) { t.Run("IgnoreGames", func(t *testing.T) { enricher1 := &mockEnricher{} - extractor, _, games, logs := setupExtractorTest(t, enricher1) + extractor, _, games, logs, _ := setupExtractorTest(t, enricher1) // Two games, one of which is ignored games.games = []gameTypes.GameMetadata{{Proxy: ignoredGames[0]}, {Proxy: common.Address{0xaa}}} enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) @@ -152,6 +153,47 @@ func TestExtractor_Extract(t *testing.T) { testlog.NewMessageFilter("Ignoring game"), testlog.NewAttributesFilter("game", ignoredGames[0].Hex()))) }) + + t.Run("UseCachedValueOnFailure", func(t *testing.T) { + enricher := &mockEnricher{} + extractor, _, games, _, cl := setupExtractorTest(t, enricher) + gameA := common.Address{0xaa} + gameB := common.Address{0xbb} + games.games = []gameTypes.GameMetadata{{Proxy: gameA}, {Proxy: gameB}} + + // First fetch succeeds and the results should be cached + enriched, ignored, failed, err := extractor.Extract(context.Background(), common.Hash{}, 0) + require.NoError(t, err) + require.Zero(t, ignored) + require.Zero(t, failed) + require.Len(t, enriched, 2) + require.Equal(t, 2, enricher.calls) + firstUpdateTime := cl.Now() + require.Equal(t, firstUpdateTime, enriched[0].LastUpdateTime) + require.Equal(t, firstUpdateTime, enriched[1].LastUpdateTime) + + cl.AdvanceTime(2 * time.Minute) + secondUpdateTime := cl.Now() + enricher.action = func(game *monTypes.EnrichedGameData) error { + if game.Proxy == gameA { + return errors.New("boom") + } + // Updated games will have a different status + game.Status = gameTypes.GameStatusChallengerWon + return nil + } + // Second fetch fails for one of the two games, it's cached value should be used. + enriched, ignored, failed, err = extractor.Extract(context.Background(), common.Hash{}, 0) + require.NoError(t, err) + require.Zero(t, ignored) + require.Equal(t, 1, failed) + require.Len(t, enriched, 2) + require.Equal(t, 4, enricher.calls) + require.Equal(t, enriched[0].Status, gameTypes.GameStatusInProgress) // Uses cached value from game A + require.Equal(t, enriched[1].Status, gameTypes.GameStatusChallengerWon) // Updates game B + require.Equal(t, firstUpdateTime, enriched[0].LastUpdateTime) + require.Equal(t, secondUpdateTime, enriched[1].LastUpdateTime) + }) } func verifyLogs(t *testing.T, logs *testlog.CapturingHandler, createErr, metadataErr, claimsErr, durationErr int) { @@ -170,20 +212,22 @@ func verifyLogs(t *testing.T, logs *testlog.CapturingHandler, createErr, metadat require.Len(t, l, durationErr) } -func setupExtractorTest(t *testing.T, enrichers ...Enricher) (*Extractor, *mockGameCallerCreator, *mockGameFetcher, *testlog.CapturingHandler) { +func setupExtractorTest(t *testing.T, enrichers ...Enricher) (*Extractor, *mockGameCallerCreator, *mockGameFetcher, *testlog.CapturingHandler, *clock.DeterministicClock) { logger, capturedLogs := testlog.CaptureLogger(t, log.LvlDebug) games := &mockGameFetcher{} caller := &mockGameCaller{rootClaim: mockRootClaim} creator := &mockGameCallerCreator{caller: caller} + cl := clock.NewDeterministicClock(time.Unix(48294294, 58)) extractor := NewExtractor( logger, + cl, creator.CreateGameCaller, games.FetchGames, ignoredGames, 5, enrichers..., ) - return extractor, creator, games, capturedLogs + return extractor, creator, games, capturedLogs, cl } type mockGameFetcher struct { @@ -311,11 +355,15 @@ func (m *mockGameCaller) IsResolved(_ context.Context, _ rpcblock.Block, claims } type mockEnricher struct { - err error - calls int + err error + calls int + action func(game *monTypes.EnrichedGameData) error } -func (m *mockEnricher) Enrich(_ context.Context, _ rpcblock.Block, _ GameCaller, _ *monTypes.EnrichedGameData) error { +func (m *mockEnricher) Enrich(_ context.Context, _ rpcblock.Block, _ GameCaller, game *monTypes.EnrichedGameData) error { m.calls++ + if m.action != nil { + return m.action(game) + } return m.err } diff --git a/op-dispute-mon/mon/monitor.go b/op-dispute-mon/mon/monitor.go index 039255608c73..5f7764adccbd 100644 --- a/op-dispute-mon/mon/monitor.go +++ b/op-dispute-mon/mon/monitor.go @@ -14,8 +14,6 @@ import ( ) type ForecastResolution func(games []*types.EnrichedGameData, ignoredCount, failedCount int) -type Bonds func(games []*types.EnrichedGameData) -type Resolutions func(games []*types.EnrichedGameData) type Monitor func(games []*types.EnrichedGameData) type BlockHashFetcher func(ctx context.Context, number *big.Int) (common.Hash, error) type BlockNumberFetcher func(ctx context.Context) (uint64, error) @@ -38,11 +36,7 @@ type gameMonitor struct { monitorInterval time.Duration forecast ForecastResolution - bonds Bonds - resolutions Resolutions - claims Monitor - withdrawals Monitor - l2Challenges Monitor + monitors []Monitor extract Extract fetchBlockHash BlockHashFetcher fetchBlockNumber BlockNumberFetcher @@ -55,16 +49,11 @@ func newGameMonitor( metrics MonitorMetrics, monitorInterval time.Duration, gameWindow time.Duration, - forecast ForecastResolution, - bonds Bonds, - resolutions Resolutions, - claims Monitor, - withdrawals Monitor, - l2Challenges Monitor, - extract Extract, - fetchBlockNumber BlockNumberFetcher, fetchBlockHash BlockHashFetcher, -) *gameMonitor { + fetchBlockNumber BlockNumberFetcher, + extract Extract, + forecast ForecastResolution, + monitors ...Monitor) *gameMonitor { return &gameMonitor{ logger: logger, clock: cl, @@ -74,11 +63,7 @@ func newGameMonitor( monitorInterval: monitorInterval, gameWindow: gameWindow, forecast: forecast, - bonds: bonds, - resolutions: resolutions, - claims: claims, - withdrawals: withdrawals, - l2Challenges: l2Challenges, + monitors: monitors, extract: extract, fetchBlockNumber: fetchBlockNumber, fetchBlockHash: fetchBlockHash, @@ -101,12 +86,10 @@ func (m *gameMonitor) monitorGames() error { if err != nil { return fmt.Errorf("failed to load games: %w", err) } - m.resolutions(enrichedGames) m.forecast(enrichedGames, ignored, failed) - m.bonds(enrichedGames) - m.claims(enrichedGames) - m.withdrawals(enrichedGames) - m.l2Challenges(enrichedGames) + for _, monitor := range m.monitors { + monitor(enrichedGames) + } timeTaken := m.clock.Since(start) m.metrics.RecordMonitorDuration(timeTaken) m.logger.Info("Completed monitoring update", "blockNumber", blockNumber, "blockHash", blockHash, "duration", timeTaken, "games", len(enrichedGames), "ignored", ignored, "failed", failed) diff --git a/op-dispute-mon/mon/monitor_test.go b/op-dispute-mon/mon/monitor_test.go index 180c29bb26ce..1181c57b4ecb 100644 --- a/op-dispute-mon/mon/monitor_test.go +++ b/op-dispute-mon/mon/monitor_test.go @@ -25,7 +25,7 @@ func TestMonitor_MonitorGames(t *testing.T) { t.Parallel() t.Run("FailedFetchBlocknumber", func(t *testing.T) { - monitor, _, _, _, _, _, _, _ := setupMonitorTest(t) + monitor, _, _, _ := setupMonitorTest(t) boom := errors.New("boom") monitor.fetchBlockNumber = func(ctx context.Context) (uint64, error) { return 0, boom @@ -35,7 +35,7 @@ func TestMonitor_MonitorGames(t *testing.T) { }) t.Run("FailedFetchBlockHash", func(t *testing.T) { - monitor, _, _, _, _, _, _, _ := setupMonitorTest(t) + monitor, _, _, _ := setupMonitorTest(t) boom := errors.New("boom") monitor.fetchBlockHash = func(ctx context.Context, number *big.Int) (common.Hash, error) { return common.Hash{}, boom @@ -45,29 +45,25 @@ func TestMonitor_MonitorGames(t *testing.T) { }) t.Run("MonitorsWithNoGames", func(t *testing.T) { - monitor, factory, forecast, bonds, withdrawals, resolutions, claims, l2Challenges := setupMonitorTest(t) + monitor, factory, forecast, monitors := setupMonitorTest(t) factory.games = []*monTypes.EnrichedGameData{} err := monitor.monitorGames() require.NoError(t, err) require.Equal(t, 1, forecast.calls) - require.Equal(t, 1, bonds.calls) - require.Equal(t, 1, resolutions.calls) - require.Equal(t, 1, claims.calls) - require.Equal(t, 1, withdrawals.calls) - require.Equal(t, 1, l2Challenges.calls) + for _, m := range monitors { + require.Equal(t, 1, m.calls) + } }) t.Run("MonitorsMultipleGames", func(t *testing.T) { - monitor, factory, forecast, bonds, withdrawals, resolutions, claims, l2Challenges := setupMonitorTest(t) + monitor, factory, forecast, monitors := setupMonitorTest(t) factory.games = []*monTypes.EnrichedGameData{{}, {}, {}} err := monitor.monitorGames() require.NoError(t, err) require.Equal(t, 1, forecast.calls) - require.Equal(t, 1, bonds.calls) - require.Equal(t, 1, resolutions.calls) - require.Equal(t, 1, claims.calls) - require.Equal(t, 1, withdrawals.calls) - require.Equal(t, 1, l2Challenges.calls) + for _, m := range monitors { + require.Equal(t, 1, m.calls) + } }) } @@ -75,7 +71,7 @@ func TestMonitor_StartMonitoring(t *testing.T) { t.Run("MonitorsGames", func(t *testing.T) { addr1 := common.Address{0xaa} addr2 := common.Address{0xbb} - monitor, factory, forecaster, _, _, _, _, _ := setupMonitorTest(t) + monitor, factory, forecaster, _ := setupMonitorTest(t) factory.games = []*monTypes.EnrichedGameData{newEnrichedGameData(addr1, 9999), newEnrichedGameData(addr2, 9999)} factory.maxSuccess = len(factory.games) // Only allow two successful fetches @@ -88,7 +84,7 @@ func TestMonitor_StartMonitoring(t *testing.T) { }) t.Run("FailsToFetchGames", func(t *testing.T) { - monitor, factory, forecaster, _, _, _, _, _ := setupMonitorTest(t) + monitor, factory, forecaster, _ := setupMonitorTest(t) factory.fetchErr = errors.New("boom") monitor.StartMonitoring() @@ -110,7 +106,7 @@ func newEnrichedGameData(proxy common.Address, timestamp uint64) *monTypes.Enric } } -func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast, *mockBonds, *mockMonitor, *mockResolutionMonitor, *mockMonitor, *mockMonitor) { +func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast, []*mockMonitor) { logger := testlog.Logger(t, log.LvlDebug) fetchBlockNum := func(ctx context.Context) (uint64, error) { return 1, nil @@ -123,37 +119,12 @@ func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast cl.Start() extractor := &mockExtractor{} forecast := &mockForecast{} - bonds := &mockBonds{} - resolutions := &mockResolutionMonitor{} - claims := &mockMonitor{} - withdrawals := &mockMonitor{} - l2Challenges := &mockMonitor{} - monitor := newGameMonitor( - context.Background(), - logger, - cl, - metrics.NoopMetrics, - monitorInterval, - 10*time.Second, - forecast.Forecast, - bonds.CheckBonds, - resolutions.CheckResolutions, - claims.Check, - withdrawals.Check, - l2Challenges.Check, - extractor.Extract, - fetchBlockNum, - fetchBlockHash, - ) - return monitor, extractor, forecast, bonds, withdrawals, resolutions, claims, l2Challenges -} - -type mockResolutionMonitor struct { - calls int -} - -func (m *mockResolutionMonitor) CheckResolutions(games []*monTypes.EnrichedGameData) { - m.calls++ + monitor1 := &mockMonitor{} + monitor2 := &mockMonitor{} + monitor3 := &mockMonitor{} + monitor := newGameMonitor(context.Background(), logger, cl, metrics.NoopMetrics, monitorInterval, 10*time.Second, fetchBlockHash, fetchBlockNum, + extractor.Extract, forecast.Forecast, monitor1.Check, monitor2.Check, monitor3.Check) + return monitor, extractor, forecast, []*mockMonitor{monitor1, monitor2, monitor3} } type mockMonitor struct { @@ -172,14 +143,6 @@ func (m *mockForecast) Forecast(_ []*monTypes.EnrichedGameData, _, _ int) { m.calls++ } -type mockBonds struct { - calls int -} - -func (m *mockBonds) CheckBonds(_ []*monTypes.EnrichedGameData) { - m.calls++ -} - type mockExtractor struct { fetchErr error calls int diff --git a/op-dispute-mon/mon/service.go b/op-dispute-mon/mon/service.go index 8ce97b7b8dd9..083e391b9111 100644 --- a/op-dispute-mon/mon/service.go +++ b/op-dispute-mon/mon/service.go @@ -126,6 +126,7 @@ func (s *Service) initGameCallerCreator() { func (s *Service) initExtractor(cfg *config.Config) { s.extractor = extract.NewExtractor( s.logger, + s.cl, s.game.CreateContract, s.factoryContract.GetGamesAtOrAfter, cfg.IgnoredGames, @@ -217,23 +218,17 @@ func (s *Service) initMonitor(ctx context.Context, cfg *config.Config) { return block.Hash(), nil } l2ChallengesMonitor := NewL2ChallengesMonitor(s.logger, s.metrics) - s.monitor = newGameMonitor( - ctx, - s.logger, - s.cl, - s.metrics, - cfg.MonitorInterval, - cfg.GameWindow, + updateTimeMonitor := NewUpdateTimeMonitor(s.cl, s.metrics) + s.monitor = newGameMonitor(ctx, s.logger, s.cl, s.metrics, cfg.MonitorInterval, cfg.GameWindow, blockHashFetcher, + s.l1Client.BlockNumber, + s.extractor.Extract, s.forecast.Forecast, s.bonds.CheckBonds, s.resolutions.CheckResolutions, s.claims.CheckClaims, s.withdrawals.CheckWithdrawals, l2ChallengesMonitor.CheckL2Challenges, - s.extractor.Extract, - s.l1Client.BlockNumber, - blockHashFetcher, - ) + updateTimeMonitor.CheckUpdateTimes) } func (s *Service) Start(ctx context.Context) error { diff --git a/op-dispute-mon/mon/types/types.go b/op-dispute-mon/mon/types/types.go index dff06dab90bf..774ad3908cf7 100644 --- a/op-dispute-mon/mon/types/types.go +++ b/op-dispute-mon/mon/types/types.go @@ -18,6 +18,7 @@ type EnrichedClaim struct { type EnrichedGameData struct { types.GameMetadata + LastUpdateTime time.Time L1Head common.Hash L1HeadNum uint64 L2BlockNumber uint64 diff --git a/op-dispute-mon/mon/update_times.go b/op-dispute-mon/mon/update_times.go new file mode 100644 index 000000000000..3482775757b0 --- /dev/null +++ b/op-dispute-mon/mon/update_times.go @@ -0,0 +1,34 @@ +package mon + +import ( + "time" + + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types" + "github.com/ethereum-optimism/optimism/op-service/clock" +) + +type UpdateTimeMetrics interface { + RecordOldestGameUpdateTime(t time.Time) +} + +type UpdateTimeMonitor struct { + metrics UpdateTimeMetrics + clock clock.Clock +} + +func NewUpdateTimeMonitor(cl clock.Clock, metrics UpdateTimeMetrics) *UpdateTimeMonitor { + return &UpdateTimeMonitor{clock: cl, metrics: metrics} +} + +func (m *UpdateTimeMonitor) CheckUpdateTimes(games []*types.EnrichedGameData) { + // Report the current time if there are no games + // Otherwise the last update time would drop to 0 when there are no games, making it appear there were errors + earliest := m.clock.Now() + + for _, game := range games { + if game.LastUpdateTime.Before(earliest) { + earliest = game.LastUpdateTime + } + } + m.metrics.RecordOldestGameUpdateTime(earliest) +} diff --git a/op-dispute-mon/mon/update_times_test.go b/op-dispute-mon/mon/update_times_test.go new file mode 100644 index 000000000000..a16da62b3c2e --- /dev/null +++ b/op-dispute-mon/mon/update_times_test.go @@ -0,0 +1,43 @@ +package mon + +import ( + "testing" + "time" + + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types" + "github.com/ethereum-optimism/optimism/op-service/clock" + "github.com/stretchr/testify/require" +) + +func TestUpdateTimeMonitor_NoGames(t *testing.T) { + m := &mockUpdateTimeMetrics{} + cl := clock.NewDeterministicClock(time.UnixMilli(45892)) + monitor := NewUpdateTimeMonitor(cl, m) + monitor.CheckUpdateTimes(nil) + require.Equal(t, cl.Now(), m.oldestUpdateTime) + + cl.AdvanceTime(time.Minute) + monitor.CheckUpdateTimes([]*types.EnrichedGameData{}) + require.Equal(t, cl.Now(), m.oldestUpdateTime) +} + +func TestUpdateTimeMonitor_ReportsOldestUpdateTime(t *testing.T) { + m := &mockUpdateTimeMetrics{} + cl := clock.NewDeterministicClock(time.UnixMilli(45892)) + monitor := NewUpdateTimeMonitor(cl, m) + monitor.CheckUpdateTimes([]*types.EnrichedGameData{ + {LastUpdateTime: time.UnixMilli(4)}, + {LastUpdateTime: time.UnixMilli(3)}, + {LastUpdateTime: time.UnixMilli(7)}, + {LastUpdateTime: time.UnixMilli(9)}, + }) + require.Equal(t, time.UnixMilli(3), m.oldestUpdateTime) +} + +type mockUpdateTimeMetrics struct { + oldestUpdateTime time.Time +} + +func (m *mockUpdateTimeMetrics) RecordOldestGameUpdateTime(t time.Time) { + m.oldestUpdateTime = t +}