diff --git a/cmd/server/cadence/server.go b/cmd/server/cadence/server.go index c4bf306074e..21482b10698 100644 --- a/cmd/server/cadence/server.go +++ b/cmd/server/cadence/server.go @@ -182,6 +182,7 @@ func (s *server) startService() common.Daemon { params.MembershipResolver, err = membership.NewResolver( peerProvider, params.Logger, + params.MetricsClient, ) if err != nil { log.Fatalf("error creating membership monitor: %v", err) diff --git a/common/membership/resolver.go b/common/membership/resolver.go index 84014be6afe..16606495e21 100644 --- a/common/membership/resolver.go +++ b/common/membership/resolver.go @@ -31,6 +31,7 @@ import ( "github.com/uber/cadence/common" "github.com/uber/cadence/common/log" "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/metrics" "github.com/uber/cadence/common/service" ) @@ -80,7 +81,8 @@ type ( // MultiringResolver uses ring-per-service for membership information type MultiringResolver struct { - status int32 + metrics metrics.Client + status int32 provider PeerProvider rings map[string]*ring @@ -92,8 +94,9 @@ var _ Resolver = (*MultiringResolver)(nil) func NewResolver( provider PeerProvider, logger log.Logger, + metrics metrics.Client, ) (*MultiringResolver, error) { - return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver)), nil + return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver), metrics), nil } // NewMultiringResolver creates hashrings for all services @@ -101,11 +104,13 @@ func NewMultiringResolver( services []string, provider PeerProvider, logger log.Logger, + metrics metrics.Client, ) *MultiringResolver { rpo := &MultiringResolver{ status: common.DaemonStatusInitialized, provider: provider, rings: make(map[string]*ring), + metrics: metrics, } for _, s := range services { @@ -208,6 +213,7 @@ func (rpo *MultiringResolver) LookupByAddress(service, address string) (HostInfo return m, nil } } + rpo.metrics.Scope(metrics.ResolverHostNotFoundScope).IncCounter(1) return HostInfo{}, errors.New("host not found") } diff --git a/common/membership/resolver_test.go b/common/membership/resolver_test.go index c115990399d..794f621e507 100644 --- a/common/membership/resolver_test.go +++ b/common/membership/resolver_test.go @@ -30,6 +30,7 @@ import ( "github.com/uber/cadence/common" "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/metrics" ) var testServices = []string{"test-worker", "test-services"} @@ -136,5 +137,6 @@ func newTestResolver(t *testing.T) (*MultiringResolver, *MockPeerProvider) { testServices, pp, log.NewNoop(), + metrics.NewNoopMetricsClient(), ), pp } diff --git a/common/metrics/defs.go b/common/metrics/defs.go index 716584ab2e9..afc2137ae02 100644 --- a/common/metrics/defs.go +++ b/common/metrics/defs.go @@ -305,6 +305,9 @@ const ( PersistenceUpdateDynamicConfigScope // PersistenceShardRequestCountScope tracks number of persistence calls made to each shard PersistenceShardRequestCountScope + + // ResolverHostNotFoundScope is a simple low level error indicating a lookup failed in the membership resolver + ResolverHostNotFoundScope // HistoryClientStartWorkflowExecutionScope tracks RPC calls to history service HistoryClientStartWorkflowExecutionScope // HistoryClientDescribeHistoryHostScope tracks RPC calls to history service @@ -1392,6 +1395,7 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{ PersistenceFetchDynamicConfigScope: {operation: "FetchDynamicConfig"}, PersistenceUpdateDynamicConfigScope: {operation: "UpdateDynamicConfig"}, PersistenceShardRequestCountScope: {operation: "ShardIdPersistenceRequest"}, + ResolverHostNotFoundScope: {operation: "ResolverHostNotFound"}, ClusterMetadataArchivalConfigScope: {operation: "ArchivalConfig"},