Adding a metric for hosts not being found in resolver (#5414)

Adds a metric around hosts not being visible in the resolver. This is for debugging and early alerting, though it's normal for this metric to appear during rebalancing or deployments temporarily. Long periods of this metric firing may indicate a problem with service discovery.

cmd/server/cadence/server.go

@@ -182,6 +182,7 @@ func (s *server) startService() common.Daemon {
 	params.MembershipResolver, err = membership.NewResolver(
 		peerProvider,
 		params.Logger,
+		params.MetricsClient,
 	)
 	if err != nil {
 		log.Fatalf("error creating membership monitor: %v", err)

common/membership/resolver.go

@@ -31,6 +31,7 @@ import (
 	"github.com/uber/cadence/common"
 	"github.com/uber/cadence/common/log"
 	"github.com/uber/cadence/common/log/tag"
+	"github.com/uber/cadence/common/metrics"
 	"github.com/uber/cadence/common/service"
 )
 
@@ -80,7 +81,8 @@ type (
 
 // MultiringResolver uses ring-per-service for membership information
 type MultiringResolver struct {
-	status int32
+	metrics metrics.Client
+	status  int32
 
 	provider PeerProvider
 	rings    map[string]*ring
@@ -92,20 +94,23 @@ var _ Resolver = (*MultiringResolver)(nil)
 func NewResolver(
 	provider PeerProvider,
 	logger log.Logger,
+	metrics metrics.Client,
 ) (*MultiringResolver, error) {
-	return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver)), nil
+	return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver), metrics), nil
 }
 
 // NewMultiringResolver creates hashrings for all services
 func NewMultiringResolver(
 	services []string,
 	provider PeerProvider,
 	logger log.Logger,
+	metrics metrics.Client,
 ) *MultiringResolver {
 	rpo := &MultiringResolver{
 		status:   common.DaemonStatusInitialized,
 		provider: provider,
 		rings:    make(map[string]*ring),
+		metrics:  metrics,
 	}
 
 	for _, s := range services {
@@ -208,6 +213,7 @@ func (rpo *MultiringResolver) LookupByAddress(service, address string) (HostInfo
 			return m, nil
 		}
 	}
+	rpo.metrics.Scope(metrics.ResolverHostNotFoundScope).IncCounter(1)
 	return HostInfo{}, errors.New("host not found")
 }
 

common/membership/resolver_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/uber/cadence/common"
 	"github.com/uber/cadence/common/log"
+	"github.com/uber/cadence/common/metrics"
 )
 
 var testServices = []string{"test-worker", "test-services"}
@@ -136,5 +137,6 @@ func newTestResolver(t *testing.T) (*MultiringResolver, *MockPeerProvider) {
 		testServices,
 		pp,
 		log.NewNoop(),
+		metrics.NewNoopMetricsClient(),
 	), pp
 }

common/metrics/defs.go

@@ -305,6 +305,9 @@ const (
 	PersistenceUpdateDynamicConfigScope
 	// PersistenceShardRequestCountScope tracks number of persistence calls made to each shard
 	PersistenceShardRequestCountScope
+
+	// ResolverHostNotFoundScope is a simple low level error indicating a lookup failed in the membership resolver
+	ResolverHostNotFoundScope
 	// HistoryClientStartWorkflowExecutionScope tracks RPC calls to history service
 	HistoryClientStartWorkflowExecutionScope
 	// HistoryClientDescribeHistoryHostScope tracks RPC calls to history service
@@ -1392,6 +1395,7 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{
 		PersistenceFetchDynamicConfigScope:                             {operation: "FetchDynamicConfig"},
 		PersistenceUpdateDynamicConfigScope:                            {operation: "UpdateDynamicConfig"},
 		PersistenceShardRequestCountScope:                              {operation: "ShardIdPersistenceRequest"},
+		ResolverHostNotFoundScope:                                      {operation: "ResolverHostNotFound"},
 
 		ClusterMetadataArchivalConfigScope: {operation: "ArchivalConfig"},