Adding a metric for hosts not being found in resolver

cmd/server/cadence/server.go

@@ -182,6 +182,7 @@ func (s *server) startService() common.Daemon {
 	params.MembershipResolver, err = membership.NewResolver(
 		peerProvider,
 		params.Logger,
+		params.MetricsClient,
 	)
 	if err != nil {
 		log.Fatalf("error creating membership monitor: %v", err)

common/membership/resolver.go

@@ -26,6 +26,7 @@ package membership
 import (
 	"errors"
 	"fmt"
+	"github.com/uber/cadence/common/metrics"
 	"sync/atomic"
 
 	"github.com/uber/cadence/common"
@@ -80,7 +81,8 @@ type (
 
 // MultiringResolver uses ring-per-service for membership information
 type MultiringResolver struct {
-	status int32
+	metrics metrics.Client
+	status  int32
 
 	provider PeerProvider
 	rings    map[string]*ring
@@ -92,20 +94,23 @@ var _ Resolver = (*MultiringResolver)(nil)
 func NewResolver(
 	provider PeerProvider,
 	logger log.Logger,
+	metrics metrics.Client,
 ) (*MultiringResolver, error) {
-	return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver)), nil
+	return NewMultiringResolver(service.List, provider, logger.WithTags(tag.ComponentServiceResolver), metrics), nil
 }
 
 // NewMultiringResolver creates hashrings for all services
 func NewMultiringResolver(
 	services []string,
 	provider PeerProvider,
 	logger log.Logger,
+	metrics metrics.Client,
 ) *MultiringResolver {
 	rpo := &MultiringResolver{
 		status:   common.DaemonStatusInitialized,
 		provider: provider,
 		rings:    make(map[string]*ring),
+		metrics:  metrics,
 	}
 
 	for _, s := range services {
@@ -193,6 +198,7 @@ func (rpo *MultiringResolver) Unsubscribe(service string, name string) error {
 func (rpo *MultiringResolver) Members(service string) ([]HostInfo, error) {
 	ring, err := rpo.getRing(service)
 	if err != nil {
+		rpo.metrics.Scope(metrics.ResolverHostNotFoundScope).IncCounter(1)
 		return nil, err
 	}
 	return ring.Members(), nil
@@ -208,6 +214,7 @@ func (rpo *MultiringResolver) LookupByAddress(service, address string) (HostInfo
 			return m, nil
 		}
 	}
+	rpo.metrics.Scope(metrics.ResolverHostNotFoundScope).IncCounter(1)
 	return HostInfo{}, errors.New("host not found")
 }
 

common/membership/resolver_test.go

@@ -23,6 +23,7 @@
 package membership
 
 import (
+	"github.com/uber/cadence/common/metrics"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -136,5 +137,6 @@ func newTestResolver(t *testing.T) (*MultiringResolver, *MockPeerProvider) {
 		testServices,
 		pp,
 		log.NewNoop(),
+		metrics.NewNoopMetricsClient(),
 	), pp
 }

common/metrics/defs.go

@@ -303,6 +303,9 @@ const (
 	PersistenceUpdateDynamicConfigScope
 	// PersistenceShardRequestCountScope tracks number of persistence calls made to each shard
 	PersistenceShardRequestCountScope
+
+	// ResolverHostNotFoundScope is a simple low level error indicating a lookup failed in the membership resolver
+	ResolverHostNotFoundScope
 	// HistoryClientStartWorkflowExecutionScope tracks RPC calls to history service
 	HistoryClientStartWorkflowExecutionScope
 	// HistoryClientDescribeHistoryHostScope tracks RPC calls to history service
@@ -1353,6 +1356,7 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{
 		PersistenceFetchDynamicConfigScope:                             {operation: "FetchDynamicConfig"},
 		PersistenceUpdateDynamicConfigScope:                            {operation: "UpdateDynamicConfig"},
 		PersistenceShardRequestCountScope:                              {operation: "ShardIdPersistenceRequest"},
+		ResolverHostNotFoundScope:                                      {operation: "ResolverHostNotFound"},
 
 		ClusterMetadataArchivalConfigScope: {operation: "ArchivalConfig"},