Separate history/matching failure metrics from cadence failures (#488)

cadence-workflow · Jan 3, 2018 · 6ff9cbd · 6ff9cbd
1 parent bb03ce5
commit 6ff9cbd
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 24 deletions.
diff --git a/client/history/metricClient.go b/client/history/metricClient.go
@@ -55,7 +55,7 @@ func (c *metricClient) StartWorkflowExecution(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientStartWorkflowExecutionScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientStartWorkflowExecutionScope, metrics.HistoryFailures)
 	}
 
 	return resp, err
@@ -72,7 +72,7 @@ func (c *metricClient) GetMutableState(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientGetMutableStateScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientGetMutableStateScope, metrics.HistoryFailures)
 	}
 
 	return resp, err
@@ -89,7 +89,7 @@ func (c *metricClient) DescribeWorkflowExecution(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientDescribeWorkflowExecutionScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientDescribeWorkflowExecutionScope, metrics.HistoryFailures)
 	}
 
 	return resp, err
@@ -106,7 +106,7 @@ func (c *metricClient) RecordDecisionTaskStarted(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRecordDecisionTaskStartedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRecordDecisionTaskStartedScope, metrics.HistoryFailures)
 	}
 
 	return resp, err
@@ -123,7 +123,7 @@ func (c *metricClient) RecordActivityTaskStarted(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRecordActivityTaskStartedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRecordActivityTaskStartedScope, metrics.HistoryFailures)
 	}
 
 	return resp, err
@@ -140,7 +140,7 @@ func (c *metricClient) RespondDecisionTaskCompleted(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRespondDecisionTaskCompletedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRespondDecisionTaskCompletedScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -157,7 +157,7 @@ func (c *metricClient) RespondDecisionTaskFailed(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRespondDecisionTaskFailedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRespondDecisionTaskFailedScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -174,7 +174,7 @@ func (c *metricClient) RespondActivityTaskCompleted(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRespondActivityTaskCompletedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRespondActivityTaskCompletedScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -191,7 +191,7 @@ func (c *metricClient) RespondActivityTaskFailed(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRespondActivityTaskFailedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRespondActivityTaskFailedScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -208,7 +208,7 @@ func (c *metricClient) RespondActivityTaskCanceled(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRespondActivityTaskCanceledScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRespondActivityTaskCanceledScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -225,7 +225,7 @@ func (c *metricClient) RecordActivityTaskHeartbeat(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRecordActivityTaskHeartbeatScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRecordActivityTaskHeartbeatScope, metrics.HistoryFailures)
 	}
 
 	return resp, err
@@ -242,7 +242,7 @@ func (c *metricClient) RequestCancelWorkflowExecution(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRequestCancelWorkflowExecutionScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRequestCancelWorkflowExecutionScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -259,7 +259,7 @@ func (c *metricClient) SignalWorkflowExecution(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientSignalWorkflowExecutionScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientSignalWorkflowExecutionScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -276,7 +276,7 @@ func (c *metricClient) TerminateWorkflowExecution(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientTerminateWorkflowExecutionScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientTerminateWorkflowExecutionScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -293,7 +293,7 @@ func (c *metricClient) ScheduleDecisionTask(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientScheduleDecisionTaskScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientScheduleDecisionTaskScope, metrics.HistoryFailures)
 	}
 
 	return err
@@ -310,7 +310,7 @@ func (c *metricClient) RecordChildExecutionCompleted(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.HistoryClientRecordChildExecutionCompletedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.HistoryClientRecordChildExecutionCompletedScope, metrics.HistoryFailures)
 	}
 
 	return err

diff --git a/client/matching/metricClient.go b/client/matching/metricClient.go
@@ -55,7 +55,7 @@ func (c *metricClient) AddActivityTask(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientAddActivityTaskScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientAddActivityTaskScope, metrics.MatchingFailures)
 	}
 
 	return err
@@ -72,7 +72,7 @@ func (c *metricClient) AddDecisionTask(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientAddDecisionTaskScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientAddDecisionTaskScope, metrics.MatchingFailures)
 	}
 
 	return err
@@ -89,7 +89,7 @@ func (c *metricClient) PollForActivityTask(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientPollForActivityTaskScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientPollForActivityTaskScope, metrics.MatchingFailures)
 	}
 
 	return resp, err
@@ -106,7 +106,7 @@ func (c *metricClient) PollForDecisionTask(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientPollForDecisionTaskScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientPollForDecisionTaskScope, metrics.MatchingFailures)
 	}
 
 	return resp, err
@@ -123,7 +123,7 @@ func (c *metricClient) QueryWorkflow(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientQueryWorkflowScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientQueryWorkflowScope, metrics.MatchingFailures)
 	}
 
 	return resp, err
@@ -140,7 +140,7 @@ func (c *metricClient) RespondQueryTaskCompleted(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientRespondQueryTaskCompletedScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientRespondQueryTaskCompletedScope, metrics.MatchingFailures)
 	}
 
 	return err
@@ -157,7 +157,7 @@ func (c *metricClient) CancelOutstandingPoll(
 	sw.Stop()
 
 	if err != nil {
-		c.metricsClient.IncCounter(metrics.MatchingClientCancelOutstandingPollScope, metrics.CadenceFailures)
+		c.metricsClient.IncCounter(metrics.MatchingClientCancelOutstandingPollScope, metrics.MatchingFailures)
 	}
 
 	return err

diff --git a/common/metrics/defs.go b/common/metrics/defs.go
@@ -542,6 +542,7 @@ const (
 	TaskRequests = iota + NumCommonMetrics
 	TaskFailures
 	TaskLatency
+	HistoryFailures
 	AckLevelUpdateCounter
 	AckLevelUpdateFailedCounter
 	DecisionTypeScheduleActivityCounter
@@ -587,7 +588,8 @@ const (
 
 // Matching metrics enum
 const (
-	PollSuccessCounter = iota + NumCommonMetrics
+	MatchingFailures = iota + NumCommonMetrics
+	PollSuccessCounter
 	PollTimeoutCounter
 	PollErrorsCounter
 	PollSuccessWithSyncCounter
@@ -624,6 +626,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		TaskRequests:                                 {metricName: "task.requests", metricType: Counter},
 		TaskFailures:                                 {metricName: "task.errors", metricType: Counter},
 		TaskLatency:                                  {metricName: "task.latency", metricType: Counter},
+		HistoryFailures:                              {metricName: "history.errors", metricType: Counter},
 		AckLevelUpdateCounter:                        {metricName: "ack-level-update", metricType: Counter},
 		AckLevelUpdateFailedCounter:                  {metricName: "ack-level-update-failed", metricType: Counter},
 		DecisionTypeScheduleActivityCounter:          {metricName: "schedule-activity-decision", metricType: Counter},
@@ -667,6 +670,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		HistoryEventNotificationFailDeliveryCount:    {metricName: "history-event-notification-fail-delivery-count", metricType: Counter},
 	},
 	Matching: {
+		MatchingFailures:              {metricName: "matching.errors", metricType: Counter},
 		PollSuccessCounter:            {metricName: "poll.success"},
 		PollTimeoutCounter:            {metricName: "poll.timeouts"},
 		PollErrorsCounter:             {metricName: "poll.errors"},