Merge pull request #72 from jacksontj/master

Fixes for crashes with conflicting metric values
prometheus · Jul 18, 2017 · 8b40f78 · 8b40f78
2 parents 663b6a1 + bbd5227
commit 8b40f78
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 27 deletions.
diff --git a/exporter.go b/exporter.go
@@ -69,7 +69,7 @@ func NewCounterContainer() *CounterContainer {
 	}
 }
 
-func (c *CounterContainer) Get(metricName string, labels prometheus.Labels) prometheus.Counter {
+func (c *CounterContainer) Get(metricName string, labels prometheus.Labels) (prometheus.Counter, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	counter, ok := c.Elements[hash]
 	if !ok {
@@ -78,12 +78,12 @@ func (c *CounterContainer) Get(metricName string, labels prometheus.Labels) prom
 			Help:        defaultHelp,
 			ConstLabels: labels,
 		})
-		c.Elements[hash] = counter
 		if err := prometheus.Register(counter); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
+		c.Elements[hash] = counter
 	}
-	return counter
+	return counter, nil
 }
 
 type GaugeContainer struct {
@@ -96,7 +96,7 @@ func NewGaugeContainer() *GaugeContainer {
 	}
 }
 
-func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels) prometheus.Gauge {
+func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels) (prometheus.Gauge, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	gauge, ok := c.Elements[hash]
 	if !ok {
@@ -105,12 +105,12 @@ func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels) promet
 			Help:        defaultHelp,
 			ConstLabels: labels,
 		})
-		c.Elements[hash] = gauge
 		if err := prometheus.Register(gauge); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
+		c.Elements[hash] = gauge
 	}
-	return gauge
+	return gauge, nil
 }
 
 type SummaryContainer struct {
@@ -123,7 +123,7 @@ func NewSummaryContainer() *SummaryContainer {
 	}
 }
 
-func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) prometheus.Summary {
+func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) (prometheus.Summary, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	summary, ok := c.Elements[hash]
 	if !ok {
@@ -133,12 +133,12 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) prom
 				Help:        defaultHelp,
 				ConstLabels: labels,
 			})
-		c.Elements[hash] = summary
 		if err := prometheus.Register(summary); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
+		c.Elements[hash] = summary
 	}
-	return summary
+	return summary, nil
 }
 
 type Event interface {
@@ -233,43 +233,58 @@ func (b *Exporter) Listen(e <-chan Events) {
 
 			switch ev := event.(type) {
 			case *CounterEvent:
-				counter := b.Counters.Get(
-					b.suffix(metricName, "counter"),
-					prometheusLabels,
-				)
 				// We don't accept negative values for counters. Incrementing the counter with a negative number
 				// will cause the exporter to panic. Instead we will warn and continue to the next event.
 				if event.Value() < 0.0 {
 					log.Errorf("Counter %q is: '%f' (counter must be non-negative value)", metricName, event.Value())
 					continue
 				}
 
-				counter.Add(event.Value())
+				counter, err := b.Counters.Get(
+					b.suffix(metricName, "counter"),
+					prometheusLabels,
+				)
+				if err == nil {
+					counter.Add(event.Value())
 
-				eventStats.WithLabelValues("counter").Inc()
+					eventStats.WithLabelValues("counter").Inc()
+				} else {
+					log.Errorf(regErrF, metricName, err)
+					conflictingEventStats.WithLabelValues("counter").Inc()
+				}
 
 			case *GaugeEvent:
-				gauge := b.Gauges.Get(
+				gauge, err := b.Gauges.Get(
 					b.suffix(metricName, "gauge"),
 					prometheusLabels,
 				)
 
-				if ev.relative {
-					gauge.Add(event.Value())
+				if err == nil {
+					if ev.relative {
+						gauge.Add(event.Value())
+					} else {
+						gauge.Set(event.Value())
+					}
+
+					eventStats.WithLabelValues("gauge").Inc()
 				} else {
-					gauge.Set(event.Value())
+					log.Errorf(regErrF, metricName, err)
+					conflictingEventStats.WithLabelValues("gauge").Inc()
 				}
 
-				eventStats.WithLabelValues("gauge").Inc()
-
 			case *TimerEvent:
-				summary := b.Summaries.Get(
+				summary, err := b.Summaries.Get(
 					b.suffix(metricName, "timer"),
 					prometheusLabels,
 				)
-				summary.Observe(event.Value())
+				if err == nil {
+					summary.Observe(event.Value())
 
-				eventStats.WithLabelValues("timer").Inc()
+					eventStats.WithLabelValues("timer").Inc()
+				} else {
+					log.Errorf(regErrF, metricName, err)
+					conflictingEventStats.WithLabelValues("timer").Inc()
+				}
 
 			default:
 				log.Errorln("Unsupported event type")

diff --git a/telemetry.go b/telemetry.go
@@ -47,11 +47,19 @@ var (
 		Name: "statsd_exporter_loaded_mappings",
 		Help: "The current number of configured metric mappings.",
 	})
+	conflictingEventStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "statsd_exporter_events_conflict_total",
+			Help: "The total number of StatsD events with conflicting names.",
+		},
+		[]string{"type"},
+	)
 )
 
 func init() {
 	prometheus.MustRegister(eventStats)
 	prometheus.MustRegister(networkStats)
 	prometheus.MustRegister(configLoads)
 	prometheus.MustRegister(mappingsCount)
+	prometheus.MustRegister(conflictingEventStats)
 }