From dcb9cc94460db6181ad3dcd6d70ef99224f6c014 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Thu, 30 May 2024 11:40:50 -0700 Subject: [PATCH 1/7] Added DogStatsD extended aggregation support. Signed-off-by: Greg Chambers --- pkg/line/line.go | 17 +++++++++++++++-- pkg/line/line_test.go | 11 +++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pkg/line/line.go b/pkg/line/line.go index 98c70e31..e591b300 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -229,8 +229,21 @@ func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, s return events } - // disable multi-metrics - samples = elements[1:] + // handle DogStatsD extended aggregation + lineParts := strings.SplitN(elements[1], "|", 2) + if strings.Contains(lineParts[0], ":") { + aggValues := strings.Split(lineParts[0], ":") + aggLines := make([]string, len(aggValues)) + tags := lineParts[1] + + for i, aggValue := range aggValues { + aggLines[i] = strings.Join([]string{aggValue, tags}, "|") + } + samples = aggLines + } else { + // disable multi-metrics + samples = elements[1:] + } } else { samples = strings.Split(elements[1], ":") } diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index bd57c9ba..c084b22b 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -403,6 +403,17 @@ func TestLineToEvents(t *testing.T) { "datadog tag extension with both valid and invalid utf8 tag values": { in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", }, + "datadog timings with extended aggregation values": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.0005, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_timing", OValue: 3, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_timing", OValue: 20, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.00001, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + }, + }, "timings with sampling factor": { in: "foo.timing:0.5|ms|@0.1", out: event.Events{ From 921ad0771f2c8637dc70499a8f84e2a9ca43bfb8 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Sun, 2 Jun 2024 11:54:28 -0700 Subject: [PATCH 2/7] Added aggregate type checking with test validation Signed-off-by: Greg Chambers --- pkg/line/line.go | 33 +++++++++++++++++++++++++-------- pkg/line/line_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/pkg/line/line.go b/pkg/line/line.go index e591b300..33d9106a 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -229,17 +229,34 @@ func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, s return events } - // handle DogStatsD extended aggregation - lineParts := strings.SplitN(elements[1], "|", 2) + lineParts := strings.SplitN(elements[1], "|", 3) if strings.Contains(lineParts[0], ":") { - aggValues := strings.Split(lineParts[0], ":") - aggLines := make([]string, len(aggValues)) - tags := lineParts[1] + // handle DogStatsD extended aggregation + var isValidAggType bool + switch lineParts[1] { + case + "ms", // timer + "h", // histogram + "d": // distribution + isValidAggType = true + default: + isValidAggType = false + } + + if isValidAggType { + aggValues := strings.Split(lineParts[0], ":") + aggLines := make([]string, len(aggValues)) - for i, aggValue := range aggValues { - aggLines[i] = strings.Join([]string{aggValue, tags}, "|") + for i, aggValue := range aggValues { + aggLines[i] = strings.Join([]string{aggValue, lineParts[1], lineParts[2]}, "|") + } + + samples = aggLines + } else { + sampleErrors.WithLabelValues("invalid_extended_aggregate_type").Inc() + level.Debug(logger).Log("msg", "Bad line (invalid extended aggregate type) from StatsD", "line", line) + return events } - samples = aggLines } else { // disable multi-metrics samples = elements[1:] diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index c084b22b..56032ddc 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -414,6 +414,34 @@ func TestLineToEvents(t *testing.T) { &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.00001, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, }, }, + "datadog histogram with extended aggregation values": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 0.5, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 3000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 10, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 20000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + }, + }, + "datadog distribution with extended aggregation values": { + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 0.5, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 3000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 10, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 20000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + }, + }, + "datadog counter with invalid extended aggregation values": { + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,#tag2:baz", + }, + "datadog gauge with invalid extended aggregation values": { + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,#tag2:baz", + }, "timings with sampling factor": { in: "foo.timing:0.5|ms|@0.1", out: event.Events{ From 7b4f7310ae5034dc2976f8d0341ba67588429c72 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Sun, 2 Jun 2024 13:35:22 -0700 Subject: [PATCH 3/7] Added tagless and sampling handling in extended aggregation Signed-off-by: Greg Chambers --- pkg/line/line.go | 66 ++++---- pkg/line/line_test.go | 345 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 342 insertions(+), 69 deletions(-) diff --git a/pkg/line/line.go b/pkg/line/line.go index 33d9106a..b051f6f7 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -217,50 +217,46 @@ func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, s labels := map[string]string{} metric := p.parseNameAndTags(elements[0], labels, tagErrors, logger) - - var samples []string - if strings.Contains(elements[1], "|#") { + usingDogStatsDTags := strings.Contains(line, "|#") + if usingDogStatsDTags && len(labels) > 0 { // using DogStatsD tags // don't allow mixed tagging styles - if len(labels) > 0 { - sampleErrors.WithLabelValues("mixed_tagging_styles").Inc() - level.Debug(logger).Log("msg", "Bad line (multiple tagging styles) from StatsD", "line", line) - return events - } - - lineParts := strings.SplitN(elements[1], "|", 3) - if strings.Contains(lineParts[0], ":") { - // handle DogStatsD extended aggregation - var isValidAggType bool - switch lineParts[1] { - case - "ms", // timer - "h", // histogram - "d": // distribution - isValidAggType = true - default: - isValidAggType = false - } + sampleErrors.WithLabelValues("mixed_tagging_styles").Inc() + level.Debug(logger).Log("msg", "Bad line (multiple tagging styles) from StatsD", "line", line) + return events + } - if isValidAggType { - aggValues := strings.Split(lineParts[0], ":") - aggLines := make([]string, len(aggValues)) + var samples []string + lineParts := strings.SplitN(elements[1], "|", 3) + if strings.Contains(lineParts[0], ":") { + // handle DogStatsD extended aggregation + isValidAggType := false + switch lineParts[1] { + case + "ms", // timer + "h", // histogram + "d": // distribution + isValidAggType = true + } - for i, aggValue := range aggValues { - aggLines[i] = strings.Join([]string{aggValue, lineParts[1], lineParts[2]}, "|") - } + if isValidAggType { + aggValues := strings.Split(lineParts[0], ":") + aggLines := make([]string, len(aggValues)) + _, aggLineSuffix, _ := strings.Cut(elements[1], "|") - samples = aggLines - } else { - sampleErrors.WithLabelValues("invalid_extended_aggregate_type").Inc() - level.Debug(logger).Log("msg", "Bad line (invalid extended aggregate type) from StatsD", "line", line) - return events + for i, aggValue := range aggValues { + aggLines[i] = strings.Join([]string{aggValue, aggLineSuffix}, "|") } + samples = aggLines } else { - // disable multi-metrics - samples = elements[1:] + sampleErrors.WithLabelValues("invalid_extended_aggregate_type").Inc() + level.Debug(logger).Log("msg", "Bad line (invalid extended aggregate type) from StatsD", "line", line) + return events } + } else if usingDogStatsDTags { + // disable multi-metrics + samples = elements[1:] } else { samples = strings.Split(elements[1], ":") } diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index 56032ddc..a6853980 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -404,57 +404,334 @@ func TestLineToEvents(t *testing.T) { in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", }, "datadog timings with extended aggregation values": { - in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,#tag2:baz", + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz", out: event.Events{ - &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.0005, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_timing", OValue: 3, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_timing", OValue: 20, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.00001, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog timings with extended aggregation values without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, }, }, - "datadog histogram with extended aggregation values": { - in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,#tag2:baz", + "datadog timings with extended aggregation values and sampling but without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5", out: event.Events{ - &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 0.5, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 3000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 10, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 20000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_histogram", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values, sampling, and tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog histogram with extended aggregation values and tags": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, }, }, "datadog distribution with extended aggregation values": { - in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,#tag2:baz", + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz", out: event.Events{ - &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 0.5, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 3000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 10, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 20000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, - &event.ObserverEvent{OMetricName: "foo_distribution", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}}, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, }, }, "datadog counter with invalid extended aggregation values": { - in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,#tag2:baz", + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz", }, "datadog gauge with invalid extended aggregation values": { - in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,#tag2:baz", + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", }, "timings with sampling factor": { in: "foo.timing:0.5|ms|@0.1", out: event.Events{ - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, - &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo.timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, }, }, "bad line": { From ba68944beca543d274fcbba5ca0142f8840fc551 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Sun, 2 Jun 2024 13:42:16 -0700 Subject: [PATCH 4/7] Made dogstatsd tag check consistent with previous logic Signed-off-by: Greg Chambers --- pkg/line/line.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/line/line.go b/pkg/line/line.go index b051f6f7..949a5ea7 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -217,7 +217,7 @@ func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, s labels := map[string]string{} metric := p.parseNameAndTags(elements[0], labels, tagErrors, logger) - usingDogStatsDTags := strings.Contains(line, "|#") + usingDogStatsDTags := strings.Contains(elements[1], "|#") if usingDogStatsDTags && len(labels) > 0 { // using DogStatsD tags From da94065b4388121f6f5faaed6f7106631092d614 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Sun, 2 Jun 2024 14:15:07 -0700 Subject: [PATCH 5/7] Added tests to cover invalid extended aggregation in signal fx format Signed-off-by: Greg Chambers --- pkg/line/line_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index a6853980..c07c3337 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -679,6 +679,18 @@ func TestLineToEvents(t *testing.T) { "datadog gauge with invalid extended aggregation values": { in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", }, + "datadog timing with extended aggregation values and invalid signalfx tags": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|ms", + }, + "SignalFX counter with invalid Datadog style extended aggregation values": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags counter with invalid Datadog style extended aggregation values": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", + }, "timings with sampling factor": { in: "foo.timing:0.5|ms|@0.1", out: event.Events{ From 468af47dfb5365df43ae61fbd0c8091660018a35 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Sun, 2 Jun 2024 14:23:53 -0700 Subject: [PATCH 6/7] Added test for invalid Influx style extended aggregation Signed-off-by: Greg Chambers --- pkg/line/line_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index c07c3337..54bd34ea 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -691,6 +691,12 @@ func TestLineToEvents(t *testing.T) { "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", }, + "Influx no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and histogram type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, "timings with sampling factor": { in: "foo.timing:0.5|ms|@0.1", out: event.Events{ From 96609f9679b2ed8627ba0262d00d5c92fe225bc7 Mon Sep 17 00:00:00 2001 From: Greg Chambers Date: Sun, 2 Jun 2024 14:42:34 -0700 Subject: [PATCH 7/7] Added test extended aggregation test set to other test set functions Signed-off-by: Greg Chambers --- pkg/line/line_test.go | 2382 +++++++++++++++++++++++++++++++++-------- 1 file changed, 1926 insertions(+), 456 deletions(-) diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index 54bd34ea..970320c6 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -1031,6 +1031,1363 @@ func TestDisableParsingLineToEvents(t *testing.T) { }, }, }, + "datadog timings with extended aggregation values": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values and sampling but without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values, sampling, and tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog histogram with extended aggregation values and tags": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.5, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 3000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 10, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 20000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.01, + OLabels: map[string]string{}, + }, + }, + }, + "datadog distribution with extended aggregation values": { + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.5, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 3000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 10, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 20000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.01, + OLabels: map[string]string{}, + }, + }, + }, + "datadog counter with invalid extended aggregation values": { + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz", + }, + "datadog gauge with invalid extended aggregation values": { + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", + }, + "datadog timing with extended aggregation values and invalid signalfx tags": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|ms", + }, + "SignalFX counter with invalid Datadog style extended aggregation values": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags counter with invalid Datadog style extended aggregation values": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and histogram type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values and sampling but without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values, sampling, and tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog histogram with extended aggregation values and tags": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.5, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 3000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 10, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 20000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.01, + OLabels: map[string]string{}, + }, + }, + }, + "datadog distribution with extended aggregation values": { + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.5, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 3000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 10, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 20000, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.01, + OLabels: map[string]string{}, + }, + }, + }, + "datadog counter with invalid extended aggregation values": { + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz", + }, + "datadog gauge with invalid extended aggregation values": { + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", + }, + "datadog timing with extended aggregation values and invalid signalfx tags": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|ms", + }, + "SignalFX counter with invalid Datadog style extended aggregation values": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags counter with invalid Datadog style extended aggregation values": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and histogram type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog timings with extended aggregation values": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog timings with extended aggregation values without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values and sampling but without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + }, + }, + "datadog timings with extended aggregation values, sampling, and tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog histogram with extended aggregation values and tags": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog distribution with extended aggregation values": { + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog counter with invalid extended aggregation values": { + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz", + }, + "datadog gauge with invalid extended aggregation values": { + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", + }, + "datadog timing with extended aggregation values and invalid signalfx tags": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|ms", + }, + "SignalFX counter with invalid Datadog style extended aggregation values": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags counter with invalid Datadog style extended aggregation values": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and histogram type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, "librato/dogstatsd mixed tag styles without sampling": { in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", out: event.Events{}, @@ -1053,7 +2410,7 @@ func TestDisableParsingLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 1000, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag1": "foo:bar"}, }, }, }, @@ -1066,6 +2423,9 @@ func TestDisableParsingLineToEvents(t *testing.T) { } parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { @@ -1080,7 +2440,7 @@ func TestDisableParsingLineToEvents(t *testing.T) { } } -func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { +func TestDisableParsingSignalfxLineToEvents(t *testing.T) { type testCase struct { in string out event.Events @@ -1117,33 +2477,33 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { }, }, }, - "SignalFx tag extension": { + "SignalFx tag extension": { // parsed as influxdb tags in: "foo.[tag1=bar,tag2=baz]test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.test", + CMetricName: "foo.[tag1=bar", CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + CLabels: map[string]string{"tag2": "baz]test"}, }, }, }, - "SignalFx tag extension, tags at end of name": { + "SignalFx tag extension, tags at end of name": { // parsed as influxdb tags in: "foo.test[tag1=bar,tag2=baz]:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.test", + CMetricName: "foo.test[tag1=bar", CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + CLabels: map[string]string{"tag2": "baz]"}, }, }, }, - "SignalFx tag extension, tags at beginning of name": { + "SignalFx tag extension, tags at beginning of name": { // parsed as influxdb tags in: "[tag1=bar,tag2=baz]foo.test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.test", + CMetricName: "[tag1=bar", CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + CLabels: map[string]string{"tag2": "baz]foo.test"}, }, }, }, @@ -1151,39 +2511,39 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { in: "foo.[]test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.test", + CMetricName: "foo.[]test", CValue: 100, CLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, non-kv tags": { + "SignalFx tag extension, non-kv tags": { // parsed as influxdb tags in: "foo.[tag1,tag2]test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.test", + CMetricName: "foo.[tag1", CValue: 100, CLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, missing closing bracket": { + "SignalFx tag extension, missing closing bracket": { // parsed as influxdb tags in: "[tag1=bar,tag2=bazfoo.test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "[tag1=bar,tag2=bazfoo.test", + CMetricName: "[tag1=bar", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag2": "bazfoo.test"}, }, }, }, - "SignalFx tag extension, missing opening bracket": { + "SignalFx tag extension, missing opening bracket": { // parsed as influxdb tags in: "tag1=bar,tag2=baz]foo.test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "tag1=bar,tag2=baz]foo.test", + CMetricName: "tag1=bar", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag2": "baz]foo.test"}, }, }, }, @@ -1203,7 +2563,7 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, @@ -1213,7 +2573,7 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, @@ -1223,7 +2583,7 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, }, }, }, @@ -1243,7 +2603,7 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag": "value"}, }, }, }, @@ -1253,7 +2613,7 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 100, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag": "value"}, }, }, }, @@ -1263,249 +2623,304 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { &event.CounterEvent{ CMetricName: "foo", CValue: 1000, - CLabels: map[string]string{}, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "librato/dogstatsd mixed tag styles without sampling": { - in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "signalfx/dogstatsd mixed tag styles without sampling": { - in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "influxdb/dogstatsd mixed tag styles without sampling": { - in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "mixed tag styles with sampling": { - in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "datadog tag extension with multiple colons": { - in: "foo:100|c|@0.1|#tag1:foo:bar", + "datadog timings with extended aggregation values": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 1000, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with invalid utf8 tag values": { - in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", - }, - "datadog tag extension with both valid and invalid utf8 tag values": { - in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", - }, - } - - parser := NewParser() - parser.EnableInfluxdbParsing() - parser.EnableLibratoParsing() - parser.EnableSignalFXParsing() - - for name, testCase := range testCases { - t.Run(name, func(t *testing.T) { - events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) - - for j, expected := range testCase.out { - if !reflect.DeepEqual(&expected, &events[j]) { - t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) - } - } - }) - } -} - -func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { - type testCase struct { - in string - out event.Events - } - - testCases := map[string]testCase{ - "librato tag extension": { - in: "foo#tag1=bar,tag2=baz:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "librato tag extension with tag keys unsupported by prometheus": { - in: "foo#09digits=0,tag.with.dots=1:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "influxdb tag extension": { - in: "foo,tag1=bar,tag2=baz:100|c", + "datadog timings with extended aggregation values without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo,tag1=bar,tag2=baz", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, }, }, }, - "SignalFx tag extension": { - in: "foo.[tag1=bar,tag2=baz]test:100|c", + "datadog timings with extended aggregation values and sampling but without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, tags at end of name": { - in: "foo.test[tag1=bar,tag2=baz]:100|c", + "datadog timings with extended aggregation values, sampling, and tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, tags at beginning of name": { - in: "[tag1=bar,tag2=baz]foo.test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, no tags": { - in: "foo.[]test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, non-kv tags": { - in: "foo.[tag1,tag2]test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, missing closing bracket": { - in: "[tag1=bar,tag2=bazfoo.test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "[tag1=bar,tag2=bazfoo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, missing opening bracket": { - in: "tag1=bar,tag2=baz]foo.test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "tag1=bar,tag2=baz]foo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "influxdb tag extension with tag keys unsupported by prometheus": { - in: "foo,09digits=0,tag.with.dots=1:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo,09digits=0,tag.with.dots=1", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension": { - in: "foo:100|c|#tag1:bar,tag2:baz", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with # in all keys (as sent by datadog php client)": { - in: "foo:100|c|#tag1:bar,#tag2:baz", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with tag keys unsupported by prometheus": { - in: "foo:100|c|#09digits:0,tag.with.dots:1", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with valueless tags: ignored": { - in: "foo:100|c|#tag_without_a_value", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with valueless tags (edge case)": { - in: "foo:100|c|#tag_without_a_value,tag:value", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag": "value"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "datadog tag extension with empty tags (edge case)": { - in: "foo:100|c|#tag:value,,", + "datadog histogram with extended aggregation values and tags": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag": "value"}, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "datadog tag extension with sampling": { - in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + "datadog distribution with extended aggregation values": { + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 1000, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, + "datadog counter with invalid extended aggregation values": { + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz", + }, + "datadog gauge with invalid extended aggregation values": { + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", + }, + "datadog timing with extended aggregation values and invalid signalfx tags": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|ms", + }, + "SignalFX counter with invalid Datadog style extended aggregation values": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags counter with invalid Datadog style extended aggregation values": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and histogram type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, "librato/dogstatsd mixed tag styles without sampling": { in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", out: event.Events{}, @@ -1542,8 +2957,8 @@ func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { parser := NewParser() parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() parser.EnableLibratoParsing() - parser.EnableSignalFXParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { @@ -1558,30 +2973,30 @@ func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { } } -func TestDisableParsingSignalfxLineToEvents(t *testing.T) { +func TestDisableParsingLibratoLineToEvents(t *testing.T) { type testCase struct { in string out event.Events } testCases := map[string]testCase{ - "librato tag extension": { + "librato tag extension": { // parsed as influxdb tags in: "foo#tag1=bar,tag2=baz:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo", + CMetricName: "foo#tag1=bar", CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + CLabels: map[string]string{"tag2": "baz"}, }, }, }, - "librato tag extension with tag keys unsupported by prometheus": { + "librato tag extension with tag keys unsupported by prometheus": { // parsed as influxdb tags in: "foo#09digits=0,tag.with.dots=1:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo", + CMetricName: "foo#09digits=0", CValue: 100, - CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + CLabels: map[string]string{"tag_with_dots": "1"}, }, }, }, @@ -1595,33 +3010,33 @@ func TestDisableParsingSignalfxLineToEvents(t *testing.T) { }, }, }, - "SignalFx tag extension": { // parsed as influxdb tags + "SignalFx tag extension": { in: "foo.[tag1=bar,tag2=baz]test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.[tag1=bar", + CMetricName: "foo.test", CValue: 100, - CLabels: map[string]string{"tag2": "baz]test"}, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "SignalFx tag extension, tags at end of name": { // parsed as influxdb tags + "SignalFx tag extension, tags at end of name": { in: "foo.test[tag1=bar,tag2=baz]:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.test[tag1=bar", + CMetricName: "foo.test", CValue: 100, - CLabels: map[string]string{"tag2": "baz]"}, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "SignalFx tag extension, tags at beginning of name": { // parsed as influxdb tags + "SignalFx tag extension, tags at beginning of name": { in: "[tag1=bar,tag2=baz]foo.test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "[tag1=bar", + CMetricName: "foo.test", CValue: 100, - CLabels: map[string]string{"tag2": "baz]foo.test"}, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, @@ -1629,39 +3044,39 @@ func TestDisableParsingSignalfxLineToEvents(t *testing.T) { in: "foo.[]test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.[]test", + CMetricName: "foo.test", CValue: 100, CLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, non-kv tags": { // parsed as influxdb tags + "SignalFx tag extension, non-kv tags": { in: "foo.[tag1,tag2]test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "foo.[tag1", + CMetricName: "foo.test", CValue: 100, CLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, missing closing bracket": { // parsed as influxdb tags + "SignalFx tag extension, missing closing bracket": { in: "[tag1=bar,tag2=bazfoo.test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "[tag1=bar", + CMetricName: "[tag1=bar,tag2=bazfoo.test", CValue: 100, - CLabels: map[string]string{"tag2": "bazfoo.test"}, + CLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, missing opening bracket": { // parsed as influxdb tags + "SignalFx tag extension, missing opening bracket": { in: "tag1=bar,tag2=baz]foo.test:100|c", out: event.Events{ &event.CounterEvent{ - CMetricName: "tag1=bar", + CMetricName: "tag1=bar,tag2=baz]foo.test", CValue: 100, - CLabels: map[string]string{"tag2": "baz]foo.test"}, + CLabels: map[string]string{}, }, }, }, @@ -1745,245 +3160,300 @@ func TestDisableParsingSignalfxLineToEvents(t *testing.T) { }, }, }, - "librato/dogstatsd mixed tag styles without sampling": { - in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "signalfx/dogstatsd mixed tag styles without sampling": { - in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "influxdb/dogstatsd mixed tag styles without sampling": { - in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "mixed tag styles with sampling": { - in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", - out: event.Events{}, - }, - "datadog tag extension with multiple colons": { - in: "foo:100|c|@0.1|#tag1:foo:bar", + "datadog timings with extended aggregation values": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 1000, - CLabels: map[string]string{"tag1": "foo:bar"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with invalid utf8 tag values": { - in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", - }, - "datadog tag extension with both valid and invalid utf8 tag values": { - in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", - }, - } - - parser := NewParser() - parser.EnableDogstatsdParsing() - parser.EnableInfluxdbParsing() - parser.EnableLibratoParsing() - - for name, testCase := range testCases { - t.Run(name, func(t *testing.T) { - events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) - - for j, expected := range testCase.out { - if !reflect.DeepEqual(&expected, &events[j]) { - t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) - } - } - }) - } -} - -func TestDisableParsingLibratoLineToEvents(t *testing.T) { - type testCase struct { - in string - out event.Events - } - - testCases := map[string]testCase{ - "librato tag extension": { // parsed as influxdb tags - in: "foo#tag1=bar,tag2=baz:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo#tag1=bar", - CValue: 100, - CLabels: map[string]string{"tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "librato tag extension with tag keys unsupported by prometheus": { // parsed as influxdb tags - in: "foo#09digits=0,tag.with.dots=1:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo#09digits=0", - CValue: 100, - CLabels: map[string]string{"tag_with_dots": "1"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "influxdb tag extension": { - in: "foo,tag1=bar,tag2=baz:100|c", + "datadog timings with extended aggregation values without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, }, }, }, - "SignalFx tag extension": { - in: "foo.[tag1=bar,tag2=baz]test:100|c", + "datadog timings with extended aggregation values and sampling but without tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, + }, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{}, }, }, }, - "SignalFx tag extension, tags at end of name": { - in: "foo.test[tag1=bar,tag2=baz]:100|c", + "datadog timings with extended aggregation values, sampling, and tags": { + in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, tags at beginning of name": { - in: "[tag1=bar,tag2=baz]foo.test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.0005, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, no tags": { - in: "foo.[]test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, non-kv tags": { - in: "foo.[tag1,tag2]test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, missing closing bracket": { - in: "[tag1=bar,tag2=bazfoo.test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "[tag1=bar,tag2=bazfoo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "SignalFx tag extension, missing opening bracket": { - in: "tag1=bar,tag2=baz]foo.test:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "tag1=bar,tag2=baz]foo.test", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 3, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "influxdb tag extension with tag keys unsupported by prometheus": { - in: "foo,09digits=0,tag.with.dots=1:100|c", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension": { - in: "foo:100|c|#tag1:bar,tag2:baz", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with # in all keys (as sent by datadog php client)": { - in: "foo:100|c|#tag1:bar,#tag2:baz", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with tag keys unsupported by prometheus": { - in: "foo:100|c|#09digits:0,tag.with.dots:1", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 20, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with valueless tags: ignored": { - in: "foo:100|c|#tag_without_a_value", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, - }, - }, - "datadog tag extension with valueless tags (edge case)": { - in: "foo:100|c|#tag_without_a_value,tag:value", - out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag": "value"}, + &event.ObserverEvent{ + OMetricName: "foo_timing", + OValue: 0.00001, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "datadog tag extension with empty tags (edge case)": { - in: "foo:100|c|#tag:value,,", + "datadog histogram with extended aggregation values and tags": { + in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 100, - CLabels: map[string]string{"tag": "value"}, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_histogram", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, - "datadog tag extension with sampling": { - in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + "datadog distribution with extended aggregation values": { + in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz", out: event.Events{ - &event.CounterEvent{ - CMetricName: "foo", - CValue: 1000, - CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.5, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 120, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 3000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 10, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 20000, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo_distribution", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, + "datadog counter with invalid extended aggregation values": { + in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz", + }, + "datadog gauge with invalid extended aggregation values": { + in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz", + }, + "datadog timing with extended aggregation values and invalid signalfx tags": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|ms", + }, + "SignalFX counter with invalid Datadog style extended aggregation values": { + in: "foo.[tag1=bar,tag2=baz]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags counter with invalid Datadog style extended aggregation values": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|c", + }, + "SignalFX no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.[]test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and timings type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, + "Influx no tags with invalid Datadog style extended aggregation values and histogram type": { + in: "foo.test:0.5:120:3000:10:20000:0.01|ms", + }, "librato/dogstatsd mixed tag styles without sampling": { in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", out: event.Events{},