From 733b4e078661f3c8a7ae1ae3acf6997c2b462ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Fri, 28 Jul 2023 08:24:29 +0200 Subject: [PATCH] Update opentelemetry-collector-contrib fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore histogram downscaling of #5532 removed by #5531. Bring in latest improvement on Charles' fork. Signed-off-by: György Krajcsovits --- go.mod | 2 +- go.sum | 4 +- .../prometheusremotewrite/helper.go | 4 +- .../prometheusremotewrite/histograms.go | 82 ++++++++++++++----- .../prometheusremotewrite/metrics_to_prw.go | 3 +- .../number_data_points.go | 4 +- vendor/modules.txt | 4 +- 7 files changed, 72 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index c0f41d3875a..373e9ed5b1c 100644 --- a/go.mod +++ b/go.mod @@ -270,7 +270,7 @@ replace github.com/munnerz/goautoneg => github.com/charleskorn/goautoneg v0.0.0- replace github.com/opentracing-contrib/go-stdlib => github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 // Replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite with a fork until https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/24288 is merged. -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717033559-beffb82bb827 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230719070131-77d6f20f8ca9 // Use a fork of weaveworks/common while we work out if there is a better design for https://github.com/weaveworks/common/pull/293 replace github.com/weaveworks/common => github.com/weaveworks/common v0.0.0-20230714173453-d1f8877b91ce diff --git a/go.sum b/go.sum index b6339c36089..beb9a3f3a51 100644 --- a/go.sum +++ b/go.sum @@ -504,8 +504,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charleskorn/goautoneg v0.0.0-20230303030534-7248a2f4c9cc h1:r9yn26N6K60wfQPMwxNeD8U2djqObFYh6o9LXfeMecg= github.com/charleskorn/goautoneg v0.0.0-20230303030534-7248a2f4c9cc/go.mod h1:S666TGglI7cVCVpErP7i+7iFlS330LD8vZDcFk9IdCg= -github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717033559-beffb82bb827 h1:/eFbe8NJWZD9NIaOA8RJ4gjxw7tME7V7t+B3ufyH7iE= -github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717033559-beffb82bb827/go.mod h1:FLUCs/7YJX0pauUBlj+5cy2h0ckjVMno2YyXsO/j89k= +github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230719070131-77d6f20f8ca9 h1:LL9bHJSMlzEgSQFVGMY5PGREjSzqL1f6uvk8E4qoe3c= +github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230719070131-77d6f20f8ca9/go.mod h1:FLUCs/7YJX0pauUBlj+5cy2h0ckjVMno2YyXsO/j89k= github.com/chromedp/cdproto v0.0.0-20210526005521-9e51b9051fd0/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= github.com/chromedp/cdproto v0.0.0-20210706234513-2bc298e8be7f/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= github.com/chromedp/cdproto v0.0.0-20220629234738-4cfc9cdeeb92 h1:0kiAQSLWZDt4wsmcICou3C6in/OJ58FCqvXcB8Ax1Dk= diff --git a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/helper.go b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/helper.go index 68ad2a66500..058f72a1c8d 100644 --- a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/helper.go +++ b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/helper.go @@ -257,7 +257,7 @@ func isValidAggregationTemporality(metric pmetric.Metric) bool { func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings, tsMap map[string]*prompb.TimeSeries) { timestamp := convertTimeStamp(pt.Timestamp()) // sum, count, and buckets of the histogram should append suffix to baseName - baseName := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels) createLabels := func(nameSuffix string, extras ...string) []prompb.Label { @@ -456,7 +456,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res tsMap map[string]*prompb.TimeSeries) { timestamp := convertTimeStamp(pt.Timestamp()) // sum and count of the summary should append suffix to baseName - baseName := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels) createLabels := func(name string, extras ...string) []prompb.Label { diff --git a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/histograms.go b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/histograms.go index 293d318a3c5..ec25abcefcc 100644 --- a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/histograms.go +++ b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/histograms.go @@ -58,15 +58,20 @@ func addSingleExponentialHistogramDataPoint( // to Prometheus Native Histogram. func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prompb.Histogram, error) { scale := p.Scale() - if scale < -4 || scale > 8 { + if scale < -4 { return prompb.Histogram{}, fmt.Errorf("cannot convert exponential to native histogram."+ - " Scale must be <= 8 and >= -4, was %d", scale) - // TODO: downscale to 8 if scale > 8 + " Scale must be >= -4, was %d", scale) } - pSpans, pDeltas := convertBucketsLayout(p.Positive()) - nSpans, nDeltas := convertBucketsLayout(p.Negative()) + var scaleDown int32 + if scale > 8 { + scaleDown = scale - 8 + scale = 8 + } + + pSpans, pDeltas := convertBucketsLayout(p.Positive(), scaleDown) + nSpans, nDeltas := convertBucketsLayout(p.Negative(), scaleDown) h := prompb.Histogram{ Schema: scale, @@ -104,17 +109,19 @@ func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prom // The bucket indexes conversion was adjusted, since OTel exp. histogram bucket // index 0 corresponds to the range (1, base] while Prometheus bucket index 0 // to the range (base 1]. -func convertBucketsLayout(buckets pmetric.ExponentialHistogramDataPointBuckets) ([]prompb.BucketSpan, []int64) { +// +// scaleDown is the factor by which the buckets are scaled down. In other words 2^scaleDown buckets will be merged into one. +func convertBucketsLayout(buckets pmetric.ExponentialHistogramDataPointBuckets, scaleDown int32) ([]prompb.BucketSpan, []int64) { bucketCounts := buckets.BucketCounts() if bucketCounts.Len() == 0 { return nil, nil } var ( - spans []prompb.BucketSpan - deltas []int64 - prevCount int64 - nextBucketIdx int32 + spans []prompb.BucketSpan + deltas []int64 + count int64 + prevCount int64 ) appendDelta := func(count int64) { @@ -123,34 +130,67 @@ func convertBucketsLayout(buckets pmetric.ExponentialHistogramDataPointBuckets) prevCount = count } - for i := 0; i < bucketCounts.Len(); i++ { - count := int64(bucketCounts.At(i)) + // Let the compiler figure out that this is const during this function by + // moving it into a local variable. + numBuckets := bucketCounts.Len() + + // The offset is scaled and adjusted by 1 as described above. + bucketIdx := buckets.Offset()>>scaleDown + 1 + spans = append(spans, prompb.BucketSpan{ + Offset: bucketIdx, + Length: 0, + }) + + for i := 0; i < numBuckets; i++ { + // The offset is scaled and adjusted by 1 as described above. + nextBucketIdx := (int32(i)+buckets.Offset())>>scaleDown + 1 + if bucketIdx == nextBucketIdx { // We have not collected enough buckets to merge yet. + count += int64(bucketCounts.At(i)) + continue + } if count == 0 { + count = int64(bucketCounts.At(i)) continue } - // The offset is adjusted by 1 as described above. - bucketIdx := int32(i) + buckets.Offset() + 1 - delta := bucketIdx - nextBucketIdx - if i == 0 || delta > 2 { - // We have to create a new span, either because we are - // at the very beginning, or because we have found a gap + gap := nextBucketIdx - bucketIdx - 1 + if gap > 2 { + // We have to create a new span, because we have found a gap // of more than two buckets. The constant 2 is copied from the logic in // https://github.com/prometheus/client_golang/blob/27f0506d6ebbb117b6b697d0552ee5be2502c5f2/prometheus/histogram.go#L1296 spans = append(spans, prompb.BucketSpan{ - Offset: delta, + Offset: gap, Length: 0, }) } else { // We have found a small gap (or no gap at all). // Insert empty buckets as needed. - for j := int32(0); j < delta; j++ { + for j := int32(0); j < gap; j++ { appendDelta(0) } } appendDelta(count) - nextBucketIdx = bucketIdx + 1 + count = int64(bucketCounts.At(i)) + bucketIdx = nextBucketIdx + } + // Need to use the last item's index. The offset is scaled and adjusted by 1 as described above. + gap := (int32(numBuckets)+buckets.Offset()-1)>>scaleDown + 1 - bucketIdx + if gap > 2 { + // We have to create a new span, because we have found a gap + // of more than two buckets. The constant 2 is copied from the logic in + // https://github.com/prometheus/client_golang/blob/27f0506d6ebbb117b6b697d0552ee5be2502c5f2/prometheus/histogram.go#L1296 + spans = append(spans, prompb.BucketSpan{ + Offset: gap, + Length: 0, + }) + } else { + // We have found a small gap (or no gap at all). + // Insert empty buckets as needed. + for j := int32(0); j < gap; j++ { + appendDelta(0) + } } + appendDelta(count) return spans, deltas } diff --git a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/metrics_to_prw.go b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/metrics_to_prw.go index 205e279020f..14780dc186f 100644 --- a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/metrics_to_prw.go +++ b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/metrics_to_prw.go @@ -20,6 +20,7 @@ type Settings struct { ExternalLabels map[string]string DisableTargetInfo bool ExportCreatedMetric bool + AddMetricSuffixes bool } // FromMetrics converts pmetric.Metrics to prometheus remote write format. @@ -79,7 +80,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp if dataPoints.Len() == 0 { errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) } - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) for x := 0; x < dataPoints.Len(); x++ { errs = multierr.Append( errs, diff --git a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/number_data_points.go b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/number_data_points.go index d50ed652651..5614424c6ea 100644 --- a/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/number_data_points.go +++ b/vendor/github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite/number_data_points.go @@ -25,7 +25,7 @@ func addSingleGaugeNumberDataPoint( settings Settings, series map[string]*prompb.TimeSeries, ) { - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) labels := createAttributes( resource, pt.Attributes(), @@ -58,7 +58,7 @@ func addSingleSumNumberDataPoint( settings Settings, series map[string]*prompb.TimeSeries, ) { - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) labels := createAttributes( resource, pt.Attributes(), diff --git a/vendor/modules.txt b/vendor/modules.txt index f7aa635d752..3e6a5b13745 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -783,7 +783,7 @@ github.com/oklog/ulid # github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.81.0 ## explicit; go 1.19 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus -# github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717235037-3f2821e2c1b1 => github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717033559-beffb82bb827 +# github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717235037-3f2821e2c1b1 => github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230719070131-77d6f20f8ca9 ## explicit; go 1.19 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite # github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e @@ -1486,5 +1486,5 @@ sigs.k8s.io/yaml # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 # github.com/munnerz/goautoneg => github.com/charleskorn/goautoneg v0.0.0-20230303030534-7248a2f4c9cc # github.com/opentracing-contrib/go-stdlib => github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 -# github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230717033559-beffb82bb827 +# github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => github.com/charleskorn/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.0.0-20230719070131-77d6f20f8ca9 # github.com/weaveworks/common => github.com/weaveworks/common v0.0.0-20230714173453-d1f8877b91ce