From 4674f199133c029189db72cfcea1e72adcb91af6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 30 Oct 2018 15:38:49 -0700 Subject: [PATCH 1/2] fix prometheus concurrent map write bug fixes #4132 License: MIT Signed-off-by: Steven Allen --- cmd/ipfs/daemon.go | 4 +-- core/corehttp/metrics.go | 73 ++++++++++++++++++++++++++++++++++++++-- package.json | 8 ++--- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 42031c43b7c..bc9da9d2d60 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -23,11 +23,11 @@ import ( fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" + mprome "gx/ipfs/QmQXBfkuwgMaPx334WuL9NmyrKnbZ5udaWnHTHEsts2x3T/go-metrics-prometheus" cmds "gx/ipfs/QmSXUokcP4TJpFfqozT69AVAYRtzXVMUjzQVkYX41R9Svs/go-ipfs-cmds" ma "gx/ipfs/QmT4U94DnD8FRfqr21obWY32HLM5VExccPKMjQHofeYqr9/go-multiaddr" - "gx/ipfs/QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH/client_golang/prometheus" + "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus" "gx/ipfs/Qmaabb1tJZ2CX5cp6MuuiGgns71NYoxdgQP6Xdid1dVceC/go-multiaddr-net" - mprome "gx/ipfs/QmbqQP7GMwJCePfEvbMSZmyPifAz1kKZifo8vwVnVHt1wL/go-metrics-prometheus" "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" ) diff --git a/core/corehttp/metrics.go b/core/corehttp/metrics.go index 0428c8fd123..df7fba6700d 100644 --- a/core/corehttp/metrics.go +++ b/core/corehttp/metrics.go @@ -6,13 +6,14 @@ import ( core "github.com/ipfs/go-ipfs/core" - prometheus "gx/ipfs/QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH/client_golang/prometheus" + prometheus "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus" + promhttp "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus/promhttp" ) // This adds the scraping endpoint which Prometheus uses to fetch metrics. func MetricsScrapingOption(path string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - mux.Handle(path, prometheus.UninstrumentedHandler()) + mux.Handle(path, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) return mux, nil } } @@ -20,8 +21,74 @@ func MetricsScrapingOption(path string) ServeOption { // This adds collection of net/http-related metrics func MetricsCollectionOption(handlerName string) ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + // Adapted from github.com/prometheus/client_golang/prometheus/http.go + // Work around https://github.com/prometheus/client_golang/pull/311 + opts := prometheus.SummaryOpts{ + Subsystem: "http", + ConstLabels: prometheus.Labels{"handler": handlerName}, + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + } + + reqCnt := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "requests_total", + Help: "Total number of HTTP requests made.", + ConstLabels: opts.ConstLabels, + }, + []string{"method", "code"}, + ) + if err := prometheus.Register(reqCnt); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + reqCnt = are.ExistingCollector.(*prometheus.CounterVec) + } else { + return nil, err + } + } + + opts.Name = "request_duration_microseconds" + opts.Help = "The HTTP request latencies in microseconds." + reqDur := prometheus.NewSummaryVec(opts, nil) + if err := prometheus.Register(reqDur); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + reqDur = are.ExistingCollector.(*prometheus.SummaryVec) + } else { + return nil, err + } + } + + opts.Name = "request_size_bytes" + opts.Help = "The HTTP request sizes in bytes." + reqSz := prometheus.NewSummaryVec(opts, nil) + if err := prometheus.Register(reqSz); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + reqSz = are.ExistingCollector.(*prometheus.SummaryVec) + } else { + return nil, err + } + } + + opts.Name = "response_size_bytes" + opts.Help = "The HTTP response sizes in bytes." + resSz := prometheus.NewSummaryVec(opts, nil) + if err := prometheus.Register(resSz); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + resSz = are.ExistingCollector.(*prometheus.SummaryVec) + } else { + return nil, err + } + } + + // Construct the mux childMux := http.NewServeMux() - mux.HandleFunc("/", prometheus.InstrumentHandler(handlerName, childMux)) + var promMux http.Handler = childMux + promMux = promhttp.InstrumentHandlerResponseSize(resSz, promMux) + promMux = promhttp.InstrumentHandlerRequestSize(reqSz, promMux) + promMux = promhttp.InstrumentHandlerDuration(reqDur, promMux) + promMux = promhttp.InstrumentHandlerCounter(reqCnt, promMux) + mux.Handle("/", promMux) + return childMux, nil } } diff --git a/package.json b/package.json index 0b28595a4b7..9bfb5bf19e2 100644 --- a/package.json +++ b/package.json @@ -42,9 +42,9 @@ "version": "0.0.0" }, { - "hash": "QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH", + "hash": "QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS", "name": "client_golang", - "version": "0.1.3" + "version": "0.1.4" }, { "hash": "QmV1DPm5F46LvQMxCVPhu35zHgZEeMvyVtpxjb5TwfGiua", @@ -160,9 +160,9 @@ }, { "author": "ipfs", - "hash": "QmbqQP7GMwJCePfEvbMSZmyPifAz1kKZifo8vwVnVHt1wL", + "hash": "QmQXBfkuwgMaPx334WuL9NmyrKnbZ5udaWnHTHEsts2x3T", "name": "go-metrics-prometheus", - "version": "0.3.9" + "version": "0.3.10" }, { "author": "ipfs", From bf69017e251e9369bdde6961f991746b9d34e30b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 31 Oct 2018 05:12:35 -0700 Subject: [PATCH 2/2] metrics: we're now recording latency seconds, not microseconds License: MIT Signed-off-by: Steven Allen --- core/corehttp/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/corehttp/metrics.go b/core/corehttp/metrics.go index df7fba6700d..6be50dbafe7 100644 --- a/core/corehttp/metrics.go +++ b/core/corehttp/metrics.go @@ -47,8 +47,8 @@ func MetricsCollectionOption(handlerName string) ServeOption { } } - opts.Name = "request_duration_microseconds" - opts.Help = "The HTTP request latencies in microseconds." + opts.Name = "request_duration_seconds" + opts.Help = "The HTTP request latencies in seconds." reqDur := prometheus.NewSummaryVec(opts, nil) if err := prometheus.Register(reqDur); err != nil { if are, ok := err.(prometheus.AlreadyRegisteredError); ok {