feat: Add redis connection metrics (close #127)

c.go

@@ -105,7 +105,7 @@ func WithRemote(r *remote.Remote) (CoreOption, CoreOption) {
 }
 
 // WithInline is a CoreOption that creates a inline config in the configuration stack.
-func WithInline(key, entry string) CoreOption {
+func WithInline(key string, entry interface{}) CoreOption {
 	return WithConfigStack(confmap.Provider(map[string]interface{}{
 		key: entry,
 	}, "."), nil)

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/go-kit/kit v0.10.0
 	github.com/go-redis/redis/v8 v8.6.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/mock v1.3.1
+	github.com/golang/mock v1.5.0
 	github.com/golang/protobuf v1.4.3
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -185,6 +185,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

observability/metrics.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/DoNewsCode/core/contract"
 	"github.com/DoNewsCode/core/otgorm"
+	"github.com/DoNewsCode/core/otredis"
 	"github.com/go-kit/kit/metrics"
 	"github.com/go-kit/kit/metrics/prometheus"
 	stdprometheus "github.com/prometheus/client_golang/prometheus"
@@ -56,3 +57,46 @@ func ProvideGORMMetrics(appName contract.AppName, env contract.Env) *otgorm.Gaug
 		}, []string{"dbname", "driver"}),
 	}
 }
+
+// ProvideRedisMetrics returns a *otredis.Gauges that measures the connection info in redis.
+// It is meant to be consumed by the otredis.Providers.
+func ProvideRedisMetrics(appName contract.AppName, env contract.Env) *otredis.Gauges {
+	return &otredis.Gauges{
+		Hits: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "redis_hit_connections",
+			Help:      "number of times free connection was found in the pool",
+		}, []string{"dbname"}),
+		Misses: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "redis_miss_connections",
+			Help:      "number of times free connection was NOT found in the pool",
+		}, []string{"dbname"}),
+		Timeouts: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "redis_timeout_connections",
+			Help:      "number of times a wait timeout occurred",
+		}, []string{"dbname"}),
+		TotalConns: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "redis_total_connections",
+			Help:      "number of total connections in the pool",
+		}, []string{"dbname"}),
+		IdleConns: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "redis_idle_connections",
+			Help:      "number of idle connections in the pool",
+		}, []string{"dbname"}),
+		StaleConns: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: appName.String(),
+			Subsystem: env.String(),
+			Name:      "redis_stale_connections",
+			Help:      "number of stale connections removed from the pool",
+		}, []string{"dbname"}),
+	}
+}

observability/observability.go

@@ -24,6 +24,7 @@ func Providers() di.Deps {
 		ProvideOpentracing,
 		ProvideHistogramMetrics,
 		ProvideGORMMetrics,
+		ProvideRedisMetrics,
 		provideConfig,
 	}
 }

observability/observability_test.go

@@ -4,7 +4,9 @@ import (
 	"github.com/DoNewsCode/core"
 	"github.com/DoNewsCode/core/config"
 	"github.com/DoNewsCode/core/otgorm"
+	"github.com/DoNewsCode/core/otredis"
 	"github.com/go-kit/kit/log"
+	"github.com/go-redis/redis/v8"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/rawbytes"
 	"github.com/stretchr/testify/assert"
@@ -59,6 +61,35 @@ func TestProvideGORMMetrics(t *testing.T) {
 	})
 }
 
+func TestProvideRedisMetrics(t *testing.T) {
+	c := core.New()
+	c.ProvideEssentials()
+	c.Provide(Providers())
+	c.Provide(otredis.Providers())
+	c.Invoke(func(cli redis.UniversalClient, g *otredis.Gauges) {
+		stats := cli.PoolStats()
+		withValues := []string{"dbname", "default"}
+		g.Hits.
+			With(withValues...).
+			Set(float64(stats.Hits))
+		g.Misses.
+			With(withValues...).
+			Set(float64(stats.Misses))
+		g.Timeouts.
+			With(withValues...).
+			Set(float64(stats.Timeouts))
+		g.TotalConns.
+			With(withValues...).
+			Set(float64(stats.TotalConns))
+		g.IdleConns.
+			With(withValues...).
+			Set(float64(stats.IdleConns))
+		g.StaleConns.
+			With(withValues...).
+			Set(float64(stats.StaleConns))
+	})
+}
+
 func Test_provideConfig(t *testing.T) {
 	Conf := provideConfig()
 	assert.NotEmpty(t, Conf.Config)

otgorm/module_test.go

@@ -3,6 +3,7 @@ package otgorm
 import (
 	"context"
 	"database/sql"
+	"testing"
 	"time"
 
 	"github.com/DoNewsCode/core"
@@ -12,8 +13,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"gorm.io/gorm"
-
-	"testing"
 )
 
 type Mock struct {
@@ -145,7 +144,7 @@ func TestModule_ProvideRunGroup(t *testing.T) {
 		c2, _ = rawSQL.Conn(ctx)
 	})
 	go c.Serve(ctx)
-	<-time.After(100 * time.Millisecond)
+	<-time.After(10 * time.Millisecond)
 	cancel()
 	<-time.After(1000 * time.Millisecond)
 	c1.Close()

otredis/metrics.go

@@ -0,0 +1,66 @@
+//go:generate mockgen -destination=./mocks/metrics.go github.com/go-kit/kit/metrics Gauge
+
+package otredis
+
+import (
+	"time"
+
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-redis/redis/v8"
+)
+
+type collector struct {
+	factory  Factory
+	gauges   *Gauges
+	interval time.Duration
+}
+
+// Gauges is a collection of metrics for redis connection info.
+type Gauges struct {
+	Hits       metrics.Gauge
+	Misses     metrics.Gauge
+	Timeouts   metrics.Gauge
+	TotalConns metrics.Gauge
+	IdleConns  metrics.Gauge
+	StaleConns metrics.Gauge
+}
+
+// newCollector creates a new redis wrapper containing the name of the redis.
+func newCollector(factory Factory, gauges *Gauges, interval time.Duration) *collector {
+	return &collector{
+		factory:  factory,
+		gauges:   gauges,
+		interval: interval,
+	}
+}
+
+// collectConnectionStats collects redis connections for Prometheus to scrape.
+func (d *collector) collectConnectionStats() {
+	for k, v := range d.factory.List() {
+		conn := v.Conn.(redis.UniversalClient)
+		stats := conn.PoolStats()
+
+		withValues := []string{"dbname", k}
+		d.gauges.Hits.
+			With(withValues...).
+			Set(float64(stats.Hits))
+
+		d.gauges.Misses.
+			With(withValues...).
+			Set(float64(stats.Misses))
+
+		d.gauges.Timeouts.
+			With(withValues...).
+			Set(float64(stats.Timeouts))
+
+		d.gauges.TotalConns.
+			With(withValues...).
+			Set(float64(stats.TotalConns))
+		d.gauges.IdleConns.
+			With(withValues...).
+			Set(float64(stats.IdleConns))
+		d.gauges.StaleConns.
+			With(withValues...).
+			Set(float64(stats.StaleConns))
+	}
+}

otredis/metrics_test.go

@@ -0,0 +1,40 @@
+//go:generate mockery --name=Gauge
+package otredis
+
+import (
+	"testing"
+	"time"
+
+	"github.com/DoNewsCode/core"
+	"github.com/DoNewsCode/core/di"
+	"github.com/DoNewsCode/core/otredis/mocks"
+	"github.com/golang/mock/gomock"
+)
+
+func TestCollector(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	m := mock_metrics.NewMockGauge(ctrl)
+	var g = Gauges{
+		Hits:       m,
+		Misses:     m,
+		Timeouts:   m,
+		TotalConns: m,
+		IdleConns:  m,
+		StaleConns: m,
+	}
+	m.EXPECT().With(gomock.Any()).MinTimes(1).Return(m)
+	m.EXPECT().Set(gomock.Any()).MinTimes(1)
+
+	c := core.New()
+	c.ProvideEssentials()
+	c.Provide(Providers())
+	c.Provide(di.Deps{func() *Gauges { return &g }})
+
+	c.Invoke(func(factory Factory, g *Gauges) {
+		factory.Make("default")
+		c := newCollector(factory, g, time.Nanosecond)
+		c.collectConnectionStats()
+	})
+}