Port: Telemetry For Lease Expiration Times

command/server/config_test.go

@@ -30,6 +30,10 @@ func TestLoadConfigFileIntegerAndBooleanValuesJson(t *testing.T) {
 	testLoadConfigFileIntegerAndBooleanValuesJson(t)
 }
 
+func TestLoadConfigFileWithLeaseMetricTelemetry(t *testing.T) {
+	testLoadConfigFileLeaseMetrics(t)
+}
+
 func TestLoadConfigDir(t *testing.T) {
 	testLoadConfigDir(t)
 }

command/server/config_test_helpers.go

@@ -787,3 +787,93 @@ func testParseSeals(t *testing.T) {
 	}
 	require.Equal(t, config, expected)
 }
+
+func testLoadConfigFileLeaseMetrics(t *testing.T) {
+	config, err := LoadConfigFile("./test-fixtures/config5.hcl")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		SharedConfig: &configutil.SharedConfig{
+			Listeners: []*configutil.Listener{
+				{
+					Type:    "tcp",
+					Address: "127.0.0.1:443",
+				},
+			},
+
+			Telemetry: &configutil.Telemetry{
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				MetricsPrefix:               "myprefix",
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  2,
+				LeaseMetricsNameSpaceLabels: true,
+			},
+
+			DisableMlock: true,
+
+			Entropy: nil,
+
+			PidFile: "./pidfile",
+
+			ClusterName: "testcluster",
+		},
+
+		Storage: &Storage{
+			Type:         "consul",
+			RedirectAddr: "foo",
+			Config: map[string]string{
+				"foo": "bar",
+			},
+		},
+
+		HAStorage: &Storage{
+			Type:         "consul",
+			RedirectAddr: "snafu",
+			Config: map[string]string{
+				"bar": "baz",
+			},
+			DisableClustering: true,
+		},
+
+		ServiceRegistration: &ServiceRegistration{
+			Type: "consul",
+			Config: map[string]string{
+				"foo": "bar",
+			},
+		},
+
+		DisableCache:             true,
+		DisableCacheRaw:          true,
+		DisablePrintableCheckRaw: true,
+		DisablePrintableCheck:    true,
+		EnableUI:                 true,
+		EnableUIRaw:              true,
+
+		EnableRawEndpoint:    true,
+		EnableRawEndpointRaw: true,
+
+		DisableSealWrap:    true,
+		DisableSealWrapRaw: true,
+
+		MaxLeaseTTL:        10 * time.Hour,
+		MaxLeaseTTLRaw:     "10h",
+		DefaultLeaseTTL:    10 * time.Hour,
+		DefaultLeaseTTLRaw: "10h",
+	}
+
+	addExpectedEntConfig(expected, []string{})
+
+	config.Listeners[0].RawConfig = nil
+	if diff := deep.Equal(config, expected); diff != nil {
+		t.Fatal(diff)
+	}
+}

command/server/test-fixtures/config5.hcl

@@ -0,0 +1,51 @@
+disable_cache = true
+ disable_mlock = true
+
+ ui = true
+
+ listener "tcp" {
+     address = "127.0.0.1:443"
+ 	allow_stuff = true
+ }
+
+ backend "consul" {
+     foo = "bar"
+     advertise_addr = "foo"
+ }
+
+ ha_backend "consul" {
+     bar = "baz"
+     advertise_addr = "snafu"
+     disable_clustering = "true"
+ }
+
+ service_registration "consul" {
+     foo = "bar"
+ }
+
+ telemetry {
+     statsd_address = "bar"
+     usage_gauge_period = "5m"
+     maximum_gauge_cardinality = 100
+
+     statsite_address = "foo"
+     dogstatsd_addr = "127.0.0.1:7254"
+     dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"]
+     metrics_prefix = "myprefix"
+
+     lease_metrics_epsilon = "1h"
+     num_lease_metrics_buckets = 2
+     add_lease_metrics_namespace_labels = true 
+ }
+
+ sentinel {
+     additional_enabled_modules = []
+ }
+
+ max_lease_ttl = "10h"
+ default_lease_ttl = "10h"
+ cluster_name = "testcluster"
+ pid_file = "./pidfile"
+ raw_storage_endpoint = true
+ disable_sealwrap = true
+ disable_printable_check = true

helper/metricsutil/bucket.go

@@ -37,3 +37,22 @@ func TTLBucket(ttl time.Duration) string {
 	}
 
 }
+
+func ExpiryBucket(expiryTime time.Time, leaseEpsilon time.Duration, rollingWindow time.Time, labelNS string, useNS bool) *LeaseExpiryLabel {
+	if !useNS {
+		labelNS = ""
+	}
+	leaseExpiryLabel := LeaseExpiryLabel{LabelNS: labelNS}
+
+	// calculate rolling window
+	if expiryTime.Before(rollingWindow) {
+		leaseExpiryLabel.LabelName = expiryTime.Round(leaseEpsilon).String()
+		return &leaseExpiryLabel
+	}
+	return nil
+}
+
+type LeaseExpiryLabel = struct {
+	LabelName string
+	LabelNS   string
+}

helper/metricsutil/wrapped_metrics.go

@@ -28,6 +28,15 @@ type ClusterMetricSink struct {
 
 	// Sink is the go-metrics instance to send to.
 	Sink metrics.MetricSink
+
+	// Constants that are helpful for metrics within the metrics sink
+	TelemetryConsts TelemetryConstConfig
+}
+
+type TelemetryConstConfig struct {
+	LeaseMetricsEpsilon         time.Duration
+	NumLeaseMetricsTimeBuckets  int
+	LeaseMetricsNameSpaceLabels bool
 }
 
 type Metrics interface {
@@ -83,8 +92,9 @@ func BlackholeSink() *ClusterMetricSink {
 
 func NewClusterMetricSink(clusterName string, sink metrics.MetricSink) *ClusterMetricSink {
 	cms := &ClusterMetricSink{
-		ClusterName: atomic.Value{},
-		Sink:        sink,
+		ClusterName:     atomic.Value{},
+		Sink:            sink,
+		TelemetryConsts: TelemetryConstConfig{},
 	}
 	cms.ClusterName.Store(clusterName)
 	return cms

internalshared/configutil/telemetry.go

@@ -24,9 +24,11 @@ import (
 )
 
 const (
-	PrometheusDefaultRetentionTime = 24 * time.Hour
-	UsageGaugeDefaultPeriod        = 10 * time.Minute
-	MaximumGaugeCardinalityDefault = 500
+	PrometheusDefaultRetentionTime    = 24 * time.Hour
+	UsageGaugeDefaultPeriod           = 10 * time.Minute
+	MaximumGaugeCardinalityDefault    = 500
+	LeaseMetricsEpsilonDefault        = time.Hour
+	NumLeaseMetricsTimeBucketsDefault = 168
 )
 
 // Telemetry is the telemetry configuration for the server
@@ -137,6 +139,15 @@ type Telemetry struct {
 	StackdriverNamespace string `hcl:"stackdriver_namespace"`
 	// StackdriverDebugLogs will write additional stackdriver related debug logs to stderr.
 	StackdriverDebugLogs bool `hcl:"stackdriver_debug_logs"`
+
+	// How often metrics for lease expiry will be aggregated
+	LeaseMetricsEpsilon time.Duration `hcl:lease_metrics_epsilon`
+
+	// Number of buckets by time that will be used in lease aggregation
+	NumLeaseMetricsTimeBuckets int `hcl:num_lease_metrics_buckets`
+
+	// Whether or not telemetry should add labels for namespaces
+	LeaseMetricsNameSpaceLabels bool `hcl:add_lease_metrics_namespace_labels`
 }
 
 func (t *Telemetry) GoString() string {
@@ -151,11 +162,6 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error {
 	// Get our one item
 	item := list.Items[0]
 
-	var t Telemetry
-	if err := hcl.DecodeObject(&t, item.Val); err != nil {
-		return multierror.Prefix(err, "telemetry:")
-	}
-
 	if result.Telemetry == nil {
 		result.Telemetry = &Telemetry{}
 	}
@@ -191,6 +197,14 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error {
 	if result.Telemetry.MaximumGaugeCardinality == 0 {
 		result.Telemetry.MaximumGaugeCardinality = MaximumGaugeCardinalityDefault
 	}
+	var defaultDuration time.Duration
+	if result.Telemetry.LeaseMetricsEpsilon == defaultDuration {
+		result.Telemetry.LeaseMetricsEpsilon = LeaseMetricsEpsilonDefault
+	}
+
+	if result.Telemetry.NumLeaseMetricsTimeBuckets == 0 {
+		result.Telemetry.NumLeaseMetricsTimeBuckets = NumLeaseMetricsTimeBucketsDefault
+	}
 
 	return nil
 }
@@ -355,6 +369,9 @@ func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil.
 	wrapper := metricsutil.NewClusterMetricSink(opts.ClusterName, globalMetrics)
 	wrapper.MaxGaugeCardinality = opts.Config.MaximumGaugeCardinality
 	wrapper.GaugeInterval = opts.Config.UsageGaugePeriod
+	wrapper.TelemetryConsts.LeaseMetricsEpsilon = opts.Config.LeaseMetricsEpsilon
+	wrapper.TelemetryConsts.LeaseMetricsNameSpaceLabels = opts.Config.LeaseMetricsNameSpaceLabels
+	wrapper.TelemetryConsts.NumLeaseMetricsTimeBuckets = opts.Config.NumLeaseMetricsTimeBuckets
 
 	return inm, wrapper, prometheusEnabled, nil
 }

vault/core_metrics.go

@@ -116,6 +116,17 @@ func (c *Core) tokenGaugePolicyCollector(ctx context.Context) ([]metricsutil.Gau
 	return ts.gaugeCollectorByPolicy(ctx)
 }
 
+func (c *Core) leaseExpiryGaugeCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) {
+	c.stateLock.RLock()
+	e := c.expiration
+	metricsConsts := c.MetricSink().TelemetryConsts
+	c.stateLock.RUnlock()
+	if e == nil {
+		return []metricsutil.GaugeLabelValues{}, errors.New("nil expiration manager")
+	}
+	return e.leaseAggregationMetrics(ctx, metricsConsts)
+}
+
 func (c *Core) tokenGaugeMethodCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) {
 	c.stateLock.RLock()
 	ts := c.tokenStore
@@ -164,6 +175,12 @@ func (c *Core) emitMetrics(stopCh chan struct{}) {
 			c.tokenGaugePolicyCollector,
 			"",
 		},
+		{
+			[]string{"expire", "leases", "by_expiration"},
+			[]metrics.Label{{"gauge", "leases_by_expiration"}},
+			c.leaseExpiryGaugeCollector,
+			"",
+		},
 		{
 			[]string{"token", "count", "by_auth"},
 			[]metrics.Label{{"gauge", "token_by_auth"}},