diff --git a/CHANGELOG.md b/CHANGELOG.md index c2cb8718afe..acbc5bbfcf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ 1. [18387](https://github.com/influxdata/influxdb/pull/18387): Integrate query cancellation after queries have been submitted 1. [18515](https://github.com/influxdata/influxdb/pull/18515): Extend templates with the source file|url|reader. +1. [18539](https://github.com/influxdata/influxdb/pull/18539): Collect stats on installed influxdata community template usage. ## v2.0.0-beta.12 [2020-06-12] diff --git a/authorization/middleware_metrics.go b/authorization/middleware_metrics.go index c881185633a..55f3846acac 100644 --- a/authorization/middleware_metrics.go +++ b/authorization/middleware_metrics.go @@ -17,7 +17,7 @@ type AuthMetrics struct { var _ influxdb.AuthorizationService = (*AuthMetrics)(nil) -func NewAuthMetrics(reg prometheus.Registerer, s influxdb.AuthorizationService, opts ...metric.MetricsOption) *AuthMetrics { +func NewAuthMetrics(reg prometheus.Registerer, s influxdb.AuthorizationService, opts ...metric.ClientOptFn) *AuthMetrics { o := metric.ApplyMetricOpts(opts...) return &AuthMetrics{ rec: metric.New(reg, o.ApplySuffix("token")), diff --git a/http/swagger.yml b/http/swagger.yml index 8db7f0926d8..440200cae54 100644 --- a/http/swagger.yml +++ b/http/swagger.yml @@ -7650,7 +7650,7 @@ components: type: string stackID: type: string - package: + template: type: object properties: contentType: @@ -7661,7 +7661,7 @@ components: type: string package: $ref: "#/components/schemas/Pkg" - packages: + templates: type: array items: type: object diff --git a/kit/metric/client.go b/kit/metric/client.go index ae260811b74..b234c982ed0 100644 --- a/kit/metric/client.go +++ b/kit/metric/client.go @@ -1,7 +1,6 @@ package metric import ( - "fmt" "time" "github.com/influxdata/influxdb/v2" @@ -10,63 +9,143 @@ import ( // REDClient is a metrics client for collection RED metrics. type REDClient struct { - // RED metrics - reqs *prometheus.CounterVec - errs *prometheus.CounterVec - durs *prometheus.HistogramVec + metrics []metricCollector } // New creates a new REDClient. -func New(reg prometheus.Registerer, service string) *REDClient { - // MiddlewareMetrics is a metrics service middleware for the notification endpoint service. - const namespace = "service" - - reqs := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: service, - Name: "call_total", - Help: fmt.Sprintf("Number of calls to the %s service", service), - }, []string{"method"}) - - errs := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: service, - Name: "error_total", - Help: fmt.Sprintf("Number of errors encountered when calling the %s service", service), - }, []string{"method", "code"}) - - durs := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: service, - Name: "duration", - Help: fmt.Sprintf("Duration of %s service calls", service), - }, []string{"method"}) - - reg.MustRegister(reqs, errs, durs) - - return &REDClient{ - reqs: reqs, - errs: errs, - durs: durs, +func New(reg prometheus.Registerer, service string, opts ...ClientOptFn) *REDClient { + opt := metricOpts{ + namespace: "service", + service: service, + counterMetrics: map[string]VecOpts{ + "call_total": { + Help: "Number of calls", + LabelNames: []string{"method"}, + CounterFn: func(vec *prometheus.CounterVec, o CollectFnOpts) { + vec.With(prometheus.Labels{"method": o.Method}).Inc() + }, + }, + "error_total": { + Help: "Number of errors encountered", + LabelNames: []string{"method", "code"}, + CounterFn: func(vec *prometheus.CounterVec, o CollectFnOpts) { + if o.Err != nil { + vec.With(prometheus.Labels{ + "method": o.Method, + "code": influxdb.ErrorCode(o.Err), + }).Inc() + } + }, + }, + }, + histogramMetrics: map[string]VecOpts{ + "duration": { + Help: "Duration of calls", + LabelNames: []string{"method"}, + HistogramFn: func(vec *prometheus.HistogramVec, o CollectFnOpts) { + vec. + With(prometheus.Labels{"method": o.Method}). + Observe(time.Since(o.Start).Seconds()) + }, + }, + }, + } + for _, o := range opts { + o(&opt) + } + + client := new(REDClient) + for metricName, vecOpts := range opt.counterMetrics { + client.metrics = append(client.metrics, &counter{ + fn: vecOpts.CounterFn, + CounterVec: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: opt.namespace, + Subsystem: opt.serviceName(), + Name: metricName, + Help: vecOpts.Help, + }, vecOpts.LabelNames), + }) + } + + for metricName, vecOpts := range opt.histogramMetrics { + client.metrics = append(client.metrics, &histogram{ + fn: vecOpts.HistogramFn, + HistogramVec: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: opt.namespace, + Subsystem: opt.serviceName(), + Name: metricName, + Help: vecOpts.Help, + }, vecOpts.LabelNames), + }) + } + + reg.MustRegister(client.collectors()...) + + return client +} + +type RecordFn func(err error, opts ...func(opts *CollectFnOpts)) error + +// RecordAdditional provides an extension to the base method, err data provided +// to the metrics. +func RecordAdditional(props map[string]interface{}) func(opts *CollectFnOpts) { + return func(opts *CollectFnOpts) { + opts.AdditionalProps = props } } // Record returns a record fn that is called on any given return err. If an error is encountered // it will register the err metric. The err is never altered. -func (c *REDClient) Record(method string) func(error) error { +func (c *REDClient) Record(method string) RecordFn { start := time.Now() - return func(err error) error { - c.reqs.With(prometheus.Labels{"method": method}).Inc() - - if err != nil { - c.errs.With(prometheus.Labels{ - "method": method, - "code": influxdb.ErrorCode(err), - }).Inc() + return func(err error, opts ...func(opts *CollectFnOpts)) error { + opt := CollectFnOpts{ + Method: method, + Start: start, + Err: err, + } + for _, o := range opts { + o(&opt) } - c.durs.With(prometheus.Labels{"method": method}).Observe(time.Since(start).Seconds()) + for _, metric := range c.metrics { + metric.collect(opt) + } return err } } + +func (c *REDClient) collectors() []prometheus.Collector { + var collectors []prometheus.Collector + for _, metric := range c.metrics { + collectors = append(collectors, metric) + } + return collectors +} + +type metricCollector interface { + prometheus.Collector + + collect(o CollectFnOpts) +} + +type counter struct { + *prometheus.CounterVec + + fn CounterFn +} + +func (c *counter) collect(o CollectFnOpts) { + c.fn(c.CounterVec, o) +} + +type histogram struct { + *prometheus.HistogramVec + + fn HistogramFn +} + +func (h *histogram) collect(o CollectFnOpts) { + h.fn(h.HistogramVec, o) +} diff --git a/kit/metric/metrics_options.go b/kit/metric/metrics_options.go index 95e9adb48e6..8f48629c5b2 100644 --- a/kit/metric/metrics_options.go +++ b/kit/metric/metrics_options.go @@ -1,36 +1,84 @@ package metric -import "fmt" +import ( + "fmt" + "time" -type metricOpts struct { - serviceSuffix string -} + "github.com/prometheus/client_golang/prometheus" +) + +type ( + // CollectFnOpts provides arugments to the collect operation of a metric. + CollectFnOpts struct { + Method string + Start time.Time + Err error + AdditionalProps map[string]interface{} + } + + CounterFn func(vec *prometheus.CounterVec, o CollectFnOpts) + + HistogramFn func(vec *prometheus.HistogramVec, o CollectFnOpts) -func defaultOpts() *metricOpts { - return &metricOpts{} + // VecOpts expands on the + VecOpts struct { + Name string + Help string + LabelNames []string + + CounterFn CounterFn + HistogramFn HistogramFn + } +) + +type metricOpts struct { + namespace string + service string + serviceSuffix string + counterMetrics map[string]VecOpts + histogramMetrics map[string]VecOpts } -func (o *metricOpts) ApplySuffix(prefix string) string { +func (o metricOpts) serviceName() string { if o.serviceSuffix != "" { - return fmt.Sprintf("%s_%s", prefix, o.serviceSuffix) + return fmt.Sprintf("%s_%s", o.service, o.serviceSuffix) } - return prefix + return o.service } -// MetricsOption is an option used by a metric middleware. -type MetricsOption func(*metricOpts) +// ClientOptFn is an option used by a metric middleware. +type ClientOptFn func(*metricOpts) + +// WithVec sets a new counter vector to be collected. +func WithVec(opts VecOpts) ClientOptFn { + return func(o *metricOpts) { + if opts.CounterFn != nil { + if o.counterMetrics == nil { + o.counterMetrics = make(map[string]VecOpts) + } + o.counterMetrics[opts.Name] = opts + } + } +} // WithSuffix returns a metric option that applies a suffix to the service name of the metric. -func WithSuffix(suffix string) MetricsOption { +func WithSuffix(suffix string) ClientOptFn { return func(opts *metricOpts) { opts.serviceSuffix = suffix } } -func ApplyMetricOpts(opts ...MetricsOption) *metricOpts { - o := defaultOpts() +func ApplyMetricOpts(opts ...ClientOptFn) *metricOpts { + o := metricOpts{} for _, opt := range opts { - opt(o) + opt(&o) } - return o + return &o +} + +func (o *metricOpts) ApplySuffix(prefix string) string { + if o.serviceSuffix != "" { + return fmt.Sprintf("%s_%s", prefix, o.serviceSuffix) + } + return prefix } diff --git a/label/middleware_metrics.go b/label/middleware_metrics.go index 102a6d7b71d..9a84cac65a8 100644 --- a/label/middleware_metrics.go +++ b/label/middleware_metrics.go @@ -15,7 +15,7 @@ type LabelMetrics struct { labelService influxdb.LabelService } -func NewLabelMetrics(reg prometheus.Registerer, s influxdb.LabelService, opts ...metric.MetricsOption) *LabelMetrics { +func NewLabelMetrics(reg prometheus.Registerer, s influxdb.LabelService, opts ...metric.ClientOptFn) *LabelMetrics { o := metric.ApplyMetricOpts(opts...) return &LabelMetrics{ rec: metric.New(reg, o.ApplySuffix("org")), diff --git a/pkger/service_metrics.go b/pkger/service_metrics.go index c95e54c5c67..d4ec37ecb42 100644 --- a/pkger/service_metrics.go +++ b/pkger/service_metrics.go @@ -2,10 +2,13 @@ package pkger import ( "context" + "net/url" + "strings" "github.com/influxdata/influxdb/v2" "github.com/influxdata/influxdb/v2/kit/metric" "github.com/influxdata/influxdb/v2/kit/prom" + "github.com/prometheus/client_golang/prometheus" ) type mwMetrics struct { @@ -21,7 +24,7 @@ var _ SVC = (*mwMetrics)(nil) func MWMetrics(reg *prom.Registry) SVCMiddleware { return func(svc SVC) SVC { return &mwMetrics{ - rec: metric.New(reg, "pkger"), + rec: metric.New(reg, "pkger", metric.WithVec(templateVec())), next: svc, } } @@ -59,11 +62,70 @@ func (s *mwMetrics) CreatePkg(ctx context.Context, setters ...CreatePkgSetFn) (* func (s *mwMetrics) DryRun(ctx context.Context, orgID, userID influxdb.ID, opts ...ApplyOptFn) (PkgImpactSummary, error) { rec := s.rec.Record("dry_run") impact, err := s.next.DryRun(ctx, orgID, userID, opts...) - return impact, rec(err) + return impact, rec(err, metric.RecordAdditional(map[string]interface{}{ + "sources": impact.Sources, + })) } func (s *mwMetrics) Apply(ctx context.Context, orgID, userID influxdb.ID, opts ...ApplyOptFn) (PkgImpactSummary, error) { rec := s.rec.Record("apply") impact, err := s.next.Apply(ctx, orgID, userID, opts...) - return impact, rec(err) + return impact, rec(err, metric.RecordAdditional(map[string]interface{}{ + "sources": impact.Sources, + })) +} + +func templateVec() metric.VecOpts { + return metric.VecOpts{ + Name: "template_count", + Help: "Number of installations per template", + LabelNames: []string{"method", "source"}, + CounterFn: func(vec *prometheus.CounterVec, o metric.CollectFnOpts) { + if o.Err != nil { + return + } + + // safe to ignore the failed type assertion, a zero value + // provides a nil slice, so no worries. + sources, _ := o.AdditionalProps["sources"].([]string) + for _, source := range normalizeRemoteSources(sources) { + vec. + With(prometheus.Labels{ + "method": o.Method, + "source": source.String(), + }). + Inc() + } + }, + } +} + +func normalizeRemoteSources(sources []string) []url.URL { + var out []url.URL + for _, source := range sources { + u, err := url.Parse(source) + if err != nil { + continue + } + if !strings.HasPrefix(u.Scheme, "http") { + continue + } + if u.Host == "raw.githubusercontent.com" { + u.Host = "github.com" + u.Path = normalizeRawGithubPath(u.Path) + } + out = append(out, *u) + } + return out +} + +func normalizeRawGithubPath(rawPath string) string { + parts := strings.Split(rawPath, "/") + if len(parts) < 4 { + return rawPath + } + // keep /account/repo as base, then append the blob to it + tail := append([]string{"blob"}, parts[3:]...) + parts = append(parts[:3], tail...) + return strings.Join(parts, "/") } diff --git a/pkger/service_test.go b/pkger/service_test.go index f903493820e..276dbbe4729 100644 --- a/pkger/service_test.go +++ b/pkger/service_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math/rand" + "net/url" "regexp" "sort" "strconv" @@ -3215,6 +3216,58 @@ func TestService(t *testing.T) { }) } +func Test_normalizeRemoteSources(t *testing.T) { + tests := []struct { + name string + input []string + expected []url.URL + }{ + { + name: "no urls provided", + input: []string{"byte stream", "string", ""}, + expected: nil, + }, + { + name: "skips valid file url", + input: []string{"file:///example.com"}, + expected: nil, + }, + { + name: "valid http url provided", + input: []string{"http://example.com"}, + expected: []url.URL{parseURL(t, "http://example.com")}, + }, + { + name: "valid https url provided", + input: []string{"https://example.com"}, + expected: []url.URL{parseURL(t, "https://example.com")}, + }, + { + name: "converts raw github user url to base github", + input: []string{"https://raw.githubusercontent.com/influxdata/community-templates/master/github/github.yml"}, + expected: []url.URL{ + parseURL(t, "https://github.com/influxdata/community-templates/blob/master/github/github.yml"), + }, + }, + { + name: "passes base github link unchanged", + input: []string{"https://github.com/influxdata/community-templates/blob/master/github/github.yml"}, + expected: []url.URL{ + parseURL(t, "https://github.com/influxdata/community-templates/blob/master/github/github.yml"), + }, + }, + } + + for _, tt := range tests { + fn := func(t *testing.T) { + actual := normalizeRemoteSources(tt.input) + + assert.Equal(t, tt.expected, actual) + } + t.Run(tt.name, fn) + } +} + func newTestIDPtr(i int) *influxdb.ID { id := influxdb.ID(i) return &id @@ -3284,3 +3337,10 @@ func newTimeGen(t time.Time) fakeTimeGen { func (t fakeTimeGen) Now() time.Time { return t() } + +func parseURL(t *testing.T, rawAddr string) url.URL { + t.Helper() + u, err := url.Parse(rawAddr) + require.NoError(t, err) + return *u +} diff --git a/tenant/middleware_bucket_metrics.go b/tenant/middleware_bucket_metrics.go index 746195e9f78..88fecb44ac3 100644 --- a/tenant/middleware_bucket_metrics.go +++ b/tenant/middleware_bucket_metrics.go @@ -18,7 +18,7 @@ type BucketMetrics struct { var _ influxdb.BucketService = (*BucketMetrics)(nil) // NewBucketMetrics returns a metrics service middleware for the Bucket Service. -func NewBucketMetrics(reg prometheus.Registerer, s influxdb.BucketService, opts ...metric.MetricsOption) *BucketMetrics { +func NewBucketMetrics(reg prometheus.Registerer, s influxdb.BucketService, opts ...metric.ClientOptFn) *BucketMetrics { o := metric.ApplyMetricOpts(opts...) return &BucketMetrics{ rec: metric.New(reg, o.ApplySuffix("bucket")), diff --git a/tenant/middleware_onboarding_metrics.go b/tenant/middleware_onboarding_metrics.go index cbd5fe30f6e..603005cd02a 100644 --- a/tenant/middleware_onboarding_metrics.go +++ b/tenant/middleware_onboarding_metrics.go @@ -18,7 +18,7 @@ type OnboardingMetrics struct { } // NewOnboardingMetrics returns a metrics service middleware for the User Service. -func NewOnboardingMetrics(reg prometheus.Registerer, s influxdb.OnboardingService, opts ...metric.MetricsOption) *OnboardingMetrics { +func NewOnboardingMetrics(reg prometheus.Registerer, s influxdb.OnboardingService, opts ...metric.ClientOptFn) *OnboardingMetrics { o := metric.ApplyMetricOpts(opts...) return &OnboardingMetrics{ rec: metric.New(reg, o.ApplySuffix("onboard")), diff --git a/tenant/middleware_org_metrics.go b/tenant/middleware_org_metrics.go index fdab4e14f62..a0b108f1161 100644 --- a/tenant/middleware_org_metrics.go +++ b/tenant/middleware_org_metrics.go @@ -18,7 +18,7 @@ type OrgMetrics struct { var _ influxdb.OrganizationService = (*OrgMetrics)(nil) // NewOrgMetrics returns a metrics service middleware for the Organization Service. -func NewOrgMetrics(reg prometheus.Registerer, s influxdb.OrganizationService, opts ...metric.MetricsOption) *OrgMetrics { +func NewOrgMetrics(reg prometheus.Registerer, s influxdb.OrganizationService, opts ...metric.ClientOptFn) *OrgMetrics { o := metric.ApplyMetricOpts(opts...) return &OrgMetrics{ rec: metric.New(reg, o.ApplySuffix("org")), diff --git a/tenant/middleware_urm_metrics.go b/tenant/middleware_urm_metrics.go index 3341ab225e8..c65c9196d09 100644 --- a/tenant/middleware_urm_metrics.go +++ b/tenant/middleware_urm_metrics.go @@ -18,7 +18,7 @@ type UrmMetrics struct { var _ influxdb.UserResourceMappingService = (*UrmMetrics)(nil) // NewUrmMetrics returns a metrics service middleware for the User Resource Mapping Service. -func NewUrmMetrics(reg prometheus.Registerer, s influxdb.UserResourceMappingService, opts ...metric.MetricsOption) *UrmMetrics { +func NewUrmMetrics(reg prometheus.Registerer, s influxdb.UserResourceMappingService, opts ...metric.ClientOptFn) *UrmMetrics { o := metric.ApplyMetricOpts(opts...) return &UrmMetrics{ rec: metric.New(reg, o.ApplySuffix("urm")), diff --git a/tenant/middleware_user_metrics.go b/tenant/middleware_user_metrics.go index 366e746c55c..d4450cb4be0 100644 --- a/tenant/middleware_user_metrics.go +++ b/tenant/middleware_user_metrics.go @@ -19,7 +19,7 @@ type UserMetrics struct { } // NewUserMetrics returns a metrics service middleware for the User Service. -func NewUserMetrics(reg prometheus.Registerer, s influxdb.UserService, opts ...metric.MetricsOption) *UserMetrics { +func NewUserMetrics(reg prometheus.Registerer, s influxdb.UserService, opts ...metric.ClientOptFn) *UserMetrics { o := metric.ApplyMetricOpts(opts...) return &UserMetrics{ rec: metric.New(reg, o.ApplySuffix("user")), @@ -71,7 +71,7 @@ type PasswordMetrics struct { } // NewPasswordMetrics returns a metrics service middleware for the Password Service. -func NewPasswordMetrics(reg prometheus.Registerer, s influxdb.PasswordsService, opts ...metric.MetricsOption) *PasswordMetrics { +func NewPasswordMetrics(reg prometheus.Registerer, s influxdb.PasswordsService, opts ...metric.ClientOptFn) *PasswordMetrics { o := metric.ApplyMetricOpts(opts...) return &PasswordMetrics{ rec: metric.New(reg, o.ApplySuffix("password")),