From 437e47fe7cac280ba6d67eb5f9756fd605b16cf5 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Fri, 7 Oct 2022 08:13:50 -0400 Subject: [PATCH] feat: sniff latest api version and provide template override - use tenant id instead of name to discriminate tenants - include region column in buckets panel --- .../storagegrid/plugins/bucket/bucket.go | 2 + cmd/collectors/storagegrid/rest/client.go | 112 +++++++++++++----- cmd/collectors/storagegrid/storagegrid.go | 18 ++- cmd/poller/collector/collector.go | 2 +- conf/storagegrid/11.6.0/tenant.yaml | 6 +- grafana/dashboards/storagegrid/tenant.json | 101 +++++++++++++--- 6 files changed, 188 insertions(+), 53 deletions(-) diff --git a/cmd/collectors/storagegrid/plugins/bucket/bucket.go b/cmd/collectors/storagegrid/plugins/bucket/bucket.go index c37edf7ae..d473372a4 100644 --- a/cmd/collectors/storagegrid/plugins/bucket/bucket.go +++ b/cmd/collectors/storagegrid/plugins/bucket/bucket.go @@ -80,6 +80,7 @@ func (b *Bucket) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { bucketsJSON := record.Get("buckets") for _, bucketJSON := range bucketsJSON.Array() { bucket := bucketJSON.Get("name").String() + region := bucketJSON.Get("region").String() instanceKey = instKey + "#" + bucket bucketInstance, err2 := b.data.NewInstance(instanceKey) @@ -90,6 +91,7 @@ func (b *Bucket) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { b.Logger.Debug().Str("instanceKey", instanceKey).Msg("add instance") bucketInstance.SetLabel("bucket", bucket) bucketInstance.SetLabel("tenant", tenantName) + bucketInstance.SetLabel("region", region) for metricKey, m := range b.data.GetMetrics() { jsonKey := metricToJSON[metricKey] if value := bucketJSON.Get(jsonKey); value.Exists() { diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go index b9f13093a..8866a2df7 100644 --- a/cmd/collectors/storagegrid/rest/client.go +++ b/cmd/collectors/storagegrid/rest/client.go @@ -20,7 +20,8 @@ import ( ) const ( - DefaultTimeout = "1m" + DefaultTimeout = "1m" + DefaultAPIVersion = "3" ) type Client struct { @@ -35,7 +36,7 @@ type Client struct { token string Timeout time.Duration logRest bool // used to log Rest request/response - apiPath string + APIPath string } type Cluster struct { @@ -94,7 +95,6 @@ func New(poller conf.Poller, timeout time.Duration) (*Client, error) { client.baseURL = href client.Timeout = timeout - client.apiPath = "/api/v3/" // by default, enforce secure TLS, if not requested otherwise by user if x := poller.UseInsecureTLS; x != nil { @@ -187,7 +187,7 @@ func (c *Client) Fetch(request string, result *[]gjson.Result) error { err error fetched []byte ) - fetched, err = c.GetRest(request) + fetched, err = c.GetGridRest(request) if err != nil { return fmt.Errorf("error making request %w", err) } @@ -200,33 +200,50 @@ func (c *Client) Fetch(request string, result *[]gjson.Result) error { return nil } -// GetRest makes a REST request to the cluster and returns a json response as a []byte +// GetGridRest makes a grid API request to the cluster and returns a json response as a []byte // see also Fetch -func (c *Client) GetRest(request string) ([]byte, error) { - var err error - u, err := url.JoinPath(c.baseURL, c.apiPath, request) +func (c *Client) GetGridRest(request string) ([]byte, error) { + u, err := url.JoinPath(c.baseURL, c.APIPath, request) if err != nil { return nil, fmt.Errorf("failed to join URL %s err: %w", request, err) } - u2, err2 := url.QueryUnescape(u) - if err2 != nil { - return nil, fmt.Errorf("failed to unescape URL %s err: %w", u, err2) + return c.getRest(u) +} + +// GetMetricQuery makes a metrics API request to the cluster and fills the result argument +func (c *Client) GetMetricQuery(metric string, result *[]gjson.Result) error { + u, err := url.JoinPath(c.baseURL, "/metrics/api/v1/query?query="+metric) + if err != nil { + return fmt.Errorf("failed to query metric %s err: %w", metric, err) } - c.request, err = http.NewRequest("GET", u2, nil) + fetched, err := c.getRest(u) + if err != nil { + return err + } + output := gjson.GetManyBytes(fetched, "data") + data := output[0] + for _, r := range data.Array() { + *result = append(*result, r.Array()...) + } + return nil +} + +// getRest makes a request to the cluster and returns a json response as a []byte +// see also Fetch +func (c *Client) getRest(request string) ([]byte, error) { + u, err := url.QueryUnescape(request) + if err != nil { + return nil, fmt.Errorf("failed to unescape %s err: %w", request, err) + } + + c.request, err = http.NewRequest("GET", u, nil) if err != nil { return nil, err } c.request.Header.Set("accept", "application/json") c.request.Header.Set("Authorization", "Bearer "+c.token) - // ensure that we can change body dynamically - c.request.GetBody = func() (io.ReadCloser, error) { - r := bytes.NewReader(c.buffer.Bytes()) - return io.NopCloser(r), nil - } - - result, err := c.invoke() - return result, err + return c.invoke() } func (c *Client) invoke() ([]byte, error) { @@ -292,6 +309,7 @@ func (c *Client) fetch() ([]byte, error) { return body, nil } +// Init is responsible for determining the StorageGrid server version, API version, and hostname func (c *Client) Init(retries int) error { var ( err error @@ -299,29 +317,29 @@ func (c *Client) Init(retries int) error { i int ) - // Product version and cluster name are two separate endpoints for i = 0; i < retries; i++ { - if content, err = c.GetRest("grid/config/product-version"); err != nil { + // Determine which API versions are supported and then request + // product version and cluster name - both of which are separate endpoints + + err = c.sniffAPIVersion(retries) + if err != nil { continue } + if content, err = c.GetGridRest("grid/config/product-version"); err != nil { + continue + } results := gjson.GetManyBytes(content, "data.productVersion") err = c.SetVersion(results[0].String()) if err != nil { return err } - } - if err != nil { - return err - } - err = nil - for i = 0; i < retries; i++ { - if content, err = c.GetRest("grid/config"); err != nil { + if content, err = c.GetGridRest("grid/config"); err != nil { continue } - results := gjson.GetManyBytes(content, "data.hostname") + results = gjson.GetManyBytes(content, "data.hostname") c.Cluster.Name = results[0].String() return nil } @@ -358,7 +376,7 @@ func (c *Client) fetchToken() error { response *http.Response body []byte ) - u, err := url.JoinPath(c.baseURL, c.apiPath, "authorize") + u, err := url.JoinPath(c.baseURL, c.APIPath, "authorize") if err != nil { return fmt.Errorf("failed to create auth URL err: %w", err) } @@ -410,3 +428,35 @@ func (c *Client) fetchToken() error { } return nil } + +func (c *Client) sniffAPIVersion(retries int) error { + // This endpoint does not require auth and uses the /api/ endpoint instead of a versioned one + + var ( + apiVersion = DefaultAPIVersion + u string + err error + i int + ) + + u, err = url.JoinPath(c.baseURL, "/api/versions") + if err != nil { + return fmt.Errorf("failed to join getApiVersions %s err: %w", c.baseURL, err) + } + for i = 0; i < retries; i++ { + result, err := c.getRest(u) + if err != nil { + continue + } + versionB := gjson.GetBytes(result, "data") + if versionB.Exists() && versionB.IsArray() { + versions := versionB.Array() + if len(versions) > 0 { + apiVersion = versions[len(versions)-1].String() + } + } + c.APIPath = "/api/v" + apiVersion + return nil + } + return err +} diff --git a/cmd/collectors/storagegrid/storagegrid.go b/cmd/collectors/storagegrid/storagegrid.go index 29b4b9629..4c84f32b7 100644 --- a/cmd/collectors/storagegrid/storagegrid.go +++ b/cmd/collectors/storagegrid/storagegrid.go @@ -64,6 +64,7 @@ func (s *StorageGrid) Init(a *collector.AbstractCollector) error { if s.Props.TemplatePath, err = s.LoadTemplate(); err != nil { return err } + s.InitAPIPath() if err = collector.Init(s); err != nil { return err } @@ -291,9 +292,8 @@ func (s *StorageGrid) handleResults(result []gjson.Result) uint64 { func (s *StorageGrid) initClient() error { var err error - a := s.AbstractCollector - if s.client, err = srest.NewClient(s.Options.Poller, a.Params.GetChildContentS("client_timeout")); err != nil { + if s.client, err = srest.NewClient(s.Options.Poller, s.Params.GetChildContentS("client_timeout")); err != nil { return err } @@ -377,6 +377,20 @@ func (s *StorageGrid) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin return nil } +// InitAPIPath reads the REST API version from the template and uses it instead of +// the DefaultAPIVersion +func (s *StorageGrid) InitAPIPath() { + apiVersion := s.Params.GetChildContentS("api") + if !strings.HasSuffix(s.client.APIPath, apiVersion) { + cur := s.client.APIPath + s.client.APIPath = "/apiVersion/" + apiVersion + s.Logger.Debug(). + Str("clientAPI", cur). + Str("templateAPI", apiVersion). + Msg("Use template apiVersion") + } +} + // Interface guards var ( _ collector.Collector = (*StorageGrid)(nil) diff --git a/cmd/poller/collector/collector.go b/cmd/poller/collector/collector.go index 04ec3ab40..e1a2f5a2d 100644 --- a/cmd/poller/collector/collector.go +++ b/cmd/poller/collector/collector.go @@ -124,7 +124,7 @@ func New(name, object string, options *options.Options, params *node.Node) *Abst // inside its Init method, or leave it to be called // by the poller during dynamic load. // -// The important thing done here is too look what tasks are defined +// The important thing done here is to look what tasks are defined // in the "schedule" parameter of the collector and create a pointer // to the corresponding method of the collector. Example, parameter is: // diff --git a/conf/storagegrid/11.6.0/tenant.yaml b/conf/storagegrid/11.6.0/tenant.yaml index aa7f56fa8..ce095bd60 100644 --- a/conf/storagegrid/11.6.0/tenant.yaml +++ b/conf/storagegrid/11.6.0/tenant.yaml @@ -2,6 +2,7 @@ name: Tenant query: grid/accounts-cache object: tenant +api: v3 counters: - ^^id => id @@ -15,4 +16,7 @@ plugins: - Bucket export_options: - include_all_labels: true \ No newline at end of file + instance_keys: + - id + instance_labels: + - tenant \ No newline at end of file diff --git a/grafana/dashboards/storagegrid/tenant.json b/grafana/dashboards/storagegrid/tenant.json index f40a1ba0d..c568623dc 100644 --- a/grafana/dashboards/storagegrid/tenant.json +++ b/grafana/dashboards/storagegrid/tenant.json @@ -52,7 +52,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1664465454598, + "iteration": 1665158170026, "links": [], "panels": [ { @@ -148,6 +148,30 @@ "value": 0 } ] + }, + { + "matcher": { + "id": "byName", + "options": "Cluster" + }, + "properties": [ + { + "id": "custom.width", + "value": null + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Objects" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + } + ] } ] }, @@ -172,7 +196,7 @@ "targets": [ { "exemplar": true, - "expr": "tenant_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", tenant=~\"$Tenant\"}", + "expr": "tenant_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",id=~\"$Id\"}", "format": "table", "instant": true, "interval": "", @@ -181,7 +205,7 @@ }, { "exemplar": true, - "expr": "tenant_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", tenant=~\"$Tenant\"}", + "expr": "tenant_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",id=~\"$Id\"}", "format": "table", "hide": false, "instant": true, @@ -191,7 +215,7 @@ }, { "exemplar": true, - "expr": "tenant_logical_quota{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", tenant=~\"$Tenant\"}", + "expr": "tenant_logical_quota{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",id=~\"$Id\"}", "format": "table", "hide": false, "instant": true, @@ -201,36 +225,49 @@ }, { "exemplar": true, - "expr": "tenant_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", tenant=~\"$Tenant\"}", + "expr": "tenant_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",id=~\"$Id\"}", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "D" + }, + { + "exemplar": true, + "expr": "tenant_objects{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",id=~\"$Id\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" } ], "title": "Tenant Quota", "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "id" + } + }, { "id": "filterFieldsByName", "options": { "include": { "names": [ - "cluster", - "datacenter", + "cluster 1", + "datacenter 1", "tenant", "Value #B", "Value #C", - "Value #D" + "Value #D", + "Value #E" ] } } }, - { - "id": "merge", - "options": {} - }, { "id": "organize", "options": { @@ -239,16 +276,19 @@ "Value #B": 3, "Value #C": 4, "Value #D": 5, - "cluster": 1, - "datacenter": 0, + "cluster 1": 1, + "datacenter 1": 0, "tenant": 2 }, "renameByName": { "Value #B": "Logical Used", "Value #C": "Quota", "Value #D": "Used %", + "Value #E": "Objects", "cluster": "Cluster", + "cluster 1": "Cluster", "datacenter": "Datacenter", + "datacenter 1": "Datacenter", "tenant": "Tenant" } } @@ -329,7 +369,7 @@ "targets": [ { "exemplar": true, - "expr": "bucket_bytes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", tenant=~\"$Tenant\"}", + "expr": "bucket_bytes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",tenant=~\"$Tenant\"}", "format": "table", "instant": true, "interval": "", @@ -338,7 +378,7 @@ }, { "exemplar": true, - "expr": "bucket_objects{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", tenant=~\"$Tenant\"}", + "expr": "bucket_objects{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",tenant=~\"$Tenant\"}", "format": "table", "hide": false, "instant": true, @@ -359,7 +399,8 @@ "datacenter", "tenant", "Value #A", - "Value #B" + "Value #B", + "region" ] } } @@ -386,6 +427,7 @@ "bucket": "Bucket", "cluster": "Cluster", "datacenter": "Datacenter", + "region": "Region", "tenant": "Tenant" } } @@ -470,6 +512,29 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "allValue": "", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(tenant_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",tenant=~\"$Tenant\"}, id)", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "Id", + "options": [], + "query": { + "query": "label_values(tenant_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",tenant=~\"$Tenant\"}, id)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -480,6 +545,6 @@ "timepicker": {}, "timezone": "", "title": "StorageGrid: Tenants", - "uid": "IFfVPS4Vz", + "uid": "WZExvrV4z", "version": 1 } \ No newline at end of file