feat: sniff latest api version and provide template override

- use tenant id instead of name to discriminate tenants
- include region column in buckets panel

cmd/collectors/storagegrid/plugins/bucket/bucket.go

@@ -80,6 +80,7 @@ func (b *Bucket) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 			bucketsJSON := record.Get("buckets")
 			for _, bucketJSON := range bucketsJSON.Array() {
 				bucket := bucketJSON.Get("name").String()
+				region := bucketJSON.Get("region").String()
 
 				instanceKey = instKey + "#" + bucket
 				bucketInstance, err2 := b.data.NewInstance(instanceKey)
@@ -90,6 +91,7 @@ func (b *Bucket) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 				b.Logger.Debug().Str("instanceKey", instanceKey).Msg("add instance")
 				bucketInstance.SetLabel("bucket", bucket)
 				bucketInstance.SetLabel("tenant", tenantName)
+				bucketInstance.SetLabel("region", region)
 				for metricKey, m := range b.data.GetMetrics() {
 					jsonKey := metricToJSON[metricKey]
 					if value := bucketJSON.Get(jsonKey); value.Exists() {

cmd/collectors/storagegrid/rest/client.go

@@ -20,7 +20,8 @@ import (
 )
 
 const (
-	DefaultTimeout = "1m"
+	DefaultTimeout    = "1m"
+	DefaultAPIVersion = "3"
 )
 
 type Client struct {
@@ -35,7 +36,7 @@ type Client struct {
 	token    string
 	Timeout  time.Duration
 	logRest  bool // used to log Rest request/response
-	apiPath  string
+	APIPath  string
 }
 
 type Cluster struct {
@@ -94,7 +95,6 @@ func New(poller conf.Poller, timeout time.Duration) (*Client, error) {
 
 	client.baseURL = href
 	client.Timeout = timeout
-	client.apiPath = "/api/v3/"
 
 	// by default, enforce secure TLS, if not requested otherwise by user
 	if x := poller.UseInsecureTLS; x != nil {
@@ -187,7 +187,7 @@ func (c *Client) Fetch(request string, result *[]gjson.Result) error {
 		err     error
 		fetched []byte
 	)
-	fetched, err = c.GetRest(request)
+	fetched, err = c.GetGridRest(request)
 	if err != nil {
 		return fmt.Errorf("error making request %w", err)
 	}
@@ -200,33 +200,50 @@ func (c *Client) Fetch(request string, result *[]gjson.Result) error {
 	return nil
 }
 
-// GetRest makes a REST request to the cluster and returns a json response as a []byte
+// GetGridRest makes a grid API request to the cluster and returns a json response as a []byte
 // see also Fetch
-func (c *Client) GetRest(request string) ([]byte, error) {
-	var err error
-	u, err := url.JoinPath(c.baseURL, c.apiPath, request)
+func (c *Client) GetGridRest(request string) ([]byte, error) {
+	u, err := url.JoinPath(c.baseURL, c.APIPath, request)
 	if err != nil {
 		return nil, fmt.Errorf("failed to join URL %s err: %w", request, err)
 	}
-	u2, err2 := url.QueryUnescape(u)
-	if err2 != nil {
-		return nil, fmt.Errorf("failed to unescape URL %s err: %w", u, err2)
+	return c.getRest(u)
+}
+
+// GetMetricQuery makes a metrics API request to the cluster and fills the result argument
+func (c *Client) GetMetricQuery(metric string, result *[]gjson.Result) error {
+	u, err := url.JoinPath(c.baseURL, "/metrics/api/v1/query?query="+metric)
+	if err != nil {
+		return fmt.Errorf("failed to query metric %s err: %w", metric, err)
 	}
 
-	c.request, err = http.NewRequest("GET", u2, nil)
+	fetched, err := c.getRest(u)
+	if err != nil {
+		return err
+	}
+	output := gjson.GetManyBytes(fetched, "data")
+	data := output[0]
+	for _, r := range data.Array() {
+		*result = append(*result, r.Array()...)
+	}
+	return nil
+}
+
+// getRest makes a request to the cluster and returns a json response as a []byte
+// see also Fetch
+func (c *Client) getRest(request string) ([]byte, error) {
+	u, err := url.QueryUnescape(request)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unescape %s err: %w", request, err)
+	}
+
+	c.request, err = http.NewRequest("GET", u, nil)
 	if err != nil {
 		return nil, err
 	}
 	c.request.Header.Set("accept", "application/json")
 	c.request.Header.Set("Authorization", "Bearer "+c.token)
-	// ensure that we can change body dynamically
-	c.request.GetBody = func() (io.ReadCloser, error) {
-		r := bytes.NewReader(c.buffer.Bytes())
-		return io.NopCloser(r), nil
-	}
-
-	result, err := c.invoke()
-	return result, err
+	return c.invoke()
 }
 
 func (c *Client) invoke() ([]byte, error) {
@@ -292,36 +309,37 @@ func (c *Client) fetch() ([]byte, error) {
 	return body, nil
 }
 
+// Init is responsible for determining the StorageGrid server version, API version, and hostname
 func (c *Client) Init(retries int) error {
 	var (
 		err     error
 		content []byte
 		i       int
 	)
 
-	// Product version and cluster name are two separate endpoints
 	for i = 0; i < retries; i++ {
-		if content, err = c.GetRest("grid/config/product-version"); err != nil {
+		// Determine which API versions are supported and then request
+		// product version and cluster name - both of which are separate endpoints
+
+		err = c.sniffAPIVersion(retries)
+		if err != nil {
 			continue
 		}
 
+		if content, err = c.GetGridRest("grid/config/product-version"); err != nil {
+			continue
+		}
 		results := gjson.GetManyBytes(content, "data.productVersion")
 		err = c.SetVersion(results[0].String())
 		if err != nil {
 			return err
 		}
-	}
-	if err != nil {
-		return err
-	}
 
-	err = nil
-	for i = 0; i < retries; i++ {
-		if content, err = c.GetRest("grid/config"); err != nil {
+		if content, err = c.GetGridRest("grid/config"); err != nil {
 			continue
 		}
 
-		results := gjson.GetManyBytes(content, "data.hostname")
+		results = gjson.GetManyBytes(content, "data.hostname")
 		c.Cluster.Name = results[0].String()
 		return nil
 	}
@@ -358,7 +376,7 @@ func (c *Client) fetchToken() error {
 		response *http.Response
 		body     []byte
 	)
-	u, err := url.JoinPath(c.baseURL, c.apiPath, "authorize")
+	u, err := url.JoinPath(c.baseURL, c.APIPath, "authorize")
 	if err != nil {
 		return fmt.Errorf("failed to create auth URL err: %w", err)
 	}
@@ -410,3 +428,35 @@ func (c *Client) fetchToken() error {
 	}
 	return nil
 }
+
+func (c *Client) sniffAPIVersion(retries int) error {
+	// This endpoint does not require auth and uses the /api/ endpoint instead of a versioned one
+
+	var (
+		apiVersion = DefaultAPIVersion
+		u          string
+		err        error
+		i          int
+	)
+
+	u, err = url.JoinPath(c.baseURL, "/api/versions")
+	if err != nil {
+		return fmt.Errorf("failed to join getApiVersions %s err: %w", c.baseURL, err)
+	}
+	for i = 0; i < retries; i++ {
+		result, err := c.getRest(u)
+		if err != nil {
+			continue
+		}
+		versionB := gjson.GetBytes(result, "data")
+		if versionB.Exists() && versionB.IsArray() {
+			versions := versionB.Array()
+			if len(versions) > 0 {
+				apiVersion = versions[len(versions)-1].String()
+			}
+		}
+		c.APIPath = "/api/v" + apiVersion
+		return nil
+	}
+	return err
+}

cmd/collectors/storagegrid/storagegrid.go

@@ -64,6 +64,7 @@ func (s *StorageGrid) Init(a *collector.AbstractCollector) error {
 	if s.Props.TemplatePath, err = s.LoadTemplate(); err != nil {
 		return err
 	}
+	s.InitAPIPath()
 	if err = collector.Init(s); err != nil {
 		return err
 	}
@@ -291,9 +292,8 @@ func (s *StorageGrid) handleResults(result []gjson.Result) uint64 {
 
 func (s *StorageGrid) initClient() error {
 	var err error
-	a := s.AbstractCollector
 
-	if s.client, err = srest.NewClient(s.Options.Poller, a.Params.GetChildContentS("client_timeout")); err != nil {
+	if s.client, err = srest.NewClient(s.Options.Poller, s.Params.GetChildContentS("client_timeout")); err != nil {
 		return err
 	}
 
@@ -377,6 +377,20 @@ func (s *StorageGrid) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin
 	return nil
 }
 
+// InitAPIPath reads the REST API version from the template and uses it instead of
+// the DefaultAPIVersion
+func (s *StorageGrid) InitAPIPath() {
+	apiVersion := s.Params.GetChildContentS("api")
+	if !strings.HasSuffix(s.client.APIPath, apiVersion) {
+		cur := s.client.APIPath
+		s.client.APIPath = "/apiVersion/" + apiVersion
+		s.Logger.Debug().
+			Str("clientAPI", cur).
+			Str("templateAPI", apiVersion).
+			Msg("Use template apiVersion")
+	}
+}
+
 // Interface guards
 var (
 	_ collector.Collector = (*StorageGrid)(nil)

cmd/poller/collector/collector.go

@@ -124,7 +124,7 @@ func New(name, object string, options *options.Options, params *node.Node) *Abst
 // inside its Init method, or leave it to be called
 // by the poller during dynamic load.
 //
-// The important thing done here is too look what tasks are defined
+// The important thing done here is to look what tasks are defined
 // in the "schedule" parameter of the collector and create a pointer
 // to the corresponding method of the collector. Example, parameter is:
 //

conf/storagegrid/11.6.0/tenant.yaml

@@ -2,6 +2,7 @@
 name:                       Tenant
 query:                      grid/accounts-cache
 object:                     tenant
+api:                        v3
 
 counters:
   - ^^id                    => id
@@ -15,4 +16,7 @@ plugins:
   - Bucket
 
 export_options:
-  include_all_labels: true
+  instance_keys:
+    - id
+  instance_labels:
+    - tenant