feat: add StorageGrid collector and dashboard

cmd/collectors/rest/rest.go

@@ -220,11 +220,11 @@ func (r *Rest) initEndPoints() error {
 	return nil
 }
 
-func (r *Rest) getTemplateFn() string {
+func TemplateFn(n *node.Node, obj string) string {
 	var fn string
-	objects := r.Params.GetChildS("objects")
+	objects := n.GetChildS("objects")
 	if objects != nil {
-		fn = objects.GetChildContentS(r.Object)
+		fn = objects.GetChildContentS(obj)
 	}
 	return fn
 }

cmd/collectors/rest/templating.go

@@ -20,7 +20,11 @@ func (r *Rest) LoadTemplate() (string, error) {
 	)
 
 	// import template
-	if template, templatePath, err = r.ImportSubTemplate("", r.getTemplateFn(), r.Client.Cluster().Version); err != nil {
+	if template, templatePath, err = r.ImportSubTemplate(
+		"",
+		TemplateFn(r.Params, r.Object),
+		r.Client.Cluster().Version,
+	); err != nil {
 		return "", err
 	}
 
@@ -139,14 +143,14 @@ func HandleDuration(value string) float64 {
 	return 0
 }
 
+// Example: timestamp: 2020-12-02T18:36:19-08:00
+var regexTimeStamp = regexp.MustCompile(
+	`[+-]?\d{4}(-[01]\d(-[0-3]\d(T[0-2]\d:[0-5]\d:?([0-5]\d(\.\d+)?)?[+-][0-2]\d:[0-5]\d?)?)?)?`)
+
 func HandleTimestamp(value string) float64 {
 	var timestamp time.Time
 	var err error
 
-	// Example: timestamp: 2020-12-02T18:36:19-08:00
-	timestampRegex := `[+-]?\d{4}(-[01]\d(-[0-3]\d(T[0-2]\d:[0-5]\d:?([0-5]\d(\.\d+)?)?[+-][0-2]\d:[0-5]\d?)?)?)?`
-
-	regexTimeStamp := regexp.MustCompile(timestampRegex)
 	if match := regexTimeStamp.MatchString(value); match {
 		// example: 2020-12-02T18:36:19-08:00   ==>  1606962979
 		if timestamp, err = time.Parse(time.RFC3339, value); err != nil {

cmd/collectors/storagegrid/plugins/bucket/bucket.go

@@ -0,0 +1,115 @@
+package bucket
+
+import (
+	"github.com/netapp/harvest/v2/cmd/collectors/storagegrid/rest"
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/tidwall/gjson"
+)
+
+type Bucket struct {
+	*plugin.AbstractPlugin
+	client *rest.Client
+	data   *matrix.Matrix
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &Bucket{AbstractPlugin: p}
+}
+
+func (b *Bucket) Init() error {
+
+	var err error
+
+	if err = b.InitAbc(); err != nil {
+		return err
+	}
+
+	if b.client, err = rest.NewClient(b.Options.Poller, ""); err != nil {
+		b.Logger.Error().Stack().Err(err).Msg("connecting")
+		return err
+	}
+
+	if err = b.client.Init(5); err != nil {
+		return err
+	}
+
+	b.data = matrix.New(b.Parent+".Bucket", "bucket", "bucket")
+	_, _ = b.data.NewMetricFloat64("objects")
+	_, _ = b.data.NewMetricFloat64("bytes")
+
+	return nil
+}
+
+func (b *Bucket) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
+	var (
+		instanceKey string
+	)
+
+	metricToJSON := map[string]string{
+		"objects": "objectCount",
+		"bytes":   "dataBytes",
+	}
+	// Purge and reset data
+	b.data.PurgeInstances()
+	b.data.Reset()
+
+	// Set all global labels from Rest.go if already not exist
+	b.data.SetGlobalLabels(data.GetGlobalLabels())
+
+	// request the buckets for each tenant
+	for instKey, inst := range data.GetInstances() {
+		href := "grid/accounts/" + instKey + "/usage?includeBucketDetail=true"
+		var records []gjson.Result
+		err := b.client.Fetch(href, &records)
+		tenantName := inst.GetLabel("tenant")
+		if err != nil {
+			b.Logger.Error().Err(err).
+				Str("id", instKey).
+				Str("tenantName", tenantName).
+				Msg("Unable to fetch bucket details")
+			continue
+		}
+
+		for _, record := range records {
+			if !record.IsObject() {
+				b.Logger.Warn().Str("type", record.Type.String()).Msg("Bucket is not object, skipping")
+				continue
+			}
+
+			bucketsJSON := record.Get("buckets")
+			for _, bucketJSON := range bucketsJSON.Array() {
+				bucket := bucketJSON.Get("name").String()
+
+				instanceKey = instKey + "#" + bucket
+				bucketInstance, err2 := b.data.NewInstance(instanceKey)
+				if err2 != nil {
+					b.Logger.Error().Err(err).Str("instanceKey", instanceKey).Msg("Failed to add instance")
+					break
+				}
+				b.Logger.Debug().Str("instanceKey", instanceKey).Msg("add instance")
+				bucketInstance.SetLabel("bucket", bucket)
+				bucketInstance.SetLabel("tenant", tenantName)
+				for metricKey, m := range b.data.GetMetrics() {
+					jsonKey := metricToJSON[metricKey]
+					if value := bucketJSON.Get(jsonKey); value.Exists() {
+						if err = m.SetValueString(bucketInstance, value.String()); err != nil {
+							b.Logger.Error().Err(err).
+								Str("key", metricKey).
+								Str("metric", m.GetName()).
+								Str("value", value.String()).
+								Msg("Unable to set float key on metric")
+						} else {
+							b.Logger.Debug().
+								Str("metricKey", metricKey).
+								Str("value", value.String()).
+								Msg("added")
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return []*matrix.Matrix{b.data}, nil
+}

cmd/collectors/storagegrid/plugins/tenant/tenant.go

@@ -0,0 +1,62 @@
+package tenant
+
+import (
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/errs"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+)
+
+type Tenant struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &Tenant{AbstractPlugin: p}
+}
+
+func (t *Tenant) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
+
+	var (
+		used, quota, usedPercent matrix.Metric
+		err                      error
+	)
+
+	if used = data.GetMetric("dataBytes"); used == nil {
+		return nil, errs.New(errs.ErrNoMetric, "logical_used")
+	}
+
+	if quota = data.GetMetric("policy.quotaObjectBytes"); quota == nil {
+		return nil, errs.New(errs.ErrNoMetric, "logical_quota")
+	}
+
+	if usedPercent = data.GetMetric("used_percent"); usedPercent == nil {
+		if usedPercent, err = data.NewMetricFloat64("used_percent"); err == nil {
+			usedPercent.SetProperty("raw")
+		} else {
+			return nil, err
+		}
+	}
+
+	for _, instance := range data.GetInstances() {
+
+		var (
+			usedBytes, quotaBytes, percentage    float64
+			usedOK, usedPass, quotaOK, quotaPass bool
+		)
+
+		usedBytes, usedOK, usedPass = used.GetValueFloat64(instance)
+		quotaBytes, quotaOK, quotaPass = quota.GetValueFloat64(instance)
+		if (usedOK || usedPass) && (quotaOK || quotaPass) {
+			percentage = usedBytes / quotaBytes * 100
+			if quotaBytes == 0 {
+				percentage = 0
+			}
+			err := usedPercent.SetValueFloat64(instance, percentage)
+			if err != nil {
+				t.Logger.Error().Err(err).Float64("percentage", percentage).Msg("failed to set percentage")
+			}
+		}
+	}
+
+	return nil, nil
+}