From 6a219b7e0057a228298c017e23edadf868b79224 Mon Sep 17 00:00:00 2001 From: hardikl Date: Fri, 26 Jul 2024 14:53:34 +0530 Subject: [PATCH 1/4] feat: Split qtree/quota rest templates --- cmd/collectors/rest/plugins/qtree/qtree.go | 318 ------------------ .../rest/plugins/qtree/qtree_test.go | 75 ----- .../rest/plugins/qtree/testdata/quota.json | 142 -------- cmd/collectors/rest/plugins/quota/quota.go | 144 ++++++++ cmd/collectors/rest/rest.go | 6 +- conf/rest/9.12.0/qtree.yaml | 12 +- conf/rest/9.12.0/quota.yaml | 42 +++ conf/rest/default.yaml | 1 + 8 files changed, 193 insertions(+), 547 deletions(-) delete mode 100644 cmd/collectors/rest/plugins/qtree/qtree.go delete mode 100644 cmd/collectors/rest/plugins/qtree/qtree_test.go delete mode 100644 cmd/collectors/rest/plugins/qtree/testdata/quota.json create mode 100644 cmd/collectors/rest/plugins/quota/quota.go create mode 100644 conf/rest/9.12.0/quota.yaml diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go deleted file mode 100644 index 54831236f..000000000 --- a/cmd/collectors/rest/plugins/qtree/qtree.go +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Copyright NetApp Inc, 2022 All rights reserved - */ - -package qtree - -import ( - "github.com/netapp/harvest/v2/cmd/collectors" - "github.com/netapp/harvest/v2/cmd/poller/plugin" - "github.com/netapp/harvest/v2/cmd/tools/rest" - "github.com/netapp/harvest/v2/pkg/conf" - "github.com/netapp/harvest/v2/pkg/errs" - "github.com/netapp/harvest/v2/pkg/matrix" - "github.com/netapp/harvest/v2/pkg/tree/node" - "github.com/netapp/harvest/v2/pkg/util" - "github.com/tidwall/gjson" - "strings" - "time" -) - -type Qtree struct { - *plugin.AbstractPlugin - data *matrix.Matrix - instanceKeys map[string]string - instanceLabels map[string]map[string]string - client *rest.Client - query string - quotaType []string - historicalLabels bool // supports labels, metrics for 22.05 - qtreeMetrics bool // supports quota metrics with qtree prefix -} - -func New(p *plugin.AbstractPlugin) plugin.Plugin { - return &Qtree{AbstractPlugin: p} -} - -func (q *Qtree) Init() error { - - var err error - quotaMetric := []string{ - "space.hard_limit => disk_limit", - "space.used.total => disk_used", - "space.used.hard_limit_percent => disk_used_pct_disk_limit", - "space.used.soft_limit_percent => disk_used_pct_soft_disk_limit", - "space.soft_limit => soft_disk_limit", - // "disk-used-pct-threshold" # deprecated and workaround to use same as disk_used_pct_soft_disk_limit - "files.hard_limit => file_limit", - "files.used.total => files_used", - "files.used.hard_limit_percent => files_used_pct_file_limit", - "files.used.soft_limit_percent => files_used_pct_soft_file_limit", - "files.soft_limit => soft_file_limit", - "threshold => threshold", - } - - if err := q.InitAbc(); err != nil { - return err - } - - clientTimeout := q.ParentParams.GetChildContentS("client_timeout") - timeout, _ := time.ParseDuration(rest.DefaultTimeout) - duration, err := time.ParseDuration(clientTimeout) - if err == nil { - timeout = duration - } else { - q.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout") - } - if q.client, err = rest.New(conf.ZapiPoller(q.ParentParams), timeout, q.Auth); err != nil { - q.Logger.Error().Stack().Err(err).Msg("connecting") - return err - } - - if err := q.client.Init(5); err != nil { - return err - } - - q.query = "api/storage/quota/reports" - - q.data = matrix.New(q.Parent+".Qtree", "quota", "quota") - q.instanceKeys = make(map[string]string) - q.instanceLabels = make(map[string]map[string]string) - q.historicalLabels = false - - if q.Params.HasChildS("qtreeMetrics") { - q.qtreeMetrics = true - } - - if q.Params.HasChildS("historicalLabels") { - exportOptions := node.NewS("export_options") - instanceKeys := exportOptions.NewChildS("instance_keys", "") - - // apply all instance keys, instance labels from parent (qtree.yaml) to all quota metrics - if exportOption := q.ParentParams.GetChildS("export_options"); exportOption != nil { - // parent instancekeys would be added in plugin metrics - if parentKeys := exportOption.GetChildS("instance_keys"); parentKeys != nil { - for _, parentKey := range parentKeys.GetAllChildContentS() { - instanceKeys.NewChildS("", parentKey) - } - } - // parent instacelabels would be added in plugin metrics - if parentLabels := exportOption.GetChildS("instance_labels"); parentLabels != nil { - for _, parentLabel := range parentLabels.GetAllChildContentS() { - instanceKeys.NewChildS("", parentLabel) - } - } - } - - instanceKeys.NewChildS("", "type") - instanceKeys.NewChildS("", "unit") - - q.data.SetExportOptions(exportOptions) - q.historicalLabels = true - } - - quotaType := q.Params.GetChildS("quotaType") - if quotaType != nil { - q.quotaType = []string{} - q.quotaType = append(q.quotaType, quotaType.GetAllChildContentS()...) - - } else { - q.quotaType = []string{"user", "group", "tree"} - } - - for _, obj := range quotaMetric { - metricName, display, _, _ := util.ParseMetric(obj) - - _, err := q.data.NewMetricFloat64(metricName, display) - if err != nil { - q.Logger.Error().Stack().Err(err).Msg("add metric") - return err - } - } - - q.Logger.Debug().Msgf("added data with %d metrics", len(q.data.GetMetrics())) - - return nil -} - -func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { - var ( - result []gjson.Result - err error - numMetrics int - ) - data := dataMap[q.Object] - q.client.Metadata.Reset() - - // Purge and reset data - q.data.PurgeInstances() - q.data.Reset() - - // Set all global labels from Rest.go if already not exist - q.data.SetGlobalLabels(data.GetGlobalLabels()) - - filter := []string{"return_unmatched_nested_array_objects=true", "show_default_records=false", "type=" + strings.Join(q.quotaType, "|")} - - // In 22.05, all qtrees were exported - if q.historicalLabels { - for _, qtreeInstance := range data.GetInstances() { - qtreeInstance.SetExportable(true) - } - // In 22.05, we would need default records as well. - filter = []string{"return_unmatched_nested_array_objects=true", "show_default_records=true", "type=" + strings.Join(q.quotaType, "|")} - } - - href := rest.NewHrefBuilder(). - APIPath(q.query). - Fields([]string{"*"}). - Filter(filter). - Build() - - if result, err = collectors.InvokeRestCall(q.client, href, q.Logger); err != nil { - return nil, nil, err - } - - quotaCount := 0 - - // Populate metrics with quota prefix - err = q.handlingQuotaMetrics(result, data, "aCount, &numMetrics) - - if err != nil { - return nil, nil, err - } - - q.client.Metadata.PluginInstances = uint64(quotaCount) - - q.Logger.Info(). - Int("numQuotas", quotaCount). - Int("metrics", numMetrics). - Msg("Collected") - - if q.qtreeMetrics || q.historicalLabels { - // metrics with qtree prefix and quota prefix are available to support backward compatibility - qtreePluginData := q.data.Clone(matrix.With{Data: true, Metrics: true, Instances: true, ExportInstances: true}) - qtreePluginData.UUID = q.Parent + ".Qtree" - qtreePluginData.Object = "qtree" - qtreePluginData.Identifier = "qtree" - return []*matrix.Matrix{qtreePluginData, q.data}, q.client.Metadata, nil - } - return []*matrix.Matrix{q.data}, q.client.Metadata, nil -} - -func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, data *matrix.Matrix, quotaCount *int, numMetrics *int) error { - for _, quota := range result { - var tree string - var qtreeInstance *matrix.Instance - var value float64 - - if !quota.IsObject() { - q.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping") - return errs.New(errs.ErrNoInstance, "quota is not an object") - } - - if quota.Get("qtree.name").Exists() { - tree = quota.Get("qtree.name").String() - } - quotaType := quota.Get("type").String() - volume := quota.Get("volume.name").String() - vserver := quota.Get("svm.name").String() - uName := quota.Get("users.0.name").String() - uid := quota.Get("users.0.id").String() - group := quota.Get("group.name").String() - *quotaCount++ - - // In 22.05, populate metrics with qtree labels - if q.historicalLabels { - // qtree instancekey would be qtree, svm and volume(sorted keys) - qtreeInstance = data.GetInstance(tree + vserver + volume) - if qtreeInstance == nil { - q.Logger.Warn(). - Str("tree", tree). - Str("volume", volume). - Str("vserver", vserver). - Msg("No instance matching tree.vserver.volume") - continue - } - if !qtreeInstance.IsExportable() { - continue - } - } - - for attribute, m := range q.data.GetMetrics() { - // In 22.05, populate metrics value with 0 - if q.historicalLabels { - value = 0.0 - } else { - // set -1 for unlimited - value = -1.0 - } - - quotaInstanceKey := vserver + "." + volume + "." + tree + "." + group + "." + uName + "." + attribute - - quotaInstance, err := q.data.NewInstance(quotaInstanceKey) - if err != nil { - q.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err) - return err - } - - // set labels - quotaInstance.SetLabel("type", quotaType) - quotaInstance.SetLabel("qtree", tree) - quotaInstance.SetLabel("volume", volume) - quotaInstance.SetLabel("svm", vserver) - - // In 22.05, populate metrics with qtree labels - if q.historicalLabels { - for _, label := range q.data.GetExportOptions().GetChildS("instance_keys").GetAllChildContentS() { - if val := qtreeInstance.GetLabel(label); val != "" { - quotaInstance.SetLabel(label, val) - } - } - // If the Qtree is the volume itself, then qtree label is empty, so copy the volume name to qtree. - if tree == "" { - quotaInstance.SetLabel("qtree", volume) - } - } - - if quotaType == "user" { - if uName != "" { - quotaInstance.SetLabel("user", uName) - } else if uid != "" { - quotaInstance.SetLabel("user", uid) - } - } else if quotaType == "group" { - if group != "" { - quotaInstance.SetLabel("group", group) - } else if uid != "" { - quotaInstance.SetLabel("group", uid) - } - } - - if attrValue := quota.Get(attribute); attrValue.Exists() { - // space limits are in bytes, converted to kilobytes to match ZAPI - if attribute == "space.hard_limit" || attribute == "space.soft_limit" || attribute == "space.used.total" { - value = attrValue.Float() / 1024 - quotaInstance.SetLabel("unit", "Kbyte") - if attribute == "space.soft_limit" { - t := q.data.GetMetric("threshold") - if err = t.SetValueFloat64(quotaInstance, value); err != nil { - q.Logger.Error().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value") - } else { - *numMetrics++ - } - } - } else { - value = attrValue.Float() - } - } - - // populate numeric data - if err = m.SetValueFloat64(quotaInstance, value); err != nil { - q.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value") - } else { - *numMetrics++ - } - } - } - return nil -} diff --git a/cmd/collectors/rest/plugins/qtree/qtree_test.go b/cmd/collectors/rest/plugins/qtree/qtree_test.go deleted file mode 100644 index 4d157e8e2..000000000 --- a/cmd/collectors/rest/plugins/qtree/qtree_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package qtree - -import ( - "github.com/netapp/harvest/v2/cmd/poller/plugin" - "github.com/netapp/harvest/v2/pkg/logging" - "github.com/netapp/harvest/v2/pkg/matrix" - "github.com/netapp/harvest/v2/pkg/tree/node" - "github.com/tidwall/gjson" - "os" - "testing" -) - -func NewQtree() *Qtree { - q := &Qtree{AbstractPlugin: plugin.New("qtree", nil, nil, nil, "qtree", nil)} - q.Logger = logging.Get() - q.data = matrix.New(q.Parent+".Qtree", "quota", "quota") - _, _ = q.data.NewMetricFloat64("space.hard_limit", "disk_limit") - _, _ = q.data.NewMetricFloat64("space.used.total", "disk_used") - return q -} - -func TestHandlingQuotaMetrics(t *testing.T) { - jsonResponse, err := os.ReadFile("testdata/quota.json") - if err != nil { - t.Fatalf("Failed to read JSON response from file: %v", err) - } - - result := gjson.Get(string(jsonResponse), "records").Array() - - // Case 1: with historicalLabels = false - q1 := NewQtree() - q1.historicalLabels = false - testLabels(t, q1, result, nil, "astra_300.trident_qtree_pool_trident_TIXRBILLKA.trident_pvc_19913841_a29f_4a54_8bc0_a3c1c4155826...space.hard_limit", 3, 6, 5) - - // Case 2: with historicalLabels = true - q2 := NewQtree() - data := matrix.New(q2.Parent+".Qtree", "qtree", "qtree") - qtreeInstance, _ := data.NewInstance("" + "abcde" + "abcd_root") - qtreeInstance.SetLabel("export_policy", "default") - qtreeInstance.SetLabel("oplocks", "enabled") - qtreeInstance.SetLabel("security_style", "unix") - qtreeInstance.SetLabel("status", "normal") - - exportOptions := node.NewS("export_options") - instanceKeys := exportOptions.NewChildS("instance_keys", "") - // apply all instance keys, instance labels from qtree.yaml to all quota metrics - keys := []string{"export_policy", "oplocks", "security_style", "status"} - for _, key := range keys { - instanceKeys.NewChildS("", key) - } - q2.data.SetExportOptions(exportOptions) - q2.historicalLabels = true - testLabels(t, q2, result, data, "abcde.abcd_root...root.space.used.total", 3, 4, 10) -} - -func testLabels(t *testing.T, q *Qtree, quotas []gjson.Result, data *matrix.Matrix, quotaInstanceKey string, expectedQuotaCount int, expectedQuotaMetricCount int, expectedQuotaLabels int) { - quotaCount := 0 - numMetrics := 0 - err := q.handlingQuotaMetrics(quotas, data, "aCount, &numMetrics) - if err != nil { - t.Errorf("handlingQuotaMetrics returned an error: %v", err) - } - - if quotaCount != expectedQuotaCount { - t.Errorf("quotaCount = %d; want %d", quotaCount, expectedQuotaCount) - } - if numMetrics != expectedQuotaMetricCount { - t.Errorf("numMetrics = %d; want %d", numMetrics, expectedQuotaMetricCount) - } - - quotaInstance := q.data.GetInstance(quotaInstanceKey) - if len(quotaInstance.GetLabels()) != expectedQuotaLabels { - t.Errorf("labels = %d; want %d", len(quotaInstance.GetLabels()), expectedQuotaLabels) - } -} diff --git a/cmd/collectors/rest/plugins/qtree/testdata/quota.json b/cmd/collectors/rest/plugins/qtree/testdata/quota.json deleted file mode 100644 index 6dd78a09c..000000000 --- a/cmd/collectors/rest/plugins/qtree/testdata/quota.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "records": [ - { - "svm": { - "name": "abcde", - "uuid": "0eb01187-2536-11ee-90be-00a098d390f2", - "_links": { - "self": { - "href": "/api/svm/svms/0eb01187-2536-11ee-90be-00a098d390f2" - } - } - }, - "volume": { - "name": "abcd_root", - "uuid": "0ffaa742-2536-11ee-90be-00a098d390f2", - "_links": { - "self": { - "href": "/api/storage/volumes/0ffaa742-2536-11ee-90be-00a098d390f2" - } - } - }, - "index": 0, - "type": "user", - "users": [ - { - "name": "root", - "id": "0" - } - ], - "space": { - "used": { - "total": 0 - } - }, - "files": { - "used": { - "total": 6 - } - }, - "_links": { - "self": { - "href": "/api/storage/quota/reports/0ffaa742-2536-11ee-90be-00a098d390f2/0" - } - } - }, - { - "svm": { - "name": "abcde", - "uuid": "0eb01187-2536-11ee-90be-00a098d390f2", - "_links": { - "self": { - "href": "/api/svm/svms/0eb01187-2536-11ee-90be-00a098d390f2" - } - } - }, - "volume": { - "name": "abcd_root", - "uuid": "0ffaa742-2536-11ee-90be-00a098d390f2", - "_links": { - "self": { - "href": "/api/storage/volumes/0ffaa742-2536-11ee-90be-00a098d390f2" - } - } - }, - "index": 1152921504606846976, - "type": "group", - "group": { - "name": "root", - "id": "0" - }, - "space": { - "used": { - "total": 0 - } - }, - "files": { - "used": { - "total": 6 - } - }, - "_links": { - "self": { - "href": "/api/storage/quota/reports/0ffaa742-2536-11ee-90be-00a098d390f2/1152921504606846976" - } - } - }, - { - "svm": { - "name": "astra_300", - "uuid": "835c0d25-ceb0-11eb-80e3-00a098d39e12", - "_links": { - "self": { - "href": "/api/svm/svms/835c0d25-ceb0-11eb-80e3-00a098d39e12" - } - } - }, - "volume": { - "name": "trident_qtree_pool_trident_TIXRBILLKA", - "uuid": "60972440-6fa6-11ed-a5cd-00a098d39e12", - "_links": { - "self": { - "href": "/api/storage/volumes/60972440-6fa6-11ed-a5cd-00a098d39e12" - } - } - }, - "index": 2305843013508661248, - "type": "tree", - "qtree": { - "name": "trident_pvc_19913841_a29f_4a54_8bc0_a3c1c4155826", - "id": 1, - "_links": { - "self": { - "href": "/api/storage/qtrees/60972440-6fa6-11ed-a5cd-00a098d39e12/1" - } - } - }, - "space": { - "hard_limit": 2147483648, - "used": { - "total": 0, - "hard_limit_percent": 0 - } - }, - "files": { - "used": { - "total": 1 - } - }, - "_links": { - "self": { - "href": "/api/storage/quota/reports/60972440-6fa6-11ed-a5cd-00a098d39e12/2305843013508661248" - } - } - } - ], - "num_records": 3, - "_links": { - "self": { - "href": "/api/storage/quota/reports?fields=%2A&return_records=true&show_default_records=false" - } - } -} \ No newline at end of file diff --git a/cmd/collectors/rest/plugins/quota/quota.go b/cmd/collectors/rest/plugins/quota/quota.go new file mode 100644 index 000000000..680aeec40 --- /dev/null +++ b/cmd/collectors/rest/plugins/quota/quota.go @@ -0,0 +1,144 @@ +package quota + +import ( + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/cmd/tools/rest" + "github.com/netapp/harvest/v2/pkg/conf" + "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" + "time" +) + +type Quota struct { + *plugin.AbstractPlugin + client *rest.Client + qtreeMetrics bool // supports quota metrics with qtree prefix +} + +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &Quota{AbstractPlugin: p} +} + +func (q *Quota) Init() error { + if err := q.InitAbc(); err != nil { + return err + } + + clientTimeout := q.ParentParams.GetChildContentS("client_timeout") + timeout, _ := time.ParseDuration(rest.DefaultTimeout) + duration, err := time.ParseDuration(clientTimeout) + if err == nil { + timeout = duration + } else { + q.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout") + } + if q.client, err = rest.New(conf.ZapiPoller(q.ParentParams), timeout, q.Auth); err != nil { + q.Logger.Error().Stack().Err(err).Msg("connecting") + return err + } + + if err := q.client.Init(5); err != nil { + return err + } + if q.Params.HasChildS("qtreeMetrics") { + q.qtreeMetrics = true + } + return nil +} + +func (q *Quota) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { + data := dataMap[q.Object] + q.client.Metadata.Reset() + + // Purge and reset data + instanceMap := data.GetInstances() + metricsMap := data.GetMetrics() + data.PurgeInstances() + data.PurgeMetrics() + + for metricName, m := range metricsMap { + _, err := data.NewMetricFloat64(metricName, m.GetName()) + if err != nil { + q.Logger.Error().Stack().Err(err).Msg("add metric") + } + } + + if err := q.handlingQuotaMetrics(instanceMap, metricsMap, data); err != nil { + return nil, nil, err + } + + if q.qtreeMetrics { + // metrics with qtree prefix and quota prefix are available to support backward compatibility + qtreePluginData := data.Clone(matrix.With{Data: true, Metrics: true, Instances: true, ExportInstances: true}) + qtreePluginData.UUID = q.Parent + ".Qtree" + qtreePluginData.Object = "qtree" + qtreePluginData.Identifier = "qtree" + return []*matrix.Matrix{qtreePluginData}, q.client.Metadata, nil + } + return nil, q.client.Metadata, nil +} + +func (q *Quota) handlingQuotaMetrics(instanceMap map[string]*matrix.Instance, metricMap map[string]*matrix.Metric, data *matrix.Matrix) error { + for _, quota := range instanceMap { + if !quota.IsExportable() { + continue + } + index := quota.GetLabel("index") + uName := quota.GetLabel("userName") + uid := quota.GetLabel("userId") + group := quota.GetLabel("groupName") + quotaType := quota.GetLabel("type") + + if quotaType == "user" { + if uName != "" { + quota.SetLabel("user", uName) + } else if uid != "" { + quota.SetLabel("user", uid) + } + } else if quotaType == "group" { + if group != "" { + quota.SetLabel("group", group) + } else if uid != "" { + quota.SetLabel("group", uid) + } + } + + for metricName, m := range metricMap { + // set -1 for unlimited + value := -1.0 + quotaInstanceKey := index + metricName + quotaInstance, err := data.NewInstance(quotaInstanceKey) + if err != nil { + q.Logger.Debug().Msgf("add (%s) instance: %v", metricName, err) + return err + } + // set labels + for k, v := range quota.GetLabels() { + quotaInstance.SetLabel(k, v) + } + + if v, ok := m.GetValueFloat64(quota); ok { + // space limits are in bytes, converted to kilobytes to match ZAPI + if metricName == "space.hard_limit" || metricName == "space.soft_limit" || metricName == "space.used.total" { + value = v / 1024 + quotaInstance.SetLabel("unit", "Kbyte") + if metricName == "space.soft_limit" { + t := data.GetMetric("threshold") + if err := t.SetValueFloat64(quotaInstance, value); err != nil { + q.Logger.Error().Err(err).Str("metricName", metricName).Float64("value", value).Msg("Failed to parse value") + } + } + } else { + value = v + } + } + + // populate numeric data + t := data.GetMetric(metricName) + if err = t.SetValueFloat64(quotaInstance, value); err != nil { + q.Logger.Error().Stack().Err(err).Str("metricName", metricName).Float64("value", value).Msg("Failed to parse value") + } + } + } + return nil +} diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go index 15bd029d5..5afd7e6b2 100644 --- a/cmd/collectors/rest/rest.go +++ b/cmd/collectors/rest/rest.go @@ -13,7 +13,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/ontaps3service" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qospolicyadaptive" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qospolicyfixed" - "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qtree" + "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/quota" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/securityaccount" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/shelf" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/snapmirror" @@ -493,8 +493,8 @@ func (r *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin return health.New(abc) case "NetRoute": return netroute.New(abc) - case "Qtree": - return qtree.New(abc) + case "Quota": + return quota.New(abc) case "Snapmirror": return snapmirror.New(abc) case "Volume": diff --git a/conf/rest/9.12.0/qtree.yaml b/conf/rest/9.12.0/qtree.yaml index 309ac5129..20e9228a5 100644 --- a/conf/rest/9.12.0/qtree.yaml +++ b/conf/rest/9.12.0/qtree.yaml @@ -1,5 +1,3 @@ -# If you need to collect quotas from a cluster that has no qtrees, remove the filter lines below. -# Refer https://github.com/NetApp/harvest/discussions/3057 for more details. name: Qtree query: api/storage/qtrees object: qtree @@ -13,7 +11,7 @@ counters: - ^export_policy.name => export_policy - ^security_style => security_style - id => id - - filter: # If you need to collect quotas from a cluster with no qtrees, remove this line and the next one. + - filter: - name=!"" @@ -25,14 +23,10 @@ endpoints: - ^^vserver => svm - ^oplock_mode => oplocks - ^status => status - - filter: # If you need to collect quotas from a cluster with no qtrees, remove this line and the next one. + - filter: - qtree=!"" + plugins: - - Qtree: - quotaType: - - tree -# - user -# - group - LabelAgent: replace: - oplocks oplocks `enable` `enabled` diff --git a/conf/rest/9.12.0/quota.yaml b/conf/rest/9.12.0/quota.yaml new file mode 100644 index 000000000..ca8304073 --- /dev/null +++ b/conf/rest/9.12.0/quota.yaml @@ -0,0 +1,42 @@ +name: Quota +query: api/storage/quota/reports +object: quota + +client_timeout: 2m + +counters: + - ^^index => index + - ^group.name => groupName + - ^qtree.name => qtree + - ^svm.name => svm + - ^type => type + - ^users.0.id => userId + - ^users.0.name => userName + - ^volume.name => volume + - files.hard_limit => file_limit + - files.soft_limit => soft_file_limit + - files.used.hard_limit_percent => files_used_pct_file_limit + - files.used.soft_limit_percent => files_used_pct_soft_file_limit + - files.used.total => files_used + - space.hard_limit => disk_limit + - space.soft_limit => soft_disk_limit + - space.used.hard_limit_percent => disk_used_pct_disk_limit + - space.used.soft_limit_percent => disk_used_pct_soft_disk_limit + - space.used.total => disk_used + - threshold => threshold + - filter: + - show_default_records=false + - type=tree #|user|group + +plugins: + - Quota: + +export_options: + instance_keys: + - group + - qtree + - svm + - type + - unit + - user + - volume \ No newline at end of file diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml index 6676c43b8..d1ab7f59e 100644 --- a/conf/rest/default.yaml +++ b/conf/rest/default.yaml @@ -40,6 +40,7 @@ objects: QosPolicyFixed: qos_policy_fixed.yaml QosWorkload: qos_workload.yaml Qtree: qtree.yaml + Quota: quota.yaml Security: security.yaml SecurityAccount: security_account.yaml SecurityAuditDestination: security_audit_dest.yaml From 68926f54fe950be79f59070fcdbf8ad7e1d0af37 Mon Sep 17 00:00:00 2001 From: hardikl Date: Fri, 26 Jul 2024 19:51:24 +0530 Subject: [PATCH 2/4] feat: handled review comment --- cmd/collectors/rest/plugins/quota/quota.go | 29 ++-------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/cmd/collectors/rest/plugins/quota/quota.go b/cmd/collectors/rest/plugins/quota/quota.go index 680aeec40..e9cf11de1 100644 --- a/cmd/collectors/rest/plugins/quota/quota.go +++ b/cmd/collectors/rest/plugins/quota/quota.go @@ -2,16 +2,12 @@ package quota import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" - "github.com/netapp/harvest/v2/cmd/tools/rest" - "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/util" - "time" ) type Quota struct { *plugin.AbstractPlugin - client *rest.Client qtreeMetrics bool // supports quota metrics with qtree prefix } @@ -20,26 +16,6 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } func (q *Quota) Init() error { - if err := q.InitAbc(); err != nil { - return err - } - - clientTimeout := q.ParentParams.GetChildContentS("client_timeout") - timeout, _ := time.ParseDuration(rest.DefaultTimeout) - duration, err := time.ParseDuration(clientTimeout) - if err == nil { - timeout = duration - } else { - q.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout") - } - if q.client, err = rest.New(conf.ZapiPoller(q.ParentParams), timeout, q.Auth); err != nil { - q.Logger.Error().Stack().Err(err).Msg("connecting") - return err - } - - if err := q.client.Init(5); err != nil { - return err - } if q.Params.HasChildS("qtreeMetrics") { q.qtreeMetrics = true } @@ -48,7 +24,6 @@ func (q *Quota) Init() error { func (q *Quota) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[q.Object] - q.client.Metadata.Reset() // Purge and reset data instanceMap := data.GetInstances() @@ -73,9 +48,9 @@ func (q *Quota) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util. qtreePluginData.UUID = q.Parent + ".Qtree" qtreePluginData.Object = "qtree" qtreePluginData.Identifier = "qtree" - return []*matrix.Matrix{qtreePluginData}, q.client.Metadata, nil + return []*matrix.Matrix{qtreePluginData}, nil, nil } - return nil, q.client.Metadata, nil + return nil, nil, nil } func (q *Quota) handlingQuotaMetrics(instanceMap map[string]*matrix.Instance, metricMap map[string]*matrix.Metric, data *matrix.Matrix) error { From 28d1265392650731b43cea62cbcf8edbdb1d3823 Mon Sep 17 00:00:00 2001 From: hardikl Date: Mon, 29 Jul 2024 12:52:04 +0530 Subject: [PATCH 3/4] feat: handled review comment --- cmd/collectors/rest/plugins/quota/quota.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/collectors/rest/plugins/quota/quota.go b/cmd/collectors/rest/plugins/quota/quota.go index e9cf11de1..4c0ea842e 100644 --- a/cmd/collectors/rest/plugins/quota/quota.go +++ b/cmd/collectors/rest/plugins/quota/quota.go @@ -93,7 +93,7 @@ func (q *Quota) handlingQuotaMetrics(instanceMap map[string]*matrix.Instance, me } if v, ok := m.GetValueFloat64(quota); ok { - // space limits are in bytes, converted to kilobytes to match ZAPI + // space limits are in bytes, converted to kibibytes to match ZAPI if metricName == "space.hard_limit" || metricName == "space.soft_limit" || metricName == "space.used.total" { value = v / 1024 quotaInstance.SetLabel("unit", "Kbyte") From fb1d4bf671c2a3ac22502b48644ab3302957ed81 Mon Sep 17 00:00:00 2001 From: hardikl Date: Mon, 29 Jul 2024 19:11:09 +0530 Subject: [PATCH 4/4] feat: handled review comment --- cmd/collectors/rest/plugins/quota/quota.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/collectors/rest/plugins/quota/quota.go b/cmd/collectors/rest/plugins/quota/quota.go index 4c0ea842e..1f026d392 100644 --- a/cmd/collectors/rest/plugins/quota/quota.go +++ b/cmd/collectors/rest/plugins/quota/quota.go @@ -96,7 +96,7 @@ func (q *Quota) handlingQuotaMetrics(instanceMap map[string]*matrix.Instance, me // space limits are in bytes, converted to kibibytes to match ZAPI if metricName == "space.hard_limit" || metricName == "space.soft_limit" || metricName == "space.used.total" { value = v / 1024 - quotaInstance.SetLabel("unit", "Kbyte") + quotaInstance.SetLabel("unit", "kibibytes") if metricName == "space.soft_limit" { t := data.GetMetric("threshold") if err := t.SetValueFloat64(quotaInstance, value); err != nil {