From 9db5393530b0e22f5fb314048a32c1cb4251c81f Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Thu, 20 Oct 2022 20:45:46 +0530 Subject: [PATCH] feat: quota metric renaming (#1345) * feat: quota metric renaming * feat: type change * feat: rest side changes + retain older qtree plugin metrics * feat: handle instances/metrics review comments * feat: handle review comments * feat: missed to add the file * feat: deprecation in changelog --- CHANGELOG.md | 11 ++++++++++ cmd/collectors/rest/plugins/qtree/qtree.go | 14 ++++++------- cmd/collectors/zapi/plugins/qtree/qtree.go | 14 +++++++------ grafana/dashboards/cmode/qtree.json | 16 +++++++-------- grafana/dashboards/cmode/quotaReport.json | 24 +++++++++++----------- integration/test/data/counter_data.go | 4 ++-- 6 files changed, 48 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd72d8ece..62109dd1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ [Releases](https://github.com/NetApp/harvest/releases) +## 22.11.0 / 2022-11-10 + + +:rocket: Highlights of this major release include: + +**Deprecation**: Earlier versions of Harvest published quota metrics prefixed with qtree. Harvest release 22.11 deprecates the quota metrics prefixed with qtree and instead publishes quota metrics prefixed with quota. +All dashboards have been updated. If you are consuming these metrics outside the default dashboards, please update them to use the quota prefixed metrics. Harvest release 23.02 will remove the deprecated quota metrics +prefixed with qtree. + +--- + ## 22.08.0 / 2022-08-19 diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go index b2652456c..69b390cfd 100644 --- a/cmd/collectors/rest/plugins/qtree/qtree.go +++ b/cmd/collectors/rest/plugins/qtree/qtree.go @@ -12,7 +12,6 @@ import ( "github.com/netapp/harvest/v2/pkg/dict" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" - "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "strconv" @@ -75,13 +74,10 @@ func (my *Qtree) Init() error { my.query = "api/storage/quota/reports" - my.data = matrix.New(my.Parent+".Qtree", "qtree", "qtree") + my.data = matrix.New(my.Parent+".Qtree", "quota", "quota") my.instanceKeys = make(map[string]string) my.instanceLabels = make(map[string]*dict.Dict) - exportOptions := node.NewS("export_options") - exportOptions.NewChildS("include_all_labels", "true") - quotaType := my.Params.GetChildS("quotaType") if quotaType != nil { my.quotaType = []string{} @@ -105,7 +101,6 @@ func (my *Qtree) Init() error { } my.Logger.Debug().Msgf("added data with %d metrics", len(my.data.GetMetrics())) - my.data.SetExportOptions(exportOptions) return nil } @@ -210,5 +205,10 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { Int("metrics", numMetrics). Msg("Collected") - return []*matrix.Matrix{my.data}, nil + // This would generate quota metrics prefix with `qtree_`. These are deprecated now and will be removed later. + qtreePluginData := my.data.Clone(true, true, true) + qtreePluginData.UUID = my.Parent + ".Qtree" + qtreePluginData.Object = "qtree" + qtreePluginData.Identifier = "qtree" + return []*matrix.Matrix{qtreePluginData, my.data}, nil } diff --git a/cmd/collectors/zapi/plugins/qtree/qtree.go b/cmd/collectors/zapi/plugins/qtree/qtree.go index 083f15484..fc33e7567 100644 --- a/cmd/collectors/zapi/plugins/qtree/qtree.go +++ b/cmd/collectors/zapi/plugins/qtree/qtree.go @@ -58,13 +58,10 @@ func (my *Qtree) Init() error { } my.Logger.Debug().Msg("plugin connected!") - my.data = matrix.New(my.Parent+".Qtree", "qtree", "qtree") + my.data = matrix.New(my.Parent+".Qtree", "quota", "quota") my.instanceKeys = make(map[string]string) my.instanceLabels = make(map[string]*dict.Dict) - exportOptions := node.NewS("export_options") - exportOptions.NewChildS("include_all_labels", "true") - objects := my.Params.GetChildS("objects") if objects == nil { return errs.New(errs.ErrMissingParam, "objects") @@ -93,7 +90,6 @@ func (my *Qtree) Init() error { } my.Logger.Debug().Msgf("added data with %d metrics", len(my.data.GetMetrics())) - my.data.SetExportOptions(exportOptions) // setup batchSize for request if my.client.IsClustered() { @@ -281,5 +277,11 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { Str("parseD", parseT.Round(time.Millisecond).String()). Str("batchSize", my.batchSize). Msg("Collected") - return []*matrix.Matrix{my.data}, nil + + // This would generate quota metrics prefix with `qtree_`. These are deprecated now and will be removed later. + qtreePluginData := my.data.Clone(true, true, true) + qtreePluginData.UUID = my.Parent + ".Qtree" + qtreePluginData.Object = "qtree" + qtreePluginData.Identifier = "qtree" + return []*matrix.Matrix{qtreePluginData, my.data}, nil } diff --git a/grafana/dashboards/cmode/qtree.json b/grafana/dashboards/cmode/qtree.json index 4fd379b92..d655fcb09 100644 --- a/grafana/dashboards/cmode/qtree.json +++ b/grafana/dashboards/cmode/qtree.json @@ -532,7 +532,7 @@ "targets": [ { "exemplar": true, - "expr": "topk($TopResources, qtree_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeDiskUsed\"})", + "expr": "topk($TopResources, quota_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeDiskUsed\"})", "interval": "", "legendFormat": "{{svm}} - {{volume}} - {{qtree}}", "refId": "A" @@ -618,7 +618,7 @@ "targets": [ { "exemplar": true, - "expr": "topk($TopResources, qtree_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeFileUsed\"})", + "expr": "topk($TopResources, quota_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeFileUsed\"})", "interval": "", "legendFormat": "{{svm}} - {{volume}} - {{qtree}}", "refId": "A" @@ -961,7 +961,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qtree_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(quota_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -971,7 +971,7 @@ "name": "TopQtreeDiskUsed", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qtree_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(quota_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -984,7 +984,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qtree_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(quota_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -994,7 +994,7 @@ "name": "TopQtreeFileUsed", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qtree_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(quota_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -1024,6 +1024,6 @@ }, "timezone": "", "title": "ONTAP: Qtree", - "uid": "", + "uid": "X_svTxn4k", "version": 5 -} +} \ No newline at end of file diff --git a/grafana/dashboards/cmode/quotaReport.json b/grafana/dashboards/cmode/quotaReport.json index b9747d2e5..26baaa475 100644 --- a/grafana/dashboards/cmode/quotaReport.json +++ b/grafana/dashboards/cmode/quotaReport.json @@ -425,7 +425,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -436,7 +436,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_disk_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_disk_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -447,7 +447,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -458,7 +458,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_files_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_files_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -469,7 +469,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_soft_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_soft_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -480,7 +480,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_disk_used_pct_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_disk_used_pct_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -491,7 +491,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_files_used_pct_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_files_used_pct_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -502,7 +502,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "qtree_soft_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", + "expr": "quota_soft_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}", "format": "table", "hide": false, "instant": true, @@ -691,7 +691,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "definition": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", "description": null, "error": null, "hide": 0, @@ -702,7 +702,7 @@ "name": "SVM", "options": [], "query": { - "query": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "query": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -716,7 +716,7 @@ "allValue": "(.*)", "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)", + "definition": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)", "description": null, "error": null, "hide": 0, @@ -727,7 +727,7 @@ "name": "Qtree", "options": [], "query": { - "query": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)", + "query": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)", "refId": "StandardVariableQuery" }, "refresh": 1, diff --git a/integration/test/data/counter_data.go b/integration/test/data/counter_data.go index 81436259a..477e080f7 100644 --- a/integration/test/data/counter_data.go +++ b/integration/test/data/counter_data.go @@ -50,8 +50,8 @@ func GetCounterMap() map[string][]string { "svm_ldap", "svm_vscan", "security_login", - "qtree_disk_used_pct_disk_limit", - "qtree_files_used_pct_file_limit", + "quota_files_used_pct_file_limit", + "quota_disk_used_pct_disk_limit", "nfs_clients_idle_duration", ) return counterMap