feat: quota metric renaming (#1345)

* feat: quota metric renaming * feat: type change * feat: rest side changes + retain older qtree plugin metrics * feat: handle instances/metrics review comments * feat: handle review comments * feat: missed to add the file * feat: deprecation in changelog
NetApp · Oct 20, 2022 · 9db5393 · 9db5393
1 parent a2f5fb3
commit 9db5393
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,17 @@
 
 [Releases](https://github.com/NetApp/harvest/releases)
 
+## 22.11.0 / 2022-11-10
+<!-- git log --no-decorate --no-merges --cherry-pick --right-only --oneline origin/release/22.08.0...origin/release/22.11.0 -->
+
+:rocket: Highlights of this major release include:
+
+**Deprecation**: Earlier versions of Harvest published quota metrics prefixed with qtree. Harvest release 22.11 deprecates the quota metrics prefixed with qtree and instead publishes quota metrics prefixed with quota.
+All dashboards have been updated. If you are consuming these metrics outside the default dashboards, please update them to use the quota prefixed metrics. Harvest release 23.02 will remove the deprecated quota metrics
+prefixed with qtree.
+
+---
+
 ## 22.08.0 / 2022-08-19
 <!-- git log --no-decorate --no-merges --cherry-pick --right-only --oneline origin/release/22.05.0...origin/release/22.08.0 -->
 

diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go
@@ -12,7 +12,6 @@ import (
 	"github.com/netapp/harvest/v2/pkg/dict"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
-	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/tidwall/gjson"
 	"strconv"
@@ -75,13 +74,10 @@ func (my *Qtree) Init() error {
 
 	my.query = "api/storage/quota/reports"
 
-	my.data = matrix.New(my.Parent+".Qtree", "qtree", "qtree")
+	my.data = matrix.New(my.Parent+".Qtree", "quota", "quota")
 	my.instanceKeys = make(map[string]string)
 	my.instanceLabels = make(map[string]*dict.Dict)
 
-	exportOptions := node.NewS("export_options")
-	exportOptions.NewChildS("include_all_labels", "true")
-
 	quotaType := my.Params.GetChildS("quotaType")
 	if quotaType != nil {
 		my.quotaType = []string{}
@@ -105,7 +101,6 @@ func (my *Qtree) Init() error {
 	}
 
 	my.Logger.Debug().Msgf("added data with %d metrics", len(my.data.GetMetrics()))
-	my.data.SetExportOptions(exportOptions)
 
 	return nil
 }
@@ -210,5 +205,10 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 		Int("metrics", numMetrics).
 		Msg("Collected")
 
-	return []*matrix.Matrix{my.data}, nil
+	// This would generate quota metrics prefix with `qtree_`. These are deprecated now and will be removed later.
+	qtreePluginData := my.data.Clone(true, true, true)
+	qtreePluginData.UUID = my.Parent + ".Qtree"
+	qtreePluginData.Object = "qtree"
+	qtreePluginData.Identifier = "qtree"
+	return []*matrix.Matrix{qtreePluginData, my.data}, nil
 }
diff --git a/cmd/collectors/zapi/plugins/qtree/qtree.go b/cmd/collectors/zapi/plugins/qtree/qtree.go
@@ -58,13 +58,10 @@ func (my *Qtree) Init() error {
 	}
 	my.Logger.Debug().Msg("plugin connected!")
 
-	my.data = matrix.New(my.Parent+".Qtree", "qtree", "qtree")
+	my.data = matrix.New(my.Parent+".Qtree", "quota", "quota")
 	my.instanceKeys = make(map[string]string)
 	my.instanceLabels = make(map[string]*dict.Dict)
 
-	exportOptions := node.NewS("export_options")
-	exportOptions.NewChildS("include_all_labels", "true")
-
 	objects := my.Params.GetChildS("objects")
 	if objects == nil {
 		return errs.New(errs.ErrMissingParam, "objects")
@@ -93,7 +90,6 @@ func (my *Qtree) Init() error {
 	}
 
 	my.Logger.Debug().Msgf("added data with %d metrics", len(my.data.GetMetrics()))
-	my.data.SetExportOptions(exportOptions)
 
 	// setup batchSize for request
 	if my.client.IsClustered() {
@@ -281,5 +277,11 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 		Str("parseD", parseT.Round(time.Millisecond).String()).
 		Str("batchSize", my.batchSize).
 		Msg("Collected")
-	return []*matrix.Matrix{my.data}, nil
+
+	// This would generate quota metrics prefix with `qtree_`. These are deprecated now and will be removed later.
+	qtreePluginData := my.data.Clone(true, true, true)
+	qtreePluginData.UUID = my.Parent + ".Qtree"
+	qtreePluginData.Object = "qtree"
+	qtreePluginData.Identifier = "qtree"
+	return []*matrix.Matrix{qtreePluginData, my.data}, nil
 }
diff --git a/grafana/dashboards/cmode/qtree.json b/grafana/dashboards/cmode/qtree.json
@@ -532,7 +532,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "topk($TopResources, qtree_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeDiskUsed\"})",
+              "expr": "topk($TopResources, quota_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeDiskUsed\"})",
               "interval": "",
               "legendFormat": "{{svm}} - {{volume}} - {{qtree}}",
               "refId": "A"
@@ -618,7 +618,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "topk($TopResources, qtree_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeFileUsed\"})",
+              "expr": "topk($TopResources, quota_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$TopQtreeFileUsed\"})",
               "interval": "",
               "legendFormat": "{{svm}} - {{volume}} - {{qtree}}",
               "refId": "A"
@@ -961,7 +961,7 @@
         "allValue": null,
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "query_result(topk($TopResources, avg_over_time(qtree_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, avg_over_time(quota_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
         "description": null,
         "error": null,
         "hide": 2,
@@ -971,7 +971,7 @@
         "name": "TopQtreeDiskUsed",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(qtree_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, avg_over_time(quota_disk_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -984,7 +984,7 @@
         "allValue": null,
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "query_result(topk($TopResources, avg_over_time(qtree_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, avg_over_time(quota_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
         "description": null,
         "error": null,
         "hide": 2,
@@ -994,7 +994,7 @@
         "name": "TopQtreeFileUsed",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(qtree_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, avg_over_time(quota_files_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", qtree=~\"$Qtree\"}[${__range}])))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -1024,6 +1024,6 @@
   },
   "timezone": "",
   "title": "ONTAP: Qtree",
-  "uid": "",
+  "uid": "X_svTxn4k",
   "version": 5
-}
+}
diff --git a/grafana/dashboards/cmode/quotaReport.json b/grafana/dashboards/cmode/quotaReport.json
@@ -425,7 +425,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -436,7 +436,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_disk_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_disk_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -447,7 +447,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -458,7 +458,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_files_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_files_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -469,7 +469,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_soft_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_soft_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -480,7 +480,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_disk_used_pct_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_disk_used_pct_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -491,7 +491,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_files_used_pct_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_files_used_pct_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -502,7 +502,7 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "qtree_soft_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
+          "expr": "quota_soft_file_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",qtree=~\"$Qtree\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -691,7 +691,7 @@
         "allValue": null,
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)",
+        "definition": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)",
         "description": null,
         "error": null,
         "hide": 0,
@@ -702,7 +702,7 @@
         "name": "SVM",
         "options": [],
         "query": {
-          "query": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)",
+          "query": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -716,7 +716,7 @@
         "allValue": "(.*)",
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)",
+        "definition": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)",
         "description": null,
         "error": null,
         "hide": 0,
@@ -727,7 +727,7 @@
         "name": "Qtree",
         "options": [],
         "query": {
-          "query": "label_values(qtree_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)",
+          "query": "label_values(quota_disk_limit{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, qtree)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,

diff --git a/integration/test/data/counter_data.go b/integration/test/data/counter_data.go
@@ -50,8 +50,8 @@ func GetCounterMap() map[string][]string {
 		"svm_ldap",
 		"svm_vscan",
 		"security_login",
-		"qtree_disk_used_pct_disk_limit",
-		"qtree_files_used_pct_file_limit",
+		"quota_files_used_pct_file_limit",
+		"quota_disk_used_pct_disk_limit",
 		"nfs_clients_idle_duration",
 	)
 	return counterMap