From 8b9b93f2a118e35a37608791c4b9c45b5cc4bb76 Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Thu, 2 Dec 2021 19:17:03 +0530 Subject: [PATCH] feat: data protection dashboard - part1 - snapshots (#664) * feat: data protection dashboard - part1 - snapshots --- cmd/collectors/zapi/collector/zapi.go | 3 + .../zapi/plugins/snapshot/snapshot.go | 199 +++ conf/zapi/cdot/9.8.0/volume.yaml | 9 + .../harvest_dashboard_data_protection.json | 1590 +++++++++++++++++ integration/test/dashboard_test.go | 2 +- 5 files changed, 1802 insertions(+), 1 deletion(-) create mode 100644 cmd/collectors/zapi/plugins/snapshot/snapshot.go create mode 100644 grafana/dashboards/cmode/harvest_dashboard_data_protection.json diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go index 91185ae80..ff7529fc5 100644 --- a/cmd/collectors/zapi/collector/zapi.go +++ b/cmd/collectors/zapi/collector/zapi.go @@ -9,6 +9,7 @@ import ( "goharvest2/cmd/collectors/zapi/plugins/quota" "goharvest2/cmd/collectors/zapi/plugins/shelf" "goharvest2/cmd/collectors/zapi/plugins/snapmirror" + "goharvest2/cmd/collectors/zapi/plugins/snapshot" "goharvest2/cmd/collectors/zapiperf/plugins/fcp" "goharvest2/cmd/collectors/zapiperf/plugins/headroom" "goharvest2/cmd/collectors/zapiperf/plugins/nic" @@ -145,6 +146,8 @@ func (me *Zapi) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugi return volume.New(abc) case "Qtree": return quota.New(abc) + case "Snapshot": + return snapshot.New(abc) default: me.Logger.Info().Msgf("no zapi plugin found for %s", kind) } diff --git a/cmd/collectors/zapi/plugins/snapshot/snapshot.go b/cmd/collectors/zapi/plugins/snapshot/snapshot.go new file mode 100644 index 000000000..12288e9e7 --- /dev/null +++ b/cmd/collectors/zapi/plugins/snapshot/snapshot.go @@ -0,0 +1,199 @@ +package snapshot + +import ( + "goharvest2/cmd/poller/collector" + "goharvest2/cmd/poller/plugin" + "goharvest2/pkg/api/ontapi/zapi" + "goharvest2/pkg/conf" + "goharvest2/pkg/dict" + "goharvest2/pkg/errors" + "goharvest2/pkg/matrix" + "goharvest2/pkg/tree/node" + "strconv" + "strings" +) + +const BatchSize = "500" + +type Snapshot struct { + *plugin.AbstractPlugin + data *matrix.Matrix + instanceKeys map[string]string + instanceLabels map[string]*dict.Dict + batchSize string + client *zapi.Client + query string +} + +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &Snapshot{AbstractPlugin: p} +} + +func (my *Snapshot) Init() error { + + var err error + + if err = my.InitAbc(); err != nil { + return err + } + + if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil { + my.Logger.Error().Stack().Err(err).Msg("connecting") + return err + } + + if err = my.client.Init(5); err != nil { + return err + } + + my.query = "snapshot-policy-get-iter" + my.Logger.Debug().Msg("plugin connected!") + + my.data = matrix.New(my.Parent+".Snapshot", "snapshot", "snapshot") + my.instanceKeys = make(map[string]string) + my.instanceLabels = make(map[string]*dict.Dict) + + exportOptions := node.NewS("export_options") + instanceKeys := exportOptions.NewChildS("instance_keys", "") + + instanceKeys.NewChildS("", "vserver") + instanceKeys.NewChildS("", "snapshot_policy") + instanceKeys.NewChildS("", "comment") + + objects := my.Params.GetChildS("objects") + if objects == nil { + return errors.New(errors.MISSING_PARAM, "objects") + } + + for _, obj := range objects.GetAllChildContentS() { + metricName, display := collector.ParseMetricName(obj) + + metric, err := my.data.NewMetricFloat64(metricName) + if err != nil { + my.Logger.Error().Stack().Err(err).Msg("add metric") + return err + } + + metric.SetName(display) + my.Logger.Debug().Msgf("added metric: (%s) [%s] %s", metricName, display, metric) + } + + my.Logger.Debug().Msgf("added data with %d metrics", len(my.data.GetMetrics())) + my.data.SetExportOptions(exportOptions) + + // batching the request + if my.client.IsClustered() { + if b := my.Params.GetChildContentS("batch_size"); b != "" { + if _, err := strconv.Atoi(b); err == nil { + my.batchSize = b + my.Logger.Info().Msgf("using batch-size [%s]", my.batchSize) + } + } else { + my.batchSize = BatchSize + my.Logger.Trace().Str("BatchSize", BatchSize).Msg("Using default batch-size") + } + } + + return nil +} + +func (my *Snapshot) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { + + var ( + request, result *node.Node + policyInfos []*node.Node + tag string + err error + ) + + var output []*matrix.Matrix + + // Purge and reset data + my.data.PurgeInstances() + my.data.Reset() + + // Set all global labels from zapi.go if already not exist + my.data.SetGlobalLabels(data.GetGlobalLabels()) + + request = node.NewXmlS(my.query) + if my.client.IsClustered() && my.batchSize != "" { + request.NewChildS("max-records", my.batchSize) + } + + tag = "initial" + + for { + result, tag, err = my.client.InvokeBatchRequest(request, tag) + + if err != nil { + return nil, err + } + + if result == nil { + break + } + + if x := result.GetChildS("attributes-list"); x != nil { + policyInfos = x.GetChildren() + } + + if len(policyInfos) == 0 { + return nil, errors.New(errors.ERR_NO_INSTANCE, "no snapshot policy info instances found") + } + + my.Logger.Debug().Msgf("fetching %d snapshot policy counters", len(policyInfos)) + + for policyInfoIndex, policyInfo := range policyInfos { + + var comment string + vserver := policyInfo.GetChildContentS("vserver-name") + policyName := policyInfo.GetChildContentS("policy") + if comment = policyInfo.GetChildContentS("comment"); comment == "" { + comment = "No Comment" + } + + for attribute, m := range my.data.GetMetrics() { + + objectElem := policyInfo.GetChildS(attribute) + if objectElem == nil { + my.Logger.Warn().Msgf("no [%s] instances on this %s.%s.%s", attribute, vserver, policyName, comment) + continue + } + + if attrValue := policyInfo.GetChildContentS(attribute); attrValue != "" { + // Ex. InstanceKey: fas8040-206-21.default.Default policy with hourly, daily.12.total-schedules + instanceKey := vserver + "." + policyName + "." + comment + "." + strconv.Itoa(policyInfoIndex) + "." + attribute + //my.Logger.Info().Msgf(instanceKey) + instance, err := my.data.NewInstance(instanceKey) + + if err != nil { + my.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err) + return nil, err + } + + my.Logger.Debug().Msgf("add (%s) instance: %s.%s.%s", attribute, vserver, policyName, comment) + + instance.SetLabel("vserver", vserver) + instance.SetLabel("snapshot_policy", policyName) + instance.SetLabel("comment", comment) + + // populate numeric data + if value := strings.Split(attrValue, " ")[0]; value != "" { + if err := m.SetValueString(instance, value); err != nil { + my.Logger.Debug().Msgf("(%s) failed to parse value (%s): %v", attribute, value, err) + } else { + my.Logger.Debug().Msgf("(%s) added value (%s)", attribute, value) + } + } + + } else { + my.Logger.Debug().Msgf("instance without [%s], skipping", attribute) + } + + output = append(output, my.data) + } + } + + } + return output, nil +} diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index fffca1bf3..eaee9e810 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -58,7 +58,15 @@ counters: - ^state - ^status + - volume-snapshot-attributes: + - ^auto-snapshots-enabled => auto_snapshots_enabled + - ^snapshot-policy + - snapshot-count + plugins: + Snapshot: + objects: + - total-schedules LabelAgent: # metric label zapi_value rest_value `default_value` value_to_num: @@ -80,4 +88,5 @@ export_options: instance_labels: - state - is_sis_volume + - snapshot_policy diff --git a/grafana/dashboards/cmode/harvest_dashboard_data_protection.json b/grafana/dashboards/cmode/harvest_dashboard_data_protection.json new file mode 100644 index 000000000..ed54f190a --- /dev/null +++ b/grafana/dashboards/cmode/harvest_dashboard_data_protection.json @@ -0,0 +1,1590 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.5.4" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1637051731590, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 45, + "panels": [], + "title": "Snapshot Copies Overview", + "type": "row" + }, + { + "datasource": "Prometheus", + "description": "Volumes Protected With Snapshot Copies (local)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Volumes protected" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Volumes not protected" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "hidden", + "placement": "right", + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{})) OR on() vector(0)", + "instant": false, + "interval": "", + "legendFormat": "Volumes protected", + "refId": "A" + }, + { + "exemplar": true, + "expr": "count((volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy=~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}))) OR on() vector(0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Volumes not protected", + "refId": "B" + } + ], + "title": "Protected Status", + "transformations": [], + "type": "piechart" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 75, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{})) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volumes protected", + "type": "stat" + }, + { + "datasource": "Prometheus", + "description": "Volumes Breaching Snapshot Copy Reserve Space", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Volumes breached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Volumes not breached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "hidden", + "placement": "right", + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} > volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)", + "instant": false, + "interval": "", + "legendFormat": "Volumes breached", + "refId": "A" + }, + { + "exemplar": true, + "expr": "(count((volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{})) * on (volume, svm, cluster) group_right() (volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} <= volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)) + (count(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} == 0)) OR on() vector(0))", + "instant": false, + "interval": "", + "legendFormat": "Volumes not breached", + "refId": "B" + } + ], + "title": "Breached Status", + "transformations": [], + "type": "piechart" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-red", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 77, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} > volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volumes breached", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "color": "rgb(21, 118, 171)", + "text": "0" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-yellow", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 79, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count((volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy=~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}))) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volumes not protected", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "color": "rgb(21, 118, 171)", + "text": "0" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 81, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "(count((volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{})) * on (volume, svm, cluster) group_right() (volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} > 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} > 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} <= volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)) + (count(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} == 0)) OR on() vector(0))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volumes not breached", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "displayMode": "color-background-solid", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "custom.width", + "value": 350 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "snapshot_policy" + }, + "properties": [ + { + "id": "displayName", + "value": "Snapshot Policy" + }, + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "custom.width", + "value": 250 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 83, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": false, + "expr": "label_replace(label_replace(volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", volume!~\"MDV.*\"}, \"Status\", \"Protected\", \"snapshot_policy\", \"(.*)\") , \"Status\", \"Unprotected\", \"snapshot_policy\", \"(none.*)\") * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volumes Protected With Snapshot Copies (local)", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "snapshot_policy", + "volume", + "Status", + "svm" + ] + } + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "Status": 3, + "snapshot_policy": 2, + "svm": 1, + "volume": 0 + } + } + } + ], + "type": "table" + }, + { + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "displayMode": "auto", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "custom.width", + "value": 320 + }, + { + "id": "displayName", + "value": "Volume" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Used" + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Reserved" + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "from": -1e+25, + "result": { + "index": 0, + "text": "Not Breached" + }, + "to": 0 + }, + "type": "range" + }, + { + "options": { + "from": 0, + "result": { + "index": 1, + "text": "Breached" + }, + "to": 1e+25 + }, + "type": "range" + } + ] + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 91, + "interval": "1m", + "maxDataPoints": 2, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "volume_labels{datacenter=\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"} >= 0)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": true, + "expr": "volume_snapshots_size_used{datacenter=\"$Datacenter\", cluster=~\"$Cluster\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volumes Breaching Snapshot Copy Reserve Space", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greaterOrEqual", + "options": { + "value": 0 + } + }, + "fieldName": "Value #A" + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "volume", + "Value #A", + "Value #B", + "svm" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Status", + "binary": { + "left": "Value #B", + "operator": "-", + "reducer": "sum", + "right": "Value #A" + }, + "mode": "binary", + "reduce": { + "include": [ + "Value #A", + "Value #B" + ], + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Status": 4, + "Value #A": 2, + "Value #B": 3, + "svm": 1, + "volume": 0 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "title": "Snapshot Copies Analysis", + "type": "row" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 25 + }, + "id": 96, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count((volume_labels{datacenter=\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{} > 0 and volume_snapshot_count{} < 10)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": " <10 Copies ", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 25 + }, + "id": 97, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count((volume_labels{datacenter=\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{} >= 10 and volume_snapshot_count{} <= 100)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "10-100 Copies", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 25 + }, + "id": 98, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count((volume_labels{datacenter=\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{} > 100 and volume_snapshot_count{} <= 500)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "101-500 Copies", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 25 + }, + "id": 99, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "count((volume_labels{datacenter=\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{} > 500)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": ">500 Copies", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "left", + "displayMode": "color-background-solid", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Bucket" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "index": 0, + "text": "< 10 Copies" + }, + "to": 9 + }, + "type": "range" + }, + { + "options": { + "from": 10, + "result": { + "index": 1, + "text": "10-100 Copies" + }, + "to": 100 + }, + "type": "range" + }, + { + "options": { + "from": 101, + "result": { + "index": 2, + "text": "101-500 Copies" + }, + "to": 500 + }, + "type": "range" + }, + { + "options": { + "from": 501, + "result": { + "index": 3, + "text": "> 500 Copies" + }, + "to": 1e+24 + }, + "type": "range" + } + ] + }, + { + "id": "custom.width", + "value": 290 + }, + { + "id": "displayName", + "value": "Snapshot Copies Bucket" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "custom.width", + "value": 500 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "custom.width", + "value": 500 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Snapshot Copies" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "custom.width", + "value": 400 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 94, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": false, + "expr": "(volume_labels{datacenter=\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_total_schedules{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{} > 0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volume count by the number of Snapshot copies", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Value", + "cluster" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 3, + "cluster": 2, + "svm": 1, + "volume": 0 + }, + "renameByName": { + "Value": "Snapshot Copies", + "volume": "" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Bucket", + "mode": "reduceRow", + "reduce": { + "include": [ + "Snapshot Copies" + ], + "reducer": "last" + } + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "DC-01", + "value": "DC-01" + }, + "datasource": "Prometheus", + "definition": "label_values(volume_labels{system_type!=\"7mode\"},datacenter)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(volume_labels{system_type!=\"7mode\"},datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "fas8040-206-21", + "value": "fas8040-206-21" + }, + "datasource": "Prometheus", + "definition": "label_values(volume_labels{system_type!=\"7mode\",datacenter=\"$Datacenter\"},cluster)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(volume_labels{system_type!=\"7mode\",datacenter=\"$Datacenter\"},cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "NetApp Detail: Data Protection", + "uid": "3iYBEBcnk", + "version": 1 +} \ No newline at end of file diff --git a/integration/test/dashboard_test.go b/integration/test/dashboard_test.go index e4061c512..a34036c14 100644 --- a/integration/test/dashboard_test.go +++ b/integration/test/dashboard_test.go @@ -78,7 +78,7 @@ func (suite *DashboardImportTestSuite) TestCModeDashboardCount() { "NetApp Detail: Disk", "NetApp Detail: LUN", "NetApp Detail: Network - Details", "NetApp Detail: Network with NVMe/FC", "NetApp Detail: Node - Details", "NetApp Detail: Shelf", "NetApp Detail: SnapMirror", "NetApp Detail: SVM - Details", - "NetApp Detail: Volume - Details", "NetApp Detail: MetroCluster"} + "NetApp Detail: Volume - Details", "NetApp Detail: MetroCluster", "NetApp Detail: Data Protection"} VerifyDashboards(folderId, expectedName, suite.T()) }