feat: Harvest should collect QOS policy groups

cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go

@@ -0,0 +1,52 @@
+package qospolicyfixed
+
+import (
+	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed"
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+	"strings"
+)
+
+type QosPolicyFixed struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &QosPolicyFixed{AbstractPlugin: p}
+}
+
+func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+	data := dataMap[p.Object]
+
+	for _, instance := range data.GetInstances() {
+		p.setFixed(instance)
+	}
+
+	return nil, nil
+}
+
+func (p *QosPolicyFixed) setFixed(instance *matrix.Instance) {
+	label := instance.GetLabel("throughput_policy")
+	if label == "" {
+		return
+	}
+	before, after, found := strings.Cut(label, "-")
+	if !found {
+		p.Logger.Warn().Str("label", label).Msg("Unable to parse fixed xput label")
+		return
+	}
+	min, err := qospolicyfixed.ZapiXputToRest(before)
+	if err != nil {
+		p.Logger.Error().Err(err).Str("label", before).Msg("Failed to parse fixed xput label")
+		return
+	}
+	max, err := qospolicyfixed.ZapiXputToRest(after)
+	if err != nil {
+		p.Logger.Error().Err(err).Str("label", after).Msg("Failed to parse fixed xput label")
+		return
+	}
+	instance.SetLabel("min_throughput_iops", min.IOPS)
+	instance.SetLabel("max_throughput_iops", max.IOPS)
+	instance.SetLabel("min_throughput_mbps", min.Mbps)
+	instance.SetLabel("max_throughput_mbps", max.Mbps)
+}

cmd/collectors/rest/rest.go

@@ -5,6 +5,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/certificate"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/disk"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/netroute"
+	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qospolicyfixed"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qtree"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/securityaccount"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/sensor"
@@ -384,6 +385,8 @@ func (r *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin
 		return sensor.New(abc)
 	case "SecurityAccount":
 		return securityaccount.New(abc)
+	case "QosPolicyFixed":
+		return qospolicyfixed.New(abc)
 	default:
 		r.Logger.Warn().Str("kind", kind).Msg("no rest plugin found ")
 	}

cmd/collectors/zapi/collector/zapi.go

@@ -7,6 +7,7 @@ package zapi
 import (
 	"fmt"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/certificate"
+	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qtree"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/security"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/sensor"
@@ -149,6 +150,9 @@ func (z *Zapi) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin
 		return svm.New(abc)
 	case "Security":
 		return security.New(abc)
+	case "QosPolicyFixed":
+		return qospolicyfixed.New(abc)
+
 	default:
 		z.Logger.Info().Msgf("no zapi plugin found for %s", kind)
 	}

cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go

@@ -0,0 +1,111 @@
+package qospolicyfixed
+
+import (
+	"errors"
+	"fmt"
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type QosPolicyFixed struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &QosPolicyFixed{AbstractPlugin: p}
+}
+
+func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+	// Change ZAPI max-throughput/min-throughput to match what REST publishes
+
+	data := dataMap[p.Object]
+	for _, instance := range data.GetInstances() {
+		policyClass := instance.GetLabel("class")
+		if policyClass != "user_defined" {
+			// Only export user_defined policy classes - ignore all others
+			// REST only returns user_defined while ZAPI returns all
+			instance.SetExportable(false)
+			continue
+		}
+		p.setThroughput(instance, "max_xput", "max_throughput_iops", "max_throughput_mbps")
+		p.setThroughput(instance, "min_xput", "min_throughput_iops", "min_throughput_mbps")
+	}
+
+	return nil, nil
+}
+
+func (p *QosPolicyFixed) setThroughput(instance *matrix.Instance, labelName string, iopLabel string, mbpsLabel string) {
+	val := instance.GetLabel(labelName)
+	xput, err := ZapiXputToRest(val)
+	if err != nil {
+		p.Logger.Warn().Str(labelName, val).Msg("Unable to convert label, skipping")
+		return
+	}
+	instance.SetLabel(iopLabel, xput.IOPS)
+	instance.SetLabel(mbpsLabel, xput.Mbps)
+}
+
+var iopsRe = regexp.MustCompile(`(\d+)iops`)
+var bpsRe = regexp.MustCompile(`(\d+(\.\d+)?)(\w+)/s`)
+
+var unitToMb = map[string]float32{
+	"b":  1 / float32(1024*1024),
+	"kb": 1 / float32(1024),
+	"mb": 1,
+	"gb": 1024,
+	"tb": 1024 * 1024,
+}
+
+type MaxXput struct {
+	IOPS string
+	Mbps string
+}
+
+func ZapiXputToRest(zapi string) (MaxXput, error) {
+	lower := strings.ToLower(zapi)
+	empty := MaxXput{IOPS: "0", Mbps: "0"}
+	if lower == "inf" || lower == "0" {
+		return empty, nil
+	}
+
+	// check for a combination
+	before, after, found := strings.Cut(lower, ",")
+	if found {
+		l, err1 := ZapiXputToRest(before)
+		r, err2 := ZapiXputToRest(after)
+		if err1 != nil || err2 != nil {
+			return empty, errors.Join(err1, err2)
+		}
+		return MaxXput{
+			IOPS: l.IOPS,
+			Mbps: r.Mbps,
+		}, nil
+	}
+
+	// check for iops
+	matches := iopsRe.FindStringSubmatch(lower)
+	if len(matches) == 2 {
+		return MaxXput{IOPS: matches[1], Mbps: "0"}, nil
+	}
+
+	// check for bps and normalize units to Mbps
+	matches = bpsRe.FindStringSubmatch(lower)
+	if len(matches) != 4 {
+		return empty, fmt.Errorf("unknown qos-policy format [%s]", zapi)
+	}
+	numStr := matches[1]
+	unit := matches[3]
+	multiple, ok := unitToMb[unit]
+	if !ok {
+		return empty, fmt.Errorf("unknown qos-policy unit [%s] of [%s]", unit, zapi)
+	}
+	num, err := strconv.ParseFloat(numStr, 32)
+	if err != nil {
+		return empty, fmt.Errorf("failed to convert qos-policy unit [%s] of [%s]", numStr, zapi)
+	}
+	mbps := float32(num) * multiple
+	return MaxXput{Mbps: strconv.Itoa(int(mbps)), IOPS: "0"}, nil
+}

cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed_test.go

@@ -0,0 +1,43 @@
+package qospolicyfixed
+
+import (
+	"strconv"
+	"testing"
+)
+
+func Test_zapiXputToRest(t *testing.T) {
+	tests := []struct {
+		zapi  string
+		want  MaxXput
+		isErr bool
+	}{
+		{zapi: "100IOPS", want: MaxXput{IOPS: "100", Mbps: "0"}},
+		{zapi: "100iops", want: MaxXput{IOPS: "100", Mbps: "0"}},
+		{zapi: "111111IOPS", want: MaxXput{IOPS: "111111", Mbps: "0"}},
+		{zapi: "0", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "INF", want: MaxXput{IOPS: "0", Mbps: "0"}},
+
+		{zapi: "1GB/s", want: MaxXput{IOPS: "0", Mbps: "1024"}},
+		{zapi: "100B/s", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "10KB/s", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "1mb/s", want: MaxXput{IOPS: "0", Mbps: "1"}},
+		{zapi: "1tb/s", want: MaxXput{IOPS: "0", Mbps: "1048576"}},
+		{zapi: "1000KB/s", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "15000IOPS,468.8MB/s", want: MaxXput{IOPS: "15000", Mbps: "468"}},
+		{zapi: "50000IOPS,1.53GB/s", want: MaxXput{IOPS: "50000", Mbps: "1566"}},
+
+		{zapi: "1 foople/s", want: MaxXput{IOPS: "0", Mbps: "0"}, isErr: true},
+	}
+	for i, tt := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			got, err := ZapiXputToRest(tt.zapi)
+			if tt.isErr && err == nil {
+				t.Errorf("ZapiXputToRest(%s) got=%+v, want err but got err=%s", tt.zapi, got, err)
+				return
+			}
+			if got.IOPS != tt.want.IOPS || got.Mbps != tt.want.Mbps {
+				t.Errorf("ZapiXputToRest(%s) got=%+v, want=%+v", tt.zapi, got, tt.want)
+			}
+		})
+	}
+}

conf/rest/9.12.0/qos_policy_fixed.yaml

@@ -0,0 +1,31 @@
+
+name:               QosPolicyFixed
+query:              api/private/cli/qos/policy-group
+object:             qos_policy_fixed
+
+counters:
+  - ^^uuid                => uuid
+  - ^policy_group         => name
+  - ^vserver              => svm
+  - ^class                => class
+  - ^num_workloads        => object_count
+  - ^is_shared            => capacity_shared
+  - ^throughput_policy    => throughput_policy
+  - filter:
+      - class=user_defined
+
+plugins:
+  - QosPolicyFixed
+
+export_options:
+  instance_keys:
+    - svm
+  instance_labels:
+    - name
+    - capacity_shared
+    - max_throughput_iops
+    - max_throughput_mbps
+    - min_throughput_iops
+    - min_throughput_mbps
+    - object_count
+    - class

conf/rest/default.yaml

@@ -20,6 +20,7 @@ objects:
   NtpServer:                   ntpserver.yaml
   OntapS3:                     ontap_s3.yaml
   Port:                        port.yaml
+  QosPolicyFixed:              qos_policy_fixed.yaml
   Qtree:                       qtree.yaml
   Security:                    security.yaml
   SecurityAccount:             security_account.yaml

conf/zapi/cdot/9.8.0/qos_policy_fixed.yaml

@@ -0,0 +1,33 @@
+
+name:             QosPolicyFixed
+query:            qos-policy-group-get-iter
+object:           qos_policy_fixed
+
+counters:
+  qos-policy-group-info:
+    - ^^uuid               => uuid
+    - ^policy-group        => name
+    - ^vserver             => svm
+    - ^is-shared           => capacity_shared
+    - ^max-throughput      => max_xput
+    - ^min-throughput      => min_xput
+    - ^num-workloads       => object_count
+    - ^policy-group-class  => class
+
+collect_only_labels: true
+
+plugins:
+  - QosPolicyFixed
+
+export_options:
+  instance_keys:
+    - svm
+  instance_labels:
+    - name
+    - capacity_shared
+    - max_throughput_iops
+    - max_throughput_mbps
+    - min_throughput_iops
+    - min_throughput_mbps
+    - object_count
+    - class

conf/zapi/default.yaml

@@ -14,6 +14,7 @@ objects:
 #  NetPort:                     netPort.yaml
   Node:                        node.yaml
   NtpServer:                   ntpserver.yaml
+  QosPolicy:                   qos_policy_fixed.yaml
   Qtree:                       qtree.yaml
   Security:                    security.yaml
   SecurityAccount:             security_account.yaml