feat: add antivirus counters vscan (#718)

* feat: add antivirus counters vscan

Fixes: #346

* feat: add antivirus counters vscan

- disabled by default
Fixes: #346

* feat: disable vscan by default

* style: review comments

cmd/collectors/zapi/collector/zapi.go

@@ -14,6 +14,7 @@ import (
 	"goharvest2/cmd/collectors/zapiperf/plugins/headroom"
 	"goharvest2/cmd/collectors/zapiperf/plugins/nic"
 	"goharvest2/cmd/collectors/zapiperf/plugins/volume"
+	"goharvest2/cmd/collectors/zapiperf/plugins/vscan"
 	"goharvest2/cmd/poller/plugin"
 	"goharvest2/pkg/conf"
 	"sort"
@@ -148,6 +149,8 @@ func (me *Zapi) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugi
 		return quota.New(abc)
 	case "Snapshot":
 		return snapshot.New(abc)
+	case "Vscan":
+		return vscan.New(abc)
 	default:
 		me.Logger.Info().Msgf("no zapi plugin found for %s", kind)
 	}

cmd/collectors/zapiperf/plugins/vscan/vscan.go

@@ -0,0 +1,155 @@
+package vscan
+
+import (
+	"goharvest2/cmd/poller/plugin"
+	"goharvest2/pkg/matrix"
+	"strconv"
+	"strings"
+)
+
+type Vscan struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &Vscan{AbstractPlugin: p}
+}
+
+func (v *Vscan) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
+	// defaults plugin options
+	isPerScanner := true
+
+	if s := v.Params.GetChildContentS("metricsPerScanner"); s != "" {
+		if parseBool, err := strconv.ParseBool(s); err == nil {
+			isPerScanner = parseBool
+		} else {
+			v.Logger.Error().Err(err).Msg("Failed to parse metricsPerScanner")
+		}
+	}
+	v.Logger.Debug().Bool("isPerScanner", isPerScanner).Msg("Vscan options")
+
+	v.addSvmAndScannerLabels(data)
+	if !isPerScanner {
+		return nil, nil
+	}
+
+	return v.aggregatePerScanner(data)
+}
+
+func (v *Vscan) addSvmAndScannerLabels(data *matrix.Matrix) {
+	for _, instance := range data.GetInstances() {
+		ontapName := instance.GetLabel("instance_uuid")
+		// colon separated list of fields
+		// vs_test4   :    2.2.2.2   :    umeng-aff300-05
+		//  svm       :     scanner  :    node
+		if split := strings.Split(ontapName, ":"); len(split) >= 3 {
+			instance.SetLabel("svm", split[0])
+			instance.SetLabel("scanner", split[1])
+			instance.SetLabel("node", split[2])
+		} else {
+			v.Logger.Warn().Str("ontapName", ontapName).Msg("Failed to parse svm and scanner labels")
+		}
+	}
+}
+
+func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, error) {
+	// When isPerScanner=true, Harvest 1.6 uses this form:
+	// netapp.perf.dev.nltl-fas2520.vscan.scanner.10_64_30_62.scanner_stats_pct_mem_used 18 1501765640
+
+	// These counters are per scanner and need averaging:
+	// 		scanner_stats_pct_cpu_used
+	// 		scanner_stats_pct_mem_used
+	// 		scanner_stats_pct_network_used
+	// These counters need to be summed:
+	// 		scan_request_dispatched_rate
+
+	// create per scanner instance cache
+	cache := data.Clone(false, true, false)
+	cache.UUID += ".Vscan"
+
+	for _, i := range data.GetInstances() {
+		scanner := i.GetLabel("scanner")
+		if cache.GetInstance(scanner) == nil {
+			s, _ := cache.NewInstance(scanner)
+			s.SetLabel("scanner", scanner)
+		}
+		i.SetExportable(false)
+	}
+
+	// aggregate per scanner
+	counts := make(map[string]map[string]int) // map[scanner][counter] => value
+
+	for _, i := range data.GetInstances() {
+		scanner := i.GetLabel("scanner")
+		ps := cache.GetInstance(scanner)
+		if ps == nil {
+			v.Logger.Error().Str("scanner", scanner).Msg("Failed to find scanner instance in cache")
+			continue
+		}
+		_, ok := counts[scanner]
+		if !ok {
+			counts[scanner] = make(map[string]int)
+		}
+		for mKey, m := range data.GetMetrics() {
+			if !m.IsExportable() && m.GetType() != "float64" {
+				continue
+			}
+			psm := cache.GetMetric(mKey)
+			if psm == nil {
+				v.Logger.Error().Str("scanner", scanner).Str("metric", mKey).
+					Msg("Failed to find metric in scanner cache")
+				continue
+			}
+			v.Logger.Trace().Str("scanner", scanner).Str("metric", mKey).Msg("Handling scanner metric")
+			if value, ok := m.GetValueFloat64(i); ok {
+				fv, _ := psm.GetValueFloat64(ps)
+
+				// sum for scan_request_dispatched_rate
+				if mKey == "scan_request_dispatched_rate" {
+					err := psm.SetValueFloat64(ps, fv+value)
+					if err != nil {
+						v.Logger.Error().Err(err).Str("metric", "scan_request_dispatched_rate").
+							Msg("Error setting metric value")
+					}
+					// for tracing
+					fgv2, _ := psm.GetValueFloat64(ps)
+					v.Logger.Trace().Float64("fv", fv).
+						Float64("value", value).
+						Float64("fgv2", fgv2).
+						Msg("> simple increment fv + value = fgv2")
+					continue
+				} else if strings.HasSuffix(mKey, "_used") {
+					// these need averaging
+					counts[scanner][mKey]++
+					runningTotal, _ := psm.GetValueFloat64(ps)
+					value, _ := m.GetValueFloat64(ps)
+					err := psm.SetValueFloat64(ps, runningTotal+value)
+					if err != nil {
+						v.Logger.Error().Err(err).Str("mKey", mKey).Msg("Failed to set value")
+					}
+				}
+			}
+		}
+	}
+
+	// cook averaged values
+	for scanner, i := range cache.GetInstances() {
+		for mKey, m := range cache.GetMetrics() {
+			if m.IsExportable() && strings.HasSuffix(m.GetName(), "_used") {
+				count := counts[scanner][mKey]
+				value, ok := m.GetValueFloat64(i)
+				if !ok {
+					continue
+				}
+				if err := m.SetValueFloat64(i, value/float64(count)); err != nil {
+					v.Logger.Error().Err(err).
+						Str("mKey", mKey).
+						Str("name", m.GetName()).
+						Msg("Unable to set average")
+				}
+			}
+		}
+	}
+
+	return []*matrix.Matrix{cache}, nil
+}

cmd/collectors/zapiperf/zapiperf.go

@@ -532,8 +532,8 @@ func (me *ZapiPerf) PollData() (*matrix.Matrix, error) {
 		}
 	}
 
-	// calculate timestamp delta first since many counters require it for postprocessing
-	// timestamp has "raw" property, so won't be postprocessed automatically
+	// calculate timestamp delta first since many counters require it for postprocessing.
+	// Timestamp has "raw" property, so it isn't post-processed automatically
 	if err = timestamp.Delta(me.Matrix.GetMetric("timestamp")); err != nil {
 		me.Logger.Error().Stack().Err(err).Msg("(timestamp) calculate delta:")
 		// @TODO terminate since other counters will be incorrect

cmd/exporters/prometheus/prometheus.go

@@ -416,6 +416,10 @@ func (me *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 			}
 		}
 	}
-	me.Logger.Debug().Msgf("rendered %d data points from %d (%s) instances", len(rendered), len(data.GetInstances()), data.Object)
+	me.Logger.Debug().
+		Str("object", data.Object).
+		Int("rendered", len(rendered)).
+		Int("instances", len(data.GetInstances())).
+		Msg("Rendered data points for instances")
 	return rendered, nil
 }

conf/zapiperf/cdot/9.8.0/vscan.yaml

@@ -0,0 +1,27 @@
+# Offbox vscan counters from a per cluster perspective
+name: Vscan
+query: offbox_vscan_server
+object: vscan
+
+instance_key: uuid
+
+counters:
+  - instance_name
+  - instance_uuid
+  - scanner_stats_pct_cpu_used
+  - scanner_stats_pct_mem_used
+  - scanner_stats_pct_network_used
+  - scan_latency
+  - scan_request_dispatched_rate
+
+plugins:
+  - Vscan:
+      # when metricsPerScanner is true, the counters are aggregated per scanner
+      # otherwise, they're not
+      metricsPerScanner: true
+
+export_options:
+  instance_keys:
+    - svm
+    - scanner
+    - node

conf/zapiperf/cdot/9.8.0/vscan_svm.yaml

@@ -0,0 +1,19 @@
+# Offbox vscan counters from a per SVM perspective
+name:                     VscanSVM
+query:                    offbox_vscan
+object:                   svm_vscan
+
+instance_key:             uuid
+
+counters:
+  - instance_name                  => svm
+  - instance_uuid
+  - dispatch_latency
+  - scan_latency
+  - scan_noti_received_rate
+  - scan_request_dispatched_rate
+  - connections_active
+
+export_options:
+  instance_keys:
+    - svm

conf/zapiperf/default.yaml

@@ -46,6 +46,8 @@ objects:
   ISCSI:                  iscsi_lif.yaml
   FcpLif:                 fcp_lif.yaml
   CopyManager:            copy_manager.yaml
+#  VscanSVM:               vscan_svm.yaml
+#  Vscan:                  vscan.yaml
   WAFLCompBin:            wafl_comp_aggr_vol_bin.yaml
 
 #  Uncomment to collect workload/QOS counters