From 1ff779f95e7eb05f455569338208cd916b5a0542 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 6 Dec 2021 06:37:49 -0500 Subject: [PATCH] feat: add antivirus counters vscan (#718) * feat: add antivirus counters vscan Fixes: #346 * feat: add antivirus counters vscan - disabled by default Fixes: #346 * feat: disable vscan by default * style: review comments --- cmd/collectors/zapi/collector/zapi.go | 3 + .../zapiperf/plugins/vscan/vscan.go | 155 ++++++++++++++++++ cmd/collectors/zapiperf/zapiperf.go | 4 +- cmd/exporters/prometheus/prometheus.go | 6 +- conf/zapiperf/cdot/9.8.0/vscan.yaml | 27 +++ conf/zapiperf/cdot/9.8.0/vscan_svm.yaml | 19 +++ conf/zapiperf/default.yaml | 2 + 7 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 cmd/collectors/zapiperf/plugins/vscan/vscan.go create mode 100644 conf/zapiperf/cdot/9.8.0/vscan.yaml create mode 100644 conf/zapiperf/cdot/9.8.0/vscan_svm.yaml diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go index ff7529fc5..aceba3dc2 100644 --- a/cmd/collectors/zapi/collector/zapi.go +++ b/cmd/collectors/zapi/collector/zapi.go @@ -14,6 +14,7 @@ import ( "goharvest2/cmd/collectors/zapiperf/plugins/headroom" "goharvest2/cmd/collectors/zapiperf/plugins/nic" "goharvest2/cmd/collectors/zapiperf/plugins/volume" + "goharvest2/cmd/collectors/zapiperf/plugins/vscan" "goharvest2/cmd/poller/plugin" "goharvest2/pkg/conf" "sort" @@ -148,6 +149,8 @@ func (me *Zapi) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugi return quota.New(abc) case "Snapshot": return snapshot.New(abc) + case "Vscan": + return vscan.New(abc) default: me.Logger.Info().Msgf("no zapi plugin found for %s", kind) } diff --git a/cmd/collectors/zapiperf/plugins/vscan/vscan.go b/cmd/collectors/zapiperf/plugins/vscan/vscan.go new file mode 100644 index 000000000..68ea0b589 --- /dev/null +++ b/cmd/collectors/zapiperf/plugins/vscan/vscan.go @@ -0,0 +1,155 @@ +package vscan + +import ( + "goharvest2/cmd/poller/plugin" + "goharvest2/pkg/matrix" + "strconv" + "strings" +) + +type Vscan struct { + *plugin.AbstractPlugin +} + +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &Vscan{AbstractPlugin: p} +} + +func (v *Vscan) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { + // defaults plugin options + isPerScanner := true + + if s := v.Params.GetChildContentS("metricsPerScanner"); s != "" { + if parseBool, err := strconv.ParseBool(s); err == nil { + isPerScanner = parseBool + } else { + v.Logger.Error().Err(err).Msg("Failed to parse metricsPerScanner") + } + } + v.Logger.Debug().Bool("isPerScanner", isPerScanner).Msg("Vscan options") + + v.addSvmAndScannerLabels(data) + if !isPerScanner { + return nil, nil + } + + return v.aggregatePerScanner(data) +} + +func (v *Vscan) addSvmAndScannerLabels(data *matrix.Matrix) { + for _, instance := range data.GetInstances() { + ontapName := instance.GetLabel("instance_uuid") + // colon separated list of fields + // vs_test4 : 2.2.2.2 : umeng-aff300-05 + // svm : scanner : node + if split := strings.Split(ontapName, ":"); len(split) >= 3 { + instance.SetLabel("svm", split[0]) + instance.SetLabel("scanner", split[1]) + instance.SetLabel("node", split[2]) + } else { + v.Logger.Warn().Str("ontapName", ontapName).Msg("Failed to parse svm and scanner labels") + } + } +} + +func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, error) { + // When isPerScanner=true, Harvest 1.6 uses this form: + // netapp.perf.dev.nltl-fas2520.vscan.scanner.10_64_30_62.scanner_stats_pct_mem_used 18 1501765640 + + // These counters are per scanner and need averaging: + // scanner_stats_pct_cpu_used + // scanner_stats_pct_mem_used + // scanner_stats_pct_network_used + // These counters need to be summed: + // scan_request_dispatched_rate + + // create per scanner instance cache + cache := data.Clone(false, true, false) + cache.UUID += ".Vscan" + + for _, i := range data.GetInstances() { + scanner := i.GetLabel("scanner") + if cache.GetInstance(scanner) == nil { + s, _ := cache.NewInstance(scanner) + s.SetLabel("scanner", scanner) + } + i.SetExportable(false) + } + + // aggregate per scanner + counts := make(map[string]map[string]int) // map[scanner][counter] => value + + for _, i := range data.GetInstances() { + scanner := i.GetLabel("scanner") + ps := cache.GetInstance(scanner) + if ps == nil { + v.Logger.Error().Str("scanner", scanner).Msg("Failed to find scanner instance in cache") + continue + } + _, ok := counts[scanner] + if !ok { + counts[scanner] = make(map[string]int) + } + for mKey, m := range data.GetMetrics() { + if !m.IsExportable() && m.GetType() != "float64" { + continue + } + psm := cache.GetMetric(mKey) + if psm == nil { + v.Logger.Error().Str("scanner", scanner).Str("metric", mKey). + Msg("Failed to find metric in scanner cache") + continue + } + v.Logger.Trace().Str("scanner", scanner).Str("metric", mKey).Msg("Handling scanner metric") + if value, ok := m.GetValueFloat64(i); ok { + fv, _ := psm.GetValueFloat64(ps) + + // sum for scan_request_dispatched_rate + if mKey == "scan_request_dispatched_rate" { + err := psm.SetValueFloat64(ps, fv+value) + if err != nil { + v.Logger.Error().Err(err).Str("metric", "scan_request_dispatched_rate"). + Msg("Error setting metric value") + } + // for tracing + fgv2, _ := psm.GetValueFloat64(ps) + v.Logger.Trace().Float64("fv", fv). + Float64("value", value). + Float64("fgv2", fgv2). + Msg("> simple increment fv + value = fgv2") + continue + } else if strings.HasSuffix(mKey, "_used") { + // these need averaging + counts[scanner][mKey]++ + runningTotal, _ := psm.GetValueFloat64(ps) + value, _ := m.GetValueFloat64(ps) + err := psm.SetValueFloat64(ps, runningTotal+value) + if err != nil { + v.Logger.Error().Err(err).Str("mKey", mKey).Msg("Failed to set value") + } + } + } + } + } + + // cook averaged values + for scanner, i := range cache.GetInstances() { + for mKey, m := range cache.GetMetrics() { + if m.IsExportable() && strings.HasSuffix(m.GetName(), "_used") { + count := counts[scanner][mKey] + value, ok := m.GetValueFloat64(i) + if !ok { + continue + } + if err := m.SetValueFloat64(i, value/float64(count)); err != nil { + v.Logger.Error().Err(err). + Str("mKey", mKey). + Str("name", m.GetName()). + Msg("Unable to set average") + } + } + } + } + + return []*matrix.Matrix{cache}, nil +} diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go index db7fdd7ed..e1ee04bd8 100644 --- a/cmd/collectors/zapiperf/zapiperf.go +++ b/cmd/collectors/zapiperf/zapiperf.go @@ -532,8 +532,8 @@ func (me *ZapiPerf) PollData() (*matrix.Matrix, error) { } } - // calculate timestamp delta first since many counters require it for postprocessing - // timestamp has "raw" property, so won't be postprocessed automatically + // calculate timestamp delta first since many counters require it for postprocessing. + // Timestamp has "raw" property, so it isn't post-processed automatically if err = timestamp.Delta(me.Matrix.GetMetric("timestamp")); err != nil { me.Logger.Error().Stack().Err(err).Msg("(timestamp) calculate delta:") // @TODO terminate since other counters will be incorrect diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go index 37ba52278..68280b90f 100644 --- a/cmd/exporters/prometheus/prometheus.go +++ b/cmd/exporters/prometheus/prometheus.go @@ -416,6 +416,10 @@ func (me *Prometheus) render(data *matrix.Matrix) ([][]byte, error) { } } } - me.Logger.Debug().Msgf("rendered %d data points from %d (%s) instances", len(rendered), len(data.GetInstances()), data.Object) + me.Logger.Debug(). + Str("object", data.Object). + Int("rendered", len(rendered)). + Int("instances", len(data.GetInstances())). + Msg("Rendered data points for instances") return rendered, nil } diff --git a/conf/zapiperf/cdot/9.8.0/vscan.yaml b/conf/zapiperf/cdot/9.8.0/vscan.yaml new file mode 100644 index 000000000..e0d769cc6 --- /dev/null +++ b/conf/zapiperf/cdot/9.8.0/vscan.yaml @@ -0,0 +1,27 @@ +# Offbox vscan counters from a per cluster perspective +name: Vscan +query: offbox_vscan_server +object: vscan + +instance_key: uuid + +counters: + - instance_name + - instance_uuid + - scanner_stats_pct_cpu_used + - scanner_stats_pct_mem_used + - scanner_stats_pct_network_used + - scan_latency + - scan_request_dispatched_rate + +plugins: + - Vscan: + # when metricsPerScanner is true, the counters are aggregated per scanner + # otherwise, they're not + metricsPerScanner: true + +export_options: + instance_keys: + - svm + - scanner + - node diff --git a/conf/zapiperf/cdot/9.8.0/vscan_svm.yaml b/conf/zapiperf/cdot/9.8.0/vscan_svm.yaml new file mode 100644 index 000000000..511cc962d --- /dev/null +++ b/conf/zapiperf/cdot/9.8.0/vscan_svm.yaml @@ -0,0 +1,19 @@ +# Offbox vscan counters from a per SVM perspective +name: VscanSVM +query: offbox_vscan +object: svm_vscan + +instance_key: uuid + +counters: + - instance_name => svm + - instance_uuid + - dispatch_latency + - scan_latency + - scan_noti_received_rate + - scan_request_dispatched_rate + - connections_active + +export_options: + instance_keys: + - svm diff --git a/conf/zapiperf/default.yaml b/conf/zapiperf/default.yaml index 979d49169..80de7c291 100644 --- a/conf/zapiperf/default.yaml +++ b/conf/zapiperf/default.yaml @@ -46,6 +46,8 @@ objects: ISCSI: iscsi_lif.yaml FcpLif: fcp_lif.yaml CopyManager: copy_manager.yaml +# VscanSVM: vscan_svm.yaml +# Vscan: vscan.yaml WAFLCompBin: wafl_comp_aggr_vol_bin.yaml # Uncomment to collect workload/QOS counters