Merge branch 'main' into data_pro

Makefile

@@ -124,7 +124,7 @@ dist-tar:
 	@rm -rf ${DIST}
 	@mkdir ${TMP}
 	@mkdir ${DIST}
-	@cp -r .git cmd bin conf docker docs grafana pkg service autosupport go.mod go.sum Makefile README.md LICENSE prom-stack.yml harvest.cue ${TMP}
+	@cp -r .git cmd bin conf docker docs grafana pkg service autosupport go.mod go.sum Makefile README.md LICENSE prom-stack.tmpl harvest.cue ${TMP}
 	@cp harvest.yml ${TMP}/harvest.yml
 	@tar --directory /tmp --create --gzip --file ${DIST}/${HARVEST_PACKAGE}.tar.gz ${HARVEST_PACKAGE}
 	@rm -rf ${TMP}

cmd/collectors/zapi/plugins/shelf/shelf.go

@@ -81,6 +81,7 @@ func (my *Shelf) Init() error {
 		instanceLabels := exportOptions.NewChildS("instance_labels", "")
 		instanceKeys := exportOptions.NewChildS("instance_keys", "")
 		instanceKeys.NewChildS("", "shelf")
+		instanceKeys.NewChildS("", "channel")
 
 		// artificial metric for status of child object of shelf
 		my.data[attribute].NewMetricUint8("status")
@@ -126,9 +127,8 @@ func (my *Shelf) Init() error {
 func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 
 	var (
-		result  *node.Node
-		shelves []*node.Node
-		err     error
+		result *node.Node
+		err    error
 	)
 
 	if !my.client.IsClustered() {
@@ -146,13 +146,21 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 		my.data[a].SetGlobalLabels(data.GetGlobalLabels())
 	}
 
+	if my.client.IsClustered() {
+		return my.handleCMode(result)
+	} else {
+		return my.handle7Mode(result)
+	}
+}
+
+func (my *Shelf) handleCMode(result *node.Node) ([]*matrix.Matrix, error) {
+	var (
+		shelves []*node.Node
+	)
+
 	if x := result.GetChildS("attributes-list"); x != nil {
 		shelves = x.GetChildren()
-	} else if !my.client.IsClustered() {
-		//fallback to 7mode
-		shelves = result.SearchChildren([]string{"shelf-environ-channel-info", "shelf-environ-shelf-list", "shelf-environ-shelf-info"})
 	}
-
 	if len(shelves) == 0 {
 		return nil, errors.New(errors.ERR_NO_INSTANCE, "no shelf instances found")
 	}
@@ -201,7 +209,7 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 						instance, err := data1.NewInstance(instanceKey)
 
 						if err != nil {
-							my.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err)
+							my.Logger.Error().Err(err).Str("attribute", attribute).Msg("Failed to add instance")
 							return nil, err
 						}
 						my.Logger.Debug().Msgf("add (%s) instance: %s.%s", attribute, shelfId, key)
@@ -223,6 +231,17 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 							statusMetric.SetValueInt(instance, 0)
 						}
 
+						for metricKey, m := range data1.GetMetrics() {
+
+							if value := strings.Split(obj.GetChildContentS(metricKey), " ")[0]; value != "" {
+								if err := m.SetValueString(instance, value); err != nil {
+									my.Logger.Debug().Msgf("(%s) failed to parse value (%s): %v", metricKey, value, err)
+								} else {
+									my.Logger.Debug().Msgf("(%s) added value (%s)", metricKey, value)
+								}
+							}
+						}
+
 					} else {
 						my.Logger.Debug().Msgf("instance without [%s], skipping", my.instanceKeys[attribute])
 					}
@@ -233,50 +252,111 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 		}
 	}
 
-	// second loop to populate numeric data
+	return output, nil
+}
 
-	for _, shelf := range shelves {
+func (my *Shelf) handle7Mode(result *node.Node) ([]*matrix.Matrix, error) {
+	var (
+		shelves  []*node.Node
+		channels []*node.Node
+	)
+	//fallback to 7mode
+	channels = result.SearchChildren([]string{"shelf-environ-channel-info"})
 
-		shelfId := shelf.GetChildContentS("shelf-uid")
-		if !my.client.IsClustered() {
-			shelfId = shelf.GetChildContentS("shelf-id")
+	if len(channels) == 0 {
+		return nil, errors.New(errors.ERR_NO_INSTANCE, "no channels found")
+	}
+
+	var output []*matrix.Matrix
+
+	// Purge and reset data
+	for _, data1 := range my.data {
+		data1.PurgeInstances()
+		data1.Reset()
+	}
+
+	for _, channel := range channels {
+		channelName := channel.GetChildContentS("channel-name")
+		shelves = channel.SearchChildren([]string{"shelf-environ-shelf-list", "shelf-environ-shelf-info"})
+
+		if len(shelves) == 0 {
+			my.Logger.Warn().Str("channel", channelName).Msg("no shelves found")
+			continue
 		}
 
-		for attribute, data1 := range my.data {
+		for _, shelf := range shelves {
 
-			objectElem := shelf.GetChildS(attribute)
-			if objectElem == nil {
-				continue
-			}
+			uid := shelf.GetChildContentS("shelf-id")
+			shelfName := uid // no shelf name in 7mode
+			shelfId := uid
 
-			for _, obj := range objectElem.GetChildren() {
+			for attribute, data1 := range my.data {
+				if statusMetric := data1.GetMetric("status"); statusMetric != nil {
 
-				key := obj.GetChildContentS(my.instanceKeys[attribute])
+					if my.instanceKeys[attribute] == "" {
+						my.Logger.Warn().Str("attribute", attribute).Msg("no instance keys defined")
+						continue
+					}
 
-				if key == "" {
-					continue
-				}
+					objectElem := shelf.GetChildS(attribute)
+					if objectElem == nil {
+						my.Logger.Warn().Str("attribute", attribute).Msg("no instances on this system")
+						continue
+					}
 
-				instance := data1.GetInstance(shelfId + "." + key)
+					my.Logger.Debug().Msgf("fetching %d [%s] instances", len(objectElem.GetChildren()), attribute)
 
-				if instance == nil {
-					my.Logger.Debug().Msgf("(%s) instance [%s.%s] not found in cache skipping", attribute, shelfId, key)
-					continue
-				}
+					for _, obj := range objectElem.GetChildren() {
+
+						if key := obj.GetChildContentS(my.instanceKeys[attribute]); key != "" {
+							instanceKey := shelfId + "." + key + "." + channelName
+							instance, err := data1.NewInstance(instanceKey)
 
-				for metricKey, m := range data1.GetMetrics() {
+							if err != nil {
+								my.Logger.Error().Msgf("add (%s) instance: %v", attribute, err)
+								return nil, err
+							}
+							my.Logger.Debug().Msgf("add (%s) instance: %s.%s", attribute, shelfId, key)
+
+							for label, labelDisplay := range my.instanceLabels[attribute].Map() {
+								if value := obj.GetChildContentS(label); value != "" {
+									instance.SetLabel(labelDisplay, value)
+								}
+							}
+
+							instance.SetLabel("shelf", shelfName)
+							instance.SetLabel("shelf_id", shelfId)
+							instance.SetLabel("channel", channelName)
+
+							// Each child would have different possible values which is ugly way to write all of them,
+							// so normal value would be mapped to 1 and rest all are mapped to 0.
+							if instance.GetLabel("status") == "normal" {
+								statusMetric.SetValueInt(instance, 1)
+							} else {
+								statusMetric.SetValueInt(instance, 0)
+							}
+
+							// populate numeric data
+							for metricKey, m := range data1.GetMetrics() {
+
+								if value := strings.Split(obj.GetChildContentS(metricKey), " ")[0]; value != "" {
+									if err := m.SetValueString(instance, value); err != nil {
+										my.Logger.Debug().Msgf("(%s) failed to parse value (%s): %v", metricKey, value, err)
+									} else {
+										my.Logger.Debug().Msgf("(%s) added value (%s)", metricKey, value)
+									}
+								}
+							}
 
-					if value := strings.Split(obj.GetChildContentS(metricKey), " ")[0]; value != "" {
-						if err := m.SetValueString(instance, value); err != nil {
-							my.Logger.Debug().Msgf("(%s) failed to parse value (%s): %v", metricKey, value, err)
 						} else {
-							my.Logger.Debug().Msgf("(%s) added value (%s)", metricKey, value)
+							my.Logger.Debug().Msgf("instance without [%s], skipping", my.instanceKeys[attribute])
 						}
 					}
+
+					output = append(output, data1)
 				}
 			}
 		}
 	}
-
 	return output, nil
 }

cmd/collectors/zapiperf/plugins/nic/nic.go

@@ -78,6 +78,7 @@ func (me *Nic) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 			if err != nil {
 				me.Logger.Warn().Msgf("convert speed [%s]", s)
 			} else {
+				// NIC speed value converted from Mbps to Bps(bytes per second)
 				speed = base * 125000
 				instance.SetLabel("speed", strconv.Itoa(speed))
 				me.Logger.Debug().Msgf("converted speed (%s) to numeric (%d)", s, speed)

cmd/collectors/zapiperf/plugins/volume/volume.go

@@ -32,7 +32,8 @@ func (me *Volume) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 	// create flexgroup instance cache
 	for _, i := range data.GetInstances() {
 		if match := re.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
-			key := i.GetLabel("node") + "." + i.GetLabel("svm") + "." + match[1]
+			// instance key is svm.flexgroup-volume
+			key := i.GetLabel("svm") + "." + match[1]
 			if cache.GetInstance(key) == nil {
 				fg, _ := cache.NewInstance(key)
 				fg.SetLabels(i.GetLabels().Copy())
@@ -53,7 +54,8 @@ func (me *Volume) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 	// create summary
 	for _, i := range data.GetInstances() {
 		if match := re.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
-			key := i.GetLabel("node") + "." + i.GetLabel("svm") + "." + match[1]
+			// instance key is svm.flexgroup-volume
+			key := i.GetLabel("svm") + "." + match[1]
 			fg := cache.GetInstance(key)
 			if fg == nil {
 				me.Logger.Error().Stack().Err(nil).Msgf("instance [%s] not in local cache", key)

cmd/collectors/zapiperf/zapiperf.go

@@ -1191,7 +1191,7 @@ func parseHistogramLabels(elem *node.Node) ([]string, string) {
 	return labels, msg
 }
 
-// Update instance cache
+// PollInstance updates instance cache
 func (me *ZapiPerf) PollInstance() (*matrix.Matrix, error) {
 
 	var (
@@ -1254,7 +1254,10 @@ func (me *ZapiPerf) PollInstance() (*matrix.Matrix, error) {
 	for {
 
 		if results, batchTag, err = me.Client.InvokeBatchRequest(request, batchTag); err != nil {
-			me.Logger.Error().Stack().Err(err).Msg("instance request")
+			me.Logger.Error().Err(err).
+				Str("request", request.GetNameS()).
+				Str("batchTag", batchTag).
+				Msg("InvokeBatchRequest failed")
 			break
 		}
 

cmd/exporters/influxdb/influxdb.go

@@ -11,6 +11,7 @@ import (
 	"goharvest2/pkg/color"
 	"goharvest2/pkg/errors"
 	"goharvest2/pkg/matrix"
+	"io"
 	"io/ioutil"
 	"net/http"
 	url2 "net/url"
@@ -166,9 +167,12 @@ func (e *InfluxDB) Export(data *matrix.Matrix) error {
 				e.Logger.Debug().Msgf("M= [%s%s%s]", color.Blue, m, color.End)
 			}
 			return nil
-			// otherwise to the actual export: send to the DB
+			// otherwise, to the actual export: send to the DB
 		} else if err = e.Emit(metrics); err != nil {
-			e.Logger.Error().Stack().Err(err).Msgf("(%s.%s) --> %s", data.Object, data.UUID)
+			e.Logger.Error().Stack().Err(err).
+				Str("object", data.Object).
+				Str("uuid", data.UUID).
+				Msg("Failed to emit metrics")
 			return err
 		}
 	}
@@ -208,7 +212,7 @@ func (e *InfluxDB) Emit(data [][]byte) error {
 	}
 
 	if response.StatusCode != expectedResponseCode {
-		defer response.Body.Close()
+		defer func(Body io.ReadCloser) { _ = Body.Close() }(response.Body)
 		if body, err := ioutil.ReadAll(response.Body); err != nil {
 			return errors.New(errors.API_RESPONSE, err.Error())
 		} else {

cmd/poller/collector/helpers.go

@@ -29,12 +29,12 @@ import (
 )
 
 // ImportTemplate retrieves the config (template) of a collector, arguments are:
-// @confDir			- path of Harvest config durectory (usually /etc/harvest)
+// @confDir			- path of Harvest config directory (usually /etc/harvest)
 // @confFn			- template filename (e.g. default.yaml or custom.yaml)
 // @collectorName	- name of the collector
 func ImportTemplate(confPath, confFn, collectorName string) (*node.Node, error) {
 	fp := path.Join(confPath, "conf/", strings.ToLower(collectorName), confFn)
-	return tree.Import("yaml", fp)
+	return tree.ImportYaml(fp)
 }
 
 // ImportSubTemplate retrieves the best matching subtemplate of a collector object.
@@ -105,7 +105,7 @@ func (c *AbstractCollector) ImportSubTemplate(model, filename string, ver [3]int
 			verS, err := version.NewVersion(verWithDots)
 			if err != nil {
 				c.Logger.Trace().Msgf("error parsing ONTAP version: %s err: %s", verWithDots, err)
-				return nil, errors.New("No best-fitting subtemplate version found")
+				return nil, errors.New("no best-fitting subtemplate version found")
 			}
 			// get closest index
 			idx := getClosestIndex(versions, verS)
@@ -115,18 +115,18 @@ func (c *AbstractCollector) ImportSubTemplate(model, filename string, ver [3]int
 		}
 
 		if selectedVersion == "" {
-			return nil, errors.New("No best-fit template found")
+			return nil, errors.New("no best-fit template found")
 		}
 
 		subTemplateFp = path.Join(pathPrefix, selectedVersion, f)
 		c.Logger.Info().Msgf("best-fit template [%s] for [%s]", subTemplateFp, verWithDots)
 		if finalTemplate == nil {
-			finalTemplate, err = tree.Import("yaml", subTemplateFp)
+			finalTemplate, err = tree.ImportYaml(subTemplateFp)
 			if err == nil {
 				finalTemplate.PreprocessTemplate()
 			}
 		} else {
-			tempTemplate, err = tree.Import("yaml", subTemplateFp)
+			tempTemplate, err = tree.ImportYaml(subTemplateFp)
 			if err == nil {
 				tempTemplate.PreprocessTemplate()
 				// merge templates