fix: improve template yaml parsing

Fixes #704

cmd/poller/collector/helpers.go

@@ -29,7 +29,7 @@ import (
 )
 
 // ImportTemplate retrieves the config (template) of a collector, arguments are:
-// @confDir			- path of Harvest config durectory (usually /etc/harvest)
+// @confDir			- path of Harvest config directory (usually /etc/harvest)
 // @confFn			- template filename (e.g. default.yaml or custom.yaml)
 // @collectorName	- name of the collector
 func ImportTemplate(confPath, confFn, collectorName string) (*node.Node, error) {
@@ -121,12 +121,12 @@ func (c *AbstractCollector) ImportSubTemplate(model, filename string, ver [3]int
 		subTemplateFp = path.Join(pathPrefix, selectedVersion, f)
 		c.Logger.Info().Msgf("best-fit template [%s] for [%s]", subTemplateFp, verWithDots)
 		if finalTemplate == nil {
-			finalTemplate, err = tree.Import("yaml", subTemplateFp)
+			finalTemplate, err = tree.ImportYaml(subTemplateFp)
 			if err == nil {
 				finalTemplate.PreprocessTemplate()
 			}
 		} else {
-			tempTemplate, err = tree.Import("yaml", subTemplateFp)
+			tempTemplate, err = tree.ImportYaml(subTemplateFp)
 			if err == nil {
 				tempTemplate.PreprocessTemplate()
 				// merge templates

cmd/poller/plugin/README.md

@@ -33,7 +33,7 @@ plugins:
 # will aggregate a new Matrix based on target label LABEL
 ```
 
-If you want to speficy which labels should be included in the new instances, you can add those space-seperated after `LABEL`:
+If you want to specify which labels should be included in the new instances, you can add those space-seperated after `LABEL`:
 
 ```yaml
     - LABEL LABEL1,LABEL2
@@ -47,7 +47,7 @@ Or include all labels:
     - LABEL ...
 # copy all labels of the original instance
 ```
-By default aggregated metrics will be prefixed with `LABEL`. For example if the object of the original Matrix is `volume` (meaning metrics are prefixed with `volume_`) and `LABEL` is `aggr`, then the metric `volume_read_ops` will become `aggr_volume_read_ops`, etc. You can override this by providing the `<>OBJ` using the following syntax:
+By default, aggregated metrics will be prefixed with `LABEL`. For example if the object of the original Matrix is `volume` (meaning metrics are prefixed with `volume_`) and `LABEL` is `aggr`, then the metric `volume_read_ops` will become `aggr_volume_read_ops`, etc. You can override this by providing the `<>OBJ` using the following syntax:
 
 ```yaml
     - LABEL<>OBJ
@@ -71,14 +71,13 @@ Examples:
 ```yaml
 plugins:
   Aggregator:
+    # will aggregate metrics of the aggregate. The labels "node" and "type" are included in the new instances
     - aggr node type
-# will aggregate metrics of Anggregate level and make sure
-# labels "node" and "type" are included in the new instances
+    # aggregate instances if label "type" has value "flexgroup"
+    # include all original labels
     - type<flexgroup> ...
-# aggregate instances if label "type" has value "flexgroup"
-# include all original labels
+    # aggregate all instances if value of "volume" ends with underscore and 4 digits
     - volume<`_\d{4}$`>
-# aggregate all instances if value of "volume" ends with underscore and 4 digits
 ```
 
 ### Aggregation rules
@@ -94,7 +93,7 @@ The plugin tries to intelligently aggregate metrics based on a few rules:
 - **Ignore** - metrics created by some plugins, such as value_to_num by LabelAgent
 
 # LabelAgent
-LabelAgent allows to manipulate instance labels based on rules. You can define multiple rules, here is an example of what you could add to the yaml file of a collector:
+LabelAgent are used to manipulate instance labels based on rules. You can define multiple rules, here is an example of what you could add to the yaml file of a collector:
 
 
 ```yaml
@@ -130,23 +129,23 @@ Rule syntax:
 
 ```yaml
 split: LABEL `SEP` LABEL1,LABEL2,LABEL3
-# source label - seperator - comma-seperated target labels
+# source label - separator - comma-seperated target labels
 ```
 
-Splits the value of a given label by seperator `SEP` and creates new labels if their number matches to the number of target labels defined in rule. To discard a subvalue, just add a redundant `,` in the names of the target labels. 
+Splits the value of a given label by separator `SEP` and creates new labels if their number matches to the number of target labels defined in rule. To discard a subvalue, just add a redundant `,` in the names of the target labels. 
 
 Example:
 
 ```yaml
 split: node `/` ,aggr,plex,disk
-# will split the value of "node" using seperator "/"
+# will split the value of "node" using separator "/"
 # will expect 4 values: first will be discarded, remaining
 # three will be stored as labels "aggr", "plex" and "disk"
 ```
 
 ## split_regex
 
-Does the same as `split_regex` but uses a regular expression instead of a seperator.
+Does the same as `split_regex` but uses a regular expression instead of a separator.
 
 Rule syntax:
 
@@ -170,7 +169,7 @@ Rule syntax:
 
 ```yaml
 split_pairs: LABEL `SEP1` `SEP2`
-# source label - pair seperator - key-value seperator
+# source label - pair separator - key-value separator
 ```
 
 Extracts key-value pairs from the value of source label `LABEL`. Note that you need to add these keys in the export options, otherwise they will not be exported.
@@ -186,13 +185,13 @@ split_pairs: comment ` ` `:`
 
 ## join
 
-Join multiple label values using seperator `SEP` and create a new label. 
+Join multiple label values using separator `SEP` and create a new label. 
 
 Rule syntax:
 
 ```yaml
 join: LABEL `SEP` LABEL1,LABEL2,LABEL3
-# target label - seperator - comma-seperated source labels
+# target label - separator - comma-seperated source labels
 ```
 
 Example:
@@ -307,7 +306,7 @@ value_mapping was deprecated in 21.11 and removed in 22.02. Use [value_to_num ma
 
 ## value_to_num
 
-Maps values of a given label to a numeric metric (of type `uint8`).
+Map values of a given label to a numeric metric (of type `uint8`).
 This rule maps values of a given label to a numeric metric (of type `unit8`). Healthy is mapped to 1 and all non-healthy values are mapped to 0.
 
 This is handy to manipulate the data in the DB or Grafana (e.g. change color based on status or create alert).
@@ -318,8 +317,8 @@ Rule syntax:
 
 ```yaml
 value_to_num: METRIC LABEL ZAPI_VALUE REST_VALUE `N`
-# maps values of LABEL to 1 if it is ZAPI_VALUE or REST_VALUE
-# otherwise value of METRIC is set to N
+# map values of LABEL to 1 if it is ZAPI_VALUE or REST_VALUE
+# otherwise, value of METRIC is set to N
 ```
 The default value `N` is optional, if no default value is given and the label value does not match any of the given values, the metric value will not be set.
 
@@ -341,6 +340,6 @@ value_to_num: status state up online `4`
 ```yaml
 value_to_num: status outage - - `0` #ok_value is empty value. 
 # metric value will be set to 1 if "outage" is empty, if it's any other value, it will be set to the default, 0
-# '-' is a special symbol in this mapping and it will be converted to blank while processing.
+# '-' is a special symbol in this mapping, and it will be converted to blank while processing.
 ```
 

conf/rest/9.13.0/volume.yaml

@@ -57,7 +57,7 @@ plugins:
     replace:
       - style style `flexgroup_constituent` `flexgroup`
   Aggregator:
-    volume<style=flexgroup>volume node,svm,aggr,style
+    - volume<style=flexgroup>volume node,svm,aggr,style
 
 export_options:
   instance_keys:

conf/zapi/cdot/9.8.0/shelf.yaml

@@ -20,47 +20,47 @@ plugins:
   Shelf:
     objects:
       - cooling-fans => fan:
-        storage-shelf-cooling-fan-info:
-          - ^^fan-id
-          - ^fan-location => location
-          - ^fan-op-status => status
-          - fan-rpm => rpm
+          storage-shelf-cooling-fan-info:
+            - ^^fan-id
+            - ^fan-location => location
+            - ^fan-op-status => status
+            - fan-rpm => rpm
       - current-sensors => sensor:
-        storage-shelf-current-sensor-info:
-          - ^^current-sensor-id    => sensor_id
-          - ^current-sensor-location => location
-          - ^current-op-status => status
-          - current-sensor-reading => reading
+          storage-shelf-current-sensor-info:
+            - ^^current-sensor-id    => sensor_id
+            - ^current-sensor-location => location
+            - ^current-op-status => status
+            - current-sensor-reading => reading
       - power-supply-units => psu:
-        storage-shelf-power-supply-unit-info:
-#          - ^psu-crest-factor => crest_factor
-#          - ^psu-fw-version => fw_version
-          - ^^psu-id
-          - ^psu-is-enabled => enabled
-          - ^psu-location => location
-          - ^psu-part-number => part_number
-          - psu-power-drawn => power_drawn
-          - psu-power-rating => power_rating
-#          - ^psu-reset-capable => reset_capable
-          - ^psu-serial-number => serial
-          - ^psu-type => type
-          - ^psu-op-status => status
+          storage-shelf-power-supply-unit-info:
+  #          - ^psu-crest-factor => crest_factor
+  #          - ^psu-fw-version => fw_version
+            - ^^psu-id
+            - ^psu-is-enabled => enabled
+            - ^psu-location => location
+            - ^psu-part-number => part_number
+            - psu-power-drawn => power_drawn
+            - psu-power-rating => power_rating
+  #          - ^psu-reset-capable => reset_capable
+            - ^psu-serial-number => serial
+            - ^psu-type => type
+            - ^psu-op-status => status
       - temperature-sensors => temperature:
-        storage-shelf-temperature-sensor-info:
-          - ^high-critical-threshold => high_critical
-          - ^high-warning-threshold => high_warning
-          - ^temp-is-ambient
-          - ^temp-low-critical-threshold => low_critical
-          - ^temp-low-warning-threshold => low_warning
-          - ^^temp-sensor-id => sensor_id
-          - ^temp-sens-op-status => status
-          - temp-sensor-reading => reading
+          storage-shelf-temperature-sensor-info:
+            - ^high-critical-threshold => high_critical
+            - ^high-warning-threshold => high_warning
+            - ^temp-is-ambient
+            - ^temp-low-critical-threshold => low_critical
+            - ^temp-low-warning-threshold => low_warning
+            - ^^temp-sensor-id => sensor_id
+            - ^temp-sens-op-status => status
+            - temp-sensor-reading => reading
       - voltage-sensors => voltage:
-        storage-shelf-voltage-sensor-info:
-          - ^^voltage-sensor-id => sensor_id
-          - ^voltage-sensor-location => location
-          - ^voltage-op-status => status
-          - voltage-sensor-reading => reading
+          storage-shelf-voltage-sensor-info:
+            - ^^voltage-sensor-id => sensor_id
+            - ^voltage-sensor-location => location
+            - ^voltage-op-status => status
+            - voltage-sensor-reading => reading
 
   LabelAgent:
     # metric label zapi_value rest_value `default_value`

conf/zapi/cdot/9.8.0/volume.yaml

@@ -68,7 +68,7 @@ plugins:
     replace:
       - style style `flexgroup_constituent` `flexgroup`
   Aggregator:
-    volume<style=flexgroup>volume node,svm,aggr,style
+    - volume<style=flexgroup>volume node,svm,aggr,style
 
 export_options:
   instance_keys:

pkg/tree/tree.go

@@ -9,11 +9,50 @@ import (
 	"goharvest2/pkg/tree/node"
 	"goharvest2/pkg/tree/xml"
 	"goharvest2/pkg/tree/yaml"
+	y3 "gopkg.in/yaml.v3"
 	"io/ioutil"
 )
 
-func Print(n *node.Node) {
-	n.Print(0)
+func ImportYaml(filepath string) (*node.Node, error) {
+	data, err := ioutil.ReadFile(filepath)
+
+	if err != nil {
+		return nil, err
+	}
+
+	root := y3.Node{}
+	err = y3.Unmarshal(data, &root)
+	if err != nil || len(root.Content) == 0 {
+		return nil, err
+	}
+	r := node.New([]byte("Root"))
+	consume(r, "", root.Content[0])
+	return r, nil
+}
+
+func consume(r *node.Node, key string, y *y3.Node) {
+	if y.Kind == y3.ScalarNode {
+		r.NewChildS(key, y.Value)
+	} else if y.Kind == y3.MappingNode {
+		var s = r
+		if key != "" {
+			s = r.NewChildS(key, "")
+		}
+		for i := 0; i < len(y.Content); i += 2 {
+			k := y.Content[i].Value
+			// special case to handle incorrectly indented LabelAgent
+			if k == "LabelAgent" && y.Content[i+1].Kind == y3.ScalarNode {
+				s = r.NewChildS(k, "")
+				continue
+			}
+			consume(s, k, y.Content[i+1])
+		}
+	} else { // sequence
+		s := r.NewChildS(key, "")
+		for _, child := range y.Content {
+			consume(s, "", child)
+		}
+	}
 }
 
 func Import(format, filepath string) (*node.Node, error) {
@@ -36,36 +75,6 @@ func Import(format, filepath string) (*node.Node, error) {
 	return nil, errors.New("unknown format: " + format)
 }
 
-func Export(n *node.Node, format, filepath string) error {
-
-	var data []byte
-	var err error
-
-	switch format {
-	case "yaml":
-		data, err = yaml.Dump(n)
-	case "xml":
-		data, err = xml.Dump(n)
-	case "json":
-		data = json.Dump(n)
-	default:
-		err = errors.New("unknown format: " + format)
-	}
-
-	if err == nil {
-		err = ioutil.WriteFile(filepath, data, 0644)
-	}
-	return err
-}
-
-func LoadYaml(data []byte) (*node.Node, error) {
-	return yaml.Load(data)
-}
-
-func DumpYaml(n *node.Node) ([]byte, error) {
-	return yaml.Dump(n)
-}
-
 func LoadXml(data []byte) (*node.Node, error) {
 	return xml.Load(data)
 }