feat: Template merge functionality

cmd/poller/collector/helpers.go

@@ -122,9 +122,13 @@ func (c *AbstractCollector) ImportSubTemplate(model, filename string, ver [3]int
 		c.Logger.Info().Msgf("best-fit template [%s] for [%s]", subTemplateFp, verWithDots)
 		if finalTemplate == nil {
 			finalTemplate, err = tree.Import("yaml", subTemplateFp)
+			if err == nil {
+				finalTemplate.PreprocessTemplate()
+			}
 		} else {
 			tempTemplate, err = tree.Import("yaml", subTemplateFp)
 			if err == nil {
+				tempTemplate.PreprocessTemplate()
 				// merge templates
 				finalTemplate.Merge(tempTemplate, nil)
 			}

cmd/tools/doctor/doctor.go

@@ -16,8 +16,8 @@ import (
 type options struct {
 	ShouldPrintConfig bool
 	Color             string
-	TemplatePath1     string
-	TemplatePath2     string
+	BaseTemplate      string
+	MergeTemplate     string
 }
 
 var opts = &options{
@@ -40,29 +40,31 @@ var Cmd = &cobra.Command{
 var mergeCmd = &cobra.Command{
 	Use:    "merge",
 	Hidden: true,
-	Short:  "generate Harvest systemd target for all pollers defined in config",
+	Short:  "merge templates",
 	Run:    doMergeCmd,
 }
 
 func doMergeCmd(cmd *cobra.Command, _ []string) {
-	doMerge(opts.TemplatePath1, opts.TemplatePath2)
+	doMerge(opts.BaseTemplate, opts.MergeTemplate)
 }
 
 func doMerge(path1 string, path2 string) {
 	template, err := tree.Import("yaml", path1)
 	if err != nil {
-		fmt.Printf("error reading template file. err=%+v\n", err)
+		fmt.Printf("error reading template file [%s]. err=%+v\n", path1, err)
 		return
 	}
 	subTemplate, err := tree.Import("yaml", path2)
 	if err != nil {
-		fmt.Printf("error reading template file. err=%+v\n", err)
+		fmt.Printf("error reading template file [%s] err=%+v\n", path2, err)
 		return
 	}
+	template.PreprocessTemplate()
+	subTemplate.PreprocessTemplate()
 	template.Merge(subTemplate, nil)
 	data, err := harvestyaml.Dump(template)
 	if err != nil {
-		fmt.Printf("error reading parsing template file. err=%+v\n", err)
+		fmt.Printf("error reading parsing template file [%s]  err=%+v\n", data, err)
 		return
 	}
 	fmt.Println(string(data))
@@ -270,8 +272,8 @@ func init() {
 	Cmd.AddCommand(mergeCmd)
 	mFlags := mergeCmd.PersistentFlags()
 
-	mFlags.StringVarP(&opts.TemplatePath1, "template", "", "", "First file path ")
-	mFlags.StringVarP(&opts.TemplatePath2, "with", "", "", "Second file path. ")
+	mFlags.StringVarP(&opts.BaseTemplate, "template", "", "", "Base template path ")
+	mFlags.StringVarP(&opts.MergeTemplate, "with", "", "", "Extended file path. ")
 
 	_ = mergeCmd.MarkPersistentFlagRequired("template")
 	_ = mergeCmd.MarkPersistentFlagRequired("with")

pkg/test/mergeTemplates/21.08.0_lun_merge_21.08.0_extended.yaml

@@ -0,0 +1,38 @@
+name: Lun
+query: lun-get-iter
+object: lun
+counters:
+  lun-info:
+    - ^node
+    - ^path
+    - ^qtree
+    - size
+    - size-used
+    - ^state
+    - ^^uuid
+    - ^volume
+    - ^vserver => svm
+plugins:
+  LabelAgent:
+    value_mapping:
+      - status state online `1`
+      - custom_status state online `1`
+    value_to_num:
+      - new_status state online online `0`
+    split:
+      - path `/` ,,,lun
+    new_mapping:
+      - xyz
+  - nic
+  Aggregator:
+    - node<>node_cpu1
+    - nodetest
+export_options:
+  instance_keys:
+    - node
+    - qtree
+    - lun
+    - volume
+    - svm
+  instance_labels:
+    - state

pkg/test/mergeTemplates/21.08.0_lun_merge_extended.yaml

@@ -0,0 +1,41 @@
+name: customLun
+query: lun-get-iter
+object: lun
+counters:
+  lun-info:
+    - ^node
+    - ^path
+    - ^qtree
+    - size
+    - size-used
+    - ^state
+    - ^^uuid
+    - ^volume
+    - ^vserver => svm
+plugins:
+  LabelAgent:
+    value_mapping:
+      - status state online `1`
+      - custom_status state online `1`
+    value_to_num:
+      - new_status state online online `0`
+    split:
+      - path `/` ,,,lun
+    new_mapping:
+      - xyz
+  Aggregator:
+    - node<>node_cpu1
+export_options:
+  instance_keys:
+    - node
+    - qtree
+    - lun
+    - volume
+    - svm
+    - node1
+  instance_labels:
+    - state
+override:
+  KB_copied: delta
+  key2: test
+export_data: false

pkg/test/mergeTemplates/lun_merge.yaml

@@ -0,0 +1,42 @@
+name: customLun
+query: lun-get-iter
+object: lun
+counters:
+  lun-info:
+    - ^node
+    - ^path
+    - ^qtree
+    - size
+    - size-used
+    - ^state
+    - ^^uuid
+    - ^volume
+    - ^vserver => svm
+plugins:
+  LabelAgent:
+    value_mapping:
+      - status state online `1`
+      - custom_status state online `1`
+    value_to_num:
+      - new_status state online online `0`
+    split:
+      - path `/` ,,,lun
+    new_mapping:
+      - xyz
+  Aggregator:
+    - aggr
+    - node<>node_cpu1
+export_options:
+  instance_keys:
+    - node
+    - qtree
+    - lun
+    - volume
+    - svm
+    - node1
+  instance_labels:
+    - state
+override:
+  KB_copied: delta
+  key2: test
+export_data: false

pkg/test/mergeTemplates/lun_merge_21.08.0_extended.yaml

@@ -0,0 +1,39 @@
+name: Lun
+query: lun-get-iter
+object: lun
+counters:
+  lun-info:
+    - ^node
+    - ^path
+    - ^qtree
+    - size
+    - size-used
+    - ^state
+    - ^^uuid
+    - ^volume
+    - ^vserver => svm
+plugins:
+  LabelAgent:
+    value_mapping:
+      - status state online `1`
+      - custom_status state online `1`
+    value_to_num:
+      - new_status state online online `0`
+    split:
+      - path `/` ,,,lun
+    new_mapping:
+      - xyz
+  Aggregator:
+    - aggr
+    - node<>node_cpu1
+    - nodetest
+  - nic
+export_options:
+  instance_keys:
+    - node
+    - qtree
+    - lun
+    - volume
+    - svm
+  instance_labels:
+    - state

pkg/test/node_test.go

@@ -2,13 +2,16 @@ package test
 
 import (
 	"goharvest2/pkg/tree"
+	"goharvest2/pkg/tree/yaml"
+	"io/ioutil"
+	"strings"
 	"testing"
 )
 
 // Merge default.yaml and custom.yaml
 func TestNode_Merge(t *testing.T) {
-	defaultTemplate, _ := tree.Import("yaml", "testdata/default.yaml")
-	customTemplate, _ := tree.Import("yaml", "testdata/custom.yaml")
+	defaultTemplate, _ := tree.Import("yaml", "testdata/default_collector.yaml")
+	customTemplate, _ := tree.Import("yaml", "testdata/extend_collector.yaml")
 	defaultTemplate.Merge(customTemplate, []string{"objects"})
 
 	// count number of objects post merge
@@ -51,9 +54,20 @@ func TestNode_Merge(t *testing.T) {
 // change is LabelAgent child will have list of rules instead of key-value pair
 func TestNode_MergeCollector(t *testing.T) {
 	defaultTemplate, _ := tree.Import("yaml", "testdata/lun.yaml")
-	customTemplate, _ := tree.Import("yaml", "testdata/custom_lun.yaml")
+	customTemplate, _ := tree.Import("yaml", "testdata/extend_lun.yaml")
+	defaultTemplate.PreprocessTemplate()
+	customTemplate.PreprocessTemplate()
 	defaultTemplate.Merge(customTemplate, nil)
 
+	gotString1, _ := yaml.Dump(defaultTemplate)
+	gotString := strings.TrimRight(string(gotString1), "\r\n")
+	expected, _ := ioutil.ReadFile("mergeTemplates/lun_merge.yaml")
+	expectedString := strings.TrimRight(string(expected), "\r\n")
+
+	if gotString != expectedString {
+		t.Errorf("got %v, want %v", gotString, expectedString)
+	}
+
 	// object name overwrite
 	want := "customLun"
 	got := ""
@@ -207,7 +221,9 @@ func TestNode_MergeCollector(t *testing.T) {
 // LabelAgent child did have key-value pair of rules instead of a list
 func TestNode_MergeCollectorOld(t *testing.T) {
 	defaultTemplate, _ := tree.Import("yaml", "testdata/lun.yaml")
-	customTemplate, _ := tree.Import("yaml", "testdata/custom_lun_old.yaml")
+	customTemplate, _ := tree.Import("yaml", "testdata/21.08.0_extend_lun.yaml")
+	defaultTemplate.PreprocessTemplate()
+	customTemplate.PreprocessTemplate()
 	defaultTemplate.Merge(customTemplate, nil)
 
 	// plugins labelagent add new child to existing plugin
@@ -266,3 +282,64 @@ func TestNode_MergeCollectorOld(t *testing.T) {
 		t.Errorf("got %v, want %v", got5, want5)
 	}
 }
+
+func TestNode_PreProcessCollector(t *testing.T) {
+
+	tests := []struct {
+		name        string
+		sourceFile  string
+		compareFile string
+	}{
+		{name: "preprocess template from 21.08.0", sourceFile: "testdata/21.08.0_extend_lun.yaml", compareFile: "preProcessResultData/p_21.08.0_extend_lun.yaml"},
+		{name: "preprocess template after 21.08.0", sourceFile: "testdata/21.08.0_lun.yaml", compareFile: "preProcessResultData/p_21.08.0_lun.yaml"},
+		{name: "process collector template", sourceFile: "testdata/default_collector.yaml", compareFile: "preProcessResultData/p_default_collector.yaml"},
+		{name: "process extended collector template", sourceFile: "testdata/extend_collector.yaml", compareFile: "preProcessResultData/p_extend_collector.yaml"},
+		{name: "process extended object template", sourceFile: "testdata/extend_lun.yaml", compareFile: "preProcessResultData/p_extend_lun.yaml"},
+		{name: "process object template", sourceFile: "testdata/lun.yaml", compareFile: "preProcessResultData/p_lun.yaml"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			template, _ := tree.Import("yaml", tt.sourceFile)
+			template.PreprocessTemplate()
+			got, _ := yaml.Dump(template)
+			expected, _ := ioutil.ReadFile(tt.compareFile)
+			gotString := strings.TrimRight(string(got), "\r\n")
+			expectedString := strings.TrimRight(string(expected), "\r\n")
+			if gotString != expectedString {
+				t.Errorf("got %v, want %v", gotString, expectedString)
+			}
+		})
+	}
+}
+
+func TestNode_PreProcessMergeCollector(t *testing.T) {
+
+	tests := []struct {
+		name           string
+		baseTemplate   string
+		extendTemplate string
+		mergeTemplate  string
+	}{
+		{name: "Case1: Both base and extended template follow new convention for labelagent which is list", baseTemplate: "testdata/lun.yaml", extendTemplate: "testdata/extend_lun.yaml", mergeTemplate: "mergeTemplates/lun_merge.yaml"},
+		{name: "Case2: base template follow new convention for labelagent and extended template follow 21.08.0", baseTemplate: "testdata/lun.yaml", extendTemplate: "testdata/21.08.0_extend_lun.yaml", mergeTemplate: "mergeTemplates/lun_merge_21.08.0_extended.yaml"},
+		{name: "Case3: base template follow old convention for labelagent and extended template follow 21.08.0", baseTemplate: "testdata/21.08.0_lun.yaml", extendTemplate: "testdata/21.08.0_extend_lun.yaml", mergeTemplate: "mergeTemplates/21.08.0_lun_merge_21.08.0_extended.yaml"},
+		{name: "Case4: base template follow old convention for labelagent and extended template follow new", baseTemplate: "testdata/21.08.0_lun.yaml", extendTemplate: "testdata/extend_lun.yaml", mergeTemplate: "mergeTemplates/21.08.0_lun_merge_extended.yaml"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			baseTemplate, _ := tree.Import("yaml", tt.baseTemplate)
+			extendTemplate, _ := tree.Import("yaml", tt.extendTemplate)
+			baseTemplate.PreprocessTemplate()
+			extendTemplate.PreprocessTemplate()
+			baseTemplate.Merge(extendTemplate, nil)
+			gotString1, _ := yaml.Dump(baseTemplate)
+			gotString := strings.TrimRight(string(gotString1), "\r\n")
+			expected, _ := ioutil.ReadFile(tt.mergeTemplate)
+			expectedString := strings.TrimRight(string(expected), "\r\n")
+
+			if gotString != expectedString {
+				t.Errorf("got %v, want %v", gotString, expectedString)
+			}
+		})
+	}
+}

pkg/test/preProcessResultData/p_21.08.0_extend_lun.yaml

@@ -0,0 +1,12 @@
+plugins:
+  - nic
+  LabelAgent:
+    value_mapping:
+      - custom_status state online `1`
+    value_to_num:
+      - new_status state online online `0`
+    new_mapping:
+      - xyz
+  Aggregator:
+    - node<>node_cpu1
+    - nodetest