ci: ensure exported Prometheus metrics are unique

conf/restperf/9.12.0/fcp.yaml

@@ -1,4 +1,4 @@
-name:                     Fcp
+name:                     FcpPort
 query:                    api/cluster/counter/tables/fcp
 object:                   fcp
 

conf/restperf/9.12.0/wafl_comp_aggr_vol_bin.yaml

@@ -4,6 +4,7 @@ object:                   fabricpool
 
 counters:
   - ^^id
+  - ^cloud_target.name           => cloud_target
   - ^svm.name                    => svm
   - ^volume.name                 => volume
   - cloud_bin_op                 => cloud_bin_operation
@@ -14,6 +15,7 @@ plugins:
 
 export_options:
   instance_keys:
+    - cloud_target
     - svm
     - volume
 

conf/restperf/default.yaml

@@ -12,7 +12,7 @@ objects:
   Disk:              disk.yaml
   ExtCacheObj:       ext_cache_obj.yaml
   FCVI:              fcvi.yaml
-  Fcp:               fcp.yaml
+  FcpPort:           fcp.yaml
   HeadroomAggr:      resource_headroom_aggr.yaml
   HeadroomCPU:       resource_headroom_cpu.yaml
   HostAdapter:       hostadapter.yaml

conf/zapiperf/cdot/9.8.0/wafl_comp_aggr_vol_bin.yaml

@@ -8,11 +8,13 @@ counters:
   - cloud_bin_op_latency_average
   - cloud_bin_operation
   - instance_name
+  - object_store_name       => cloud_target
   - vol_name                => volume
   - vserver_name            => svm
 
 export_options:
   instance_keys:
+    - cloud_target
     - svm
     - volume
 

integration/go.mod

@@ -8,27 +8,27 @@ require (
 	github.com/netapp/harvest/v2 v2.0.0-20230404163343-f21b0c1a08ac
 	github.com/rs/zerolog v1.29.1
 	github.com/tidwall/gjson v1.14.4
-	golang.org/x/text v0.9.0
+	golang.org/x/text v0.10.0
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
-	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
-	github.com/shirou/gopsutil/v3 v3.23.4 // indirect
-	github.com/shoenig/go-m1cpu v0.1.5 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.5 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/sys v0.7.0 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	golang.org/x/sys v0.9.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

integration/go.sum

@@ -1,3 +1,5 @@
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -11,8 +13,6 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
-github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
@@ -23,8 +23,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
-github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -35,12 +35,12 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/shirou/gopsutil/v3 v3.23.4 h1:hZwmDxZs7Ewt75DV81r4pFMqbq+di2cbt9FsQBqLD2o=
-github.com/shirou/gopsutil/v3 v3.23.4/go.mod h1:ZcGxyfzAMRevhUR2+cfhXDH6gQdFYE/t8j1nsU4mPI8=
-github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ=
-github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ=
-github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c=
-github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y=
+github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY=
+github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
+github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
+github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
+github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -50,8 +50,8 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
 github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -61,21 +61,23 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
-github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
 github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
-github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
-github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
+github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
+golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

integration/test/metric_test.go

@@ -1,43 +1,87 @@
 package main
 
 import (
+	"fmt"
 	"github.com/Netapp/harvest-automation/test/installer"
 	"github.com/Netapp/harvest-automation/test/utils"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/rs/zerolog/log"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
 )
 
+// Skip aggr_efficiency template metrics since the Rest collector does not have a separate template
+
+var skipDuplicates = map[string]bool{
+	"aggr_logical_used_wo_snapshots":             true,
+	"aggr_logical_used_wo_snapshots_flexclones":  true,
+	"aggr_physical_used_wo_snapshots":            true,
+	"aggr_physical_used_wo_snapshots_flexclones": true,
+	"aggr_total_logical_used":                    true,
+	"aggr_total_physical_used":                   true,
+}
+
 func TestPollerMetrics(t *testing.T) {
 	utils.SkipIfMissing(t, utils.Regression)
 	err := conf.LoadHarvestConfig(installer.HarvestConfigFile)
 	if err != nil {
 		log.Fatal().Err(err).Msg("Unable to load harvest config")
 	}
+	var duplicateMetrics []string
 	for _, pollerName := range conf.Config.PollersOrdered {
 		port, _ := conf.GetPrometheusExporterPorts(pollerName, true)
 		portString := strconv.Itoa(port)
 		var validCounters = 0
+		uniqueSetOfMetricLabels := make(map[string]bool)
 		sb, err2 := utils.GetResponse("http://localhost:" + strings.TrimSpace(portString) + "/metrics")
 		if err2 != nil {
-			panic("Unable to get metric data")
+			t.Fatalf("Unable to get metric data for %s", pollerName)
 		}
 		rows := strings.Split(sb, "\n")
 		for i := range rows {
 			row := rows[i]
-			length := len(row)
-			if length == 0 {
+			if len(row) == 0 {
+				continue
+			}
+			// Ignore comments
+			if strings.HasPrefix(row, "#") {
+				continue
+			}
+			openBracket := strings.Index(row, "{")
+			firstSpace := strings.Index(row, " ")
+			if openBracket == -1 {
+				// this means the metric has this form
+				// metric_without_labels 12.47
+				if firstSpace == -1 {
+					continue
+				}
+				key := metricAndLabelKey(row[:firstSpace], "")
+				if uniqueSetOfMetricLabels[key] {
+					duplicateMetrics = append(duplicateMetrics,
+						fmt.Sprintf("Duplicate metric poller=%s, got >1 want 1 of %s", pollerName, key))
+				}
+				uniqueSetOfMetricLabels[key] = true
 				continue
 			}
-			open := strings.Index(row, "{")
 			closeBracket := strings.Index(row, "}")
-			space := strings.Index(row, " ")
-			if open > 0 && closeBracket > 0 && space > 0 {
-				//objectName := row[0:open]
-				//metricContent := row[open:(close+1)]
-				metricValue, _ := strconv.Atoi(strings.TrimSpace(row[(closeBracket + 1):length]))
+
+			if openBracket > 0 && closeBracket > 0 && firstSpace > 0 {
+				// Turn metric and labels into a unique key
+				metricName := row[:openBracket]
+				key := metricAndLabelKey(metricName, row[openBracket+1:])
+				if uniqueSetOfMetricLabels[key] {
+					_, ok := skipDuplicates[metricName]
+					if ok {
+						log.Trace().Str("metric", key).Msg("Ignore duplicate")
+						continue
+					}
+					duplicateMetrics = append(duplicateMetrics,
+						fmt.Sprintf("Duplicate metric poller=%s, got >1 want 1 of %s", pollerName, key))
+				}
+				uniqueSetOfMetricLabels[key] = true
+				metricValue, _ := strconv.Atoi(strings.TrimSpace(row[(closeBracket + 1):]))
 				if metricValue > 0 {
 					validCounters++
 				}
@@ -50,4 +94,64 @@ func TestPollerMetrics(t *testing.T) {
 		}
 		log.Info().Int("numCounters", validCounters).Str("poller", pollerName).Msg("Valid Counters for poller")
 	}
+	sort.Strings(duplicateMetrics)
+	for _, dupMetric := range duplicateMetrics {
+		t.Errorf(dupMetric)
+	}
+}
+
+func metricAndLabelKey(metric string, rest string) string {
+	var (
+		scanner int
+		labels  []string
+	)
+
+	for {
+		label, equalIndex := readLabel(rest, scanner)
+		if equalIndex == 0 {
+			break
+		}
+		if equalIndex != 0 {
+			equalIndex++
+			if string(rest[equalIndex]) == `"` {
+				// Scan until you hit another unescaped quote.
+				// Can be any sequence of UTF-8 characters, but the backslash (\),
+				// and double-quote (") characters have to be
+				// escaped as \, and \"
+				labelEnd := 0
+				for i := equalIndex + 1; i < len(rest); i++ {
+					s := string(rest[i])
+					if s == `\` {
+						i++
+						continue
+					}
+					if s == `"` {
+						// done reading quoted
+						labelEnd = i
+						break
+					}
+				}
+				labelValue := rest[equalIndex+1 : labelEnd]
+				labels = append(labels, fmt.Sprintf(`%s="%s"`, label, labelValue))
+				scanner = labelEnd + 1
+				if string(rest[scanner]) == "," {
+					scanner++
+				}
+			}
+		}
+	}
+
+	sort.Strings(labels)
+
+	return metric + "{" + strings.Join(labels, ",") + "}"
+}
+
+func readLabel(sample string, i int) (string, int) {
+	sub := sample[i:]
+	equalsIndex := strings.Index(sub, "=")
+	if equalsIndex == -1 {
+		return sample, 0
+	}
+	end := i + equalsIndex
+	return sample[i:end], end
 }