refactor: move unit testing json parser to common

cmd/collectors/collectorstest.go

@@ -0,0 +1,77 @@
+package collectors
+
+import (
+	"bytes"
+	"compress/gzip"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// Common unit testing helpers
+
+var (
+	gsonCache = make(map[string][]gjson.Result)
+)
+
+func JSONToGson(path string, flatten bool) []gjson.Result {
+	var (
+		result []gjson.Result
+		err    error
+	)
+	results, ok := gsonCache[path]
+	if ok {
+		return results
+	}
+
+	var reader io.Reader
+	if filepath.Ext(path) == ".gz" {
+		open, err := os.Open(path)
+		if err != nil {
+			panic(err)
+		}
+		defer open.Close()
+		reader, err = gzip.NewReader(open)
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil
+		}
+		reader = bytes.NewReader(data)
+	}
+	var b bytes.Buffer
+	_, err = io.Copy(&b, reader) //nolint:gosec
+	if err != nil {
+		return nil
+	}
+	bb := b.Bytes()
+	output := gjson.GetManyBytes(bb, "records", "num_records", "_links.next.href")
+
+	data := output[0]
+	numRecords := output[1]
+	isNonIterRestCall := !data.Exists()
+
+	if isNonIterRestCall {
+		contentJSON := `{"records":[]}`
+		response, err := sjson.SetRawBytes([]byte(contentJSON), "records.-1", bb)
+		if err != nil {
+			panic(err)
+		}
+		value := gjson.GetBytes(response, "records")
+		result = append(result, value)
+	} else if numRecords.Int() > 0 {
+		if flatten {
+			result = append(result, data.Array()...)
+		} else {
+			result = append(result, data)
+		}
+	}
+
+	gsonCache[path] = result
+	return result
+}

cmd/collectors/rest/rest.go

@@ -155,7 +155,7 @@ func (r *Rest) InitClient() error {
 		return err
 	}
 
-	if r.IsTest {
+	if r.Options.IsTest {
 		return nil
 	}
 	if err = r.Client.Init(5); err != nil {
@@ -199,7 +199,7 @@ func (r *Rest) getClient(a *collector.AbstractCollector, c *auth.Credentials) (*
 		return nil, errs.New(errs.ErrMissingParam, "addr")
 	}
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if a.IsTest {
+	if a.Options.IsTest {
 		return &rest.Client{}, nil
 	}
 	if client, err = rest.New(poller, timeout, c); err != nil {
@@ -355,12 +355,6 @@ func (r *Rest) processEndPoints(endpointFunc func(e *endPoint) ([]gjson.Result,
 		var (
 			records []gjson.Result
 		)
-		counterKey := make([]string, len(endpoint.prop.Counters))
-		i := 0
-		for k := range endpoint.prop.Counters {
-			counterKey[i] = k
-			i++
-		}
 
 		records, err = endpointFunc(endpoint)
 

cmd/collectors/rest/rest_test.go

@@ -1,20 +1,16 @@
 package rest
 
 import (
-	"bytes"
-	"compress/gzip"
 	"fmt"
+	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/options"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
-	"io"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -71,14 +67,13 @@ var (
 	ms           []*matrix.Matrix
 	benchRest    *Rest
 	fullPollData []gjson.Result
-	gsonCache    = make(map[string][]gjson.Result)
 )
 
 func TestMain(m *testing.M) {
 	conf.TestLoadHarvestConfig("testdata/config.yml")
 
 	benchRest = newRest("Volume", "volume.yaml")
-	fullPollData = jsonToGson("testdata/volume-1.json.gz")
+	fullPollData = collectors.JSONToGson("testdata/volume-1.json.gz", true)
 	now := time.Now().Truncate(time.Second)
 	_, _ = benchRest.pollData(now, fullPollData, volumeEndpoints)
 
@@ -128,7 +123,7 @@ func Test_pollDataVolume(t *testing.T) {
 
 			r := newRest("Volume", "volume.yaml")
 			now := time.Now().Truncate(time.Second)
-			pollData := jsonToGson(tt.pollDataPath1)
+			pollData := collectors.JSONToGson(tt.pollDataPath1, true)
 
 			mm, err := r.pollData(now, pollData, volumeEndpoints)
 			if err != nil {
@@ -151,65 +146,8 @@ func Test_pollDataVolume(t *testing.T) {
 
 func volumeEndpoints(e *endPoint) ([]gjson.Result, error) {
 	path := "testdata/" + strings.ReplaceAll(e.prop.Query, "/", "-") + ".json.gz"
-	toGjson := jsonToGson(path)
-	return toGjson, nil
-}
-
-func jsonToGson(path string) []gjson.Result {
-
-	var (
-		result []gjson.Result
-		err    error
-	)
-	results, ok := gsonCache[path]
-	if ok {
-		return results
-	}
-
-	var reader io.Reader
-	if filepath.Ext(path) == ".gz" {
-		open, err := os.Open(path)
-		if err != nil {
-			panic(err)
-		}
-		defer open.Close()
-		reader, err = gzip.NewReader(open)
-		if err != nil {
-			panic(err)
-		}
-	} else {
-		data, err := os.ReadFile(path)
-		if err != nil {
-			return nil
-		}
-		reader = bytes.NewReader(data)
-	}
-	var b bytes.Buffer
-	_, err = io.Copy(&b, reader) //nolint:gosec
-	if err != nil {
-		return nil
-	}
-	bb := b.Bytes()
-	output := gjson.GetManyBytes(bb, "records", "num_records", "_links.next.href")
-
-	data := output[0]
-	numRecords := output[1]
-	isNonIterRestCall := !data.Exists()
-
-	if isNonIterRestCall {
-		contentJSON := `{"records":[]}`
-		response, err := sjson.SetRawBytes([]byte(contentJSON), "records.-1", bb)
-		if err != nil {
-			panic(err)
-		}
-		value := gjson.GetBytes(response, "records")
-		result = append(result, value)
-	} else if numRecords.Int() > 0 {
-		result = append(result, data.Array()...)
-	}
-
-	gsonCache[path] = result
-	return result
+	gson := collectors.JSONToGson(path, true)
+	return gson, nil
 }
 
 func newRest(object string, path string) *Rest {
@@ -220,7 +158,6 @@ func newRest(object string, path string) *Rest {
 		IsTest:   true,
 	}
 	ac := collector.New("Rest", object, &opts, params(object, path), nil)
-	ac.IsTest = true
 	r := Rest{}
 	err = r.Init(ac)
 	if err != nil {

cmd/collectors/restperf/restperf_test.go

@@ -1,9 +1,8 @@
 package restperf
 
 import (
-	"bytes"
-	"compress/gzip"
 	"fmt"
+	"github.com/netapp/harvest/v2/cmd/collectors"
 	rest2 "github.com/netapp/harvest/v2/cmd/collectors/rest"
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/options"
@@ -13,10 +12,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/tree"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
-	"io"
 	"os"
-	"path/filepath"
 	"reflect"
 	"sort"
 	"testing"
@@ -208,9 +204,9 @@ func newRestPerf(object string, path string) *RestPerf {
 	opts := options.Options{
 		Poller:   pollerName,
 		HomePath: "testdata",
+		IsTest:   true,
 	}
 	ac := collector.New("RestPerf", object, &opts, params(object, path), nil)
-	ac.IsTest = true
 	r := RestPerf{}
 	err = r.Init(ac)
 	if err != nil {
@@ -224,46 +220,8 @@ func jsonToPerfRecords(path string) []rest.PerfRecord {
 		perfRecords []rest.PerfRecord
 		p           rest.PerfRecord
 	)
-	var reader io.Reader
-	if filepath.Ext(path) == ".gz" {
-		open, err := os.Open(path)
-		if err != nil {
-			panic(err)
-		}
-		defer open.Close()
-		reader, err = gzip.NewReader(open)
-		if err != nil {
-			panic(err)
-		}
-	} else {
-		data, err := os.ReadFile(path)
-		if err != nil {
-			return nil
-		}
-		reader = bytes.NewReader(data)
-	}
-	var b bytes.Buffer
-	_, err := io.Copy(&b, reader) //nolint:gosec
-	if err != nil {
-		return nil
-	}
-	bb := b.Bytes()
-	output := gjson.GetManyBytes(bb, "records", "num_records", "_links.next.href")
-
-	data := output[0]
-	numRecords := output[1]
-
-	if !data.Exists() {
-		contentJSON := `{"records":[]}`
-		response, err := sjson.SetRawBytes([]byte(contentJSON), "records.-1", bb)
-		if err != nil {
-			panic(err)
-		}
-		p = rest.PerfRecord{Records: gjson.GetBytes(response, "records")}
-	}
-	if numRecords.Int() > 0 {
-		p = rest.PerfRecord{Records: data, Timestamp: time.Now().UnixNano()}
-	}
+	gson := collectors.JSONToGson(path, false)
+	p = rest.PerfRecord{Records: gson[0], Timestamp: time.Now().UnixNano()}
 	perfRecords = append(perfRecords, p)
 
 	return perfRecords

cmd/poller/collector/collector.go

@@ -103,7 +103,6 @@ type AbstractCollector struct {
 	HostVersion string
 	HostModel   string
 	HostUUID    string
-	IsTest      bool
 }
 
 func New(name, object string, options *options.Options, params *node.Node, credentials *auth.Credentials) *AbstractCollector {