feat: reduce dependencies (#2812)

* feat: reduce dependencies feat: log current working directory on startup feat: Docker build should use Go version in .harvest.env fix: Suppress LD_WARNING when running go test on macOS doc: document how to build Docker image
NetApp · Apr 8, 2024 · 1083cfd · 1083cfd
1 parent 3b9b7d2
commit 1083cfd
Show file tree

Hide file tree

Showing 65 changed files with 122 additions and 159 deletions.
diff --git a/Makefile b/Makefile
@@ -32,6 +32,13 @@ BIN_PLATFORM ?= linux
 BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 MKDOCS_EXISTS := $(shell which mkdocs)
 FETCH_ASUP_EXISTS := $(shell which ./.github/fetch-asup)
+HARVEST_ENV := .harvest.env
+
+# Read the environment file if it exists and export the uncommented variables
+ifneq (,$(wildcard $(HARVEST_ENV)))
+    include $(HARVEST_ENV)
+	export $(shell sed '/^\#/d; s/=.*//' $(HARVEST_ENV))
+endif
 
 help:  ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
@@ -41,7 +48,6 @@ header:
 	@echo "   | || |__ _ _ ___ _____ __| |_  |_  ) /  \  "
 	@echo "   | __ / _\` | '_\ V / -_|_-<  _|  / / | () | "
 	@echo "   |_||_\__,_|_|  \_/\___/__/\__| /___(_)__/  "
-	@echo
 
 deps: header ## Check dependencies
 	@# Make sure that go exists
@@ -61,7 +67,9 @@ clean: ## Cleanup the project binary (bin) folders
 
 test: ## run tests
 	@echo "Running tests"
-	go test -race -shuffle=on ./...
+	# The ldflags force the old Apple linker to suppress ld warning messages on MacOS
+	# See https://github.com/golang/go/issues/61229#issuecomment-1988965927
+	go test -ldflags=-extldflags=-Wl,-ld_classic -race -shuffle=on ./...
 
 fmt: ## format the go source files
 	@echo "Formatting"
@@ -156,7 +164,7 @@ endif
 	-@docker volume rm harvest_grafana_data harvest_prometheus_data 2>/dev/null || true
 	@if [ "$(ci)" != "harvest.yml" ]; then cp $(ci) harvest.yml; else echo "Source and destination files are the same, skipping copy"; fi
 	@./bin/harvest generate docker full --port --output harvest-compose.yml
-	@docker build -f container/onePollerPerContainer/Dockerfile -t ghcr.io/netapp/harvest:latest . --no-cache --build-arg VERSION=${VERSION}
+	@docker build -f container/onePollerPerContainer/Dockerfile -t ghcr.io/netapp/harvest:latest . --no-cache --build-arg GO_VERSION=${GO_VERSION} --build-arg VERSION=${VERSION}
 	@docker-compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans
 	@cp harvest.yml integration/test/
 	VERSION=${VERSION} INSTALL_DOCKER=1 ./integration/test/test.sh

diff --git a/cmd/collectors/collectorstest.go b/cmd/collectors/collectorstest.go
@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"github.com/netapp/harvest/v2/pkg/tree"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
+	"github.com/netapp/harvest/v2/third_party/tidwall/sjson"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 	"io"
 	"os"
 	"path/filepath"

diff --git a/cmd/collectors/rest/plugins/health/health.go b/cmd/collectors/rest/plugins/health/health.go
@@ -2,14 +2,14 @@ package health
 
 import (
 	"fmt"
-	goversion "github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/tools/rest"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/util"
+	goversion "github.com/netapp/harvest/v2/third_party/go-version"
 	"github.com/tidwall/gjson"
 	"strconv"
 	"time"

diff --git a/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go b/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go
@@ -1,13 +1,13 @@
 package volumeanalytics
 
 import (
-	goversion "github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/tools/rest"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/util"
+	goversion "github.com/netapp/harvest/v2/third_party/go-version"
 	"github.com/tidwall/gjson"
 	"path"
 	"strconv"

diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go
@@ -5,14 +5,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/pkg/auth"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/logging"
 	"github.com/netapp/harvest/v2/pkg/requests"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
+	"github.com/netapp/harvest/v2/third_party/go-version"
 	"github.com/tidwall/gjson"
 	"io"
 	"net/http"

diff --git a/cmd/collectors/zapi/plugins/aggregate/aggregate.go b/cmd/collectors/zapi/plugins/aggregate/aggregate.go
@@ -2,7 +2,6 @@ package aggregate
 
 import (
 	"errors"
-	goversion "github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
@@ -11,6 +10,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
+	goversion "github.com/netapp/harvest/v2/third_party/go-version"
 	"strconv"
 	"strings"
 )

diff --git a/cmd/harvest/harvest.go b/cmd/harvest/harvest.go
@@ -28,7 +28,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/set"
 	"github.com/netapp/harvest/v2/pkg/util"
-	tw "github.com/olekukonko/tablewriter"
+	tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
 	"log"
 	"net"

diff --git a/cmd/harvest/version/version.go b/cmd/harvest/version/version.go
@@ -7,7 +7,7 @@ package version
 import (
 	"errors"
 	"fmt"
-	"github.com/hashicorp/go-version"
+	"github.com/netapp/harvest/v2/third_party/go-version"
 	"github.com/spf13/cobra"
 	"net/http"
 	"net/url"

diff --git a/cmd/poller/collector/helpers.go b/cmd/poller/collector/helpers.go
@@ -10,7 +10,6 @@ package collector
 import (
 	"errors"
 	"fmt"
-	"github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin/aggregator"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin/changelog"
@@ -21,6 +20,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/tree"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
+	"github.com/netapp/harvest/v2/third_party/go-version"
 	"os"
 	"path/filepath"
 	"regexp"

diff --git a/cmd/poller/collector/helpers_test.go b/cmd/poller/collector/helpers_test.go
@@ -1,8 +1,8 @@
 package collector
 
 import (
-	"github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/pkg/conf"
+	"github.com/netapp/harvest/v2/third_party/go-version"
 	"sort"
 	"testing"
 )

diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go
@@ -207,9 +207,15 @@ func (p *Poller) Init() error {
 		}()
 	}
 
+	getwd, err := os.Getwd()
+	if err != nil {
+		logger.Error().Err(err).Msg("Unable to get current working directory")
+		getwd = ""
+	}
 	logger.Info().
 		Str("logLevel", zeroLogLevel.String()).
 		Str("configPath", configPath).
+		Str("cwd", getwd).
 		Str("version", strings.TrimSpace(version.String())).
 		EmbedObject(p.options).
 		Msg("Init")

diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go
@@ -10,10 +10,9 @@ import (
 	"github.com/netapp/harvest/v2/pkg/color"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/logging"
+	"github.com/netapp/harvest/v2/third_party/tidwall/sjson"
 	"github.com/spf13/cobra"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/pretty"
-	"github.com/tidwall/sjson"
 	"io"
 	"log"
 	"os"
@@ -646,12 +645,9 @@ func generateDescription(dPath string, data []byte, counters map[string]Counter)
 	}
 
 	// Sorted json
-	sorted := pretty.PrettyOptions(data, &pretty.Options{
-		SortKeys: true,
-		Indent:   "  ",
-	})
+	result := gjson.GetBytes(data, `@pretty:{"sortKeys":true, "indent":"  ", "width":0}`)
 
-	if err = os.WriteFile(dPath, sorted, grafana.GPerm); err != nil {
+	if err = os.WriteFile(dPath, []byte(result.Raw), grafana.GPerm); err != nil {
 		log.Fatalf("failed to write dashboard=%s err=%v\n", dPath, err)
 	}
 }

diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go
@@ -3,7 +3,6 @@ package grafana
 import (
 	"fmt"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/pretty"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -1322,11 +1321,8 @@ func TestDashboardKeysAreSorted(t *testing.T) {
 		dashboards,
 		func(path string, data []byte) {
 			path = ShortPath(path)
-			sorted := pretty.PrettyOptions(data, &pretty.Options{
-				SortKeys: true,
-				Indent:   "  ",
-			})
-			if string(sorted) != string(data) {
+			sorted := gjson.GetBytes(data, `@pretty:{"sortKeys":true, "indent":"  ", "width":0}`).String()
+			if sorted != string(data) {
 				sortedPath := writeSorted(t, path, sorted)
 				path = "grafana/dashboards/" + path
 				t.Errorf("dashboard=%s should have sorted keys but does not. Sorted version created at path=%s.\ncp %s %s",
@@ -1335,7 +1331,7 @@ func TestDashboardKeysAreSorted(t *testing.T) {
 		})
 }
 
-func writeSorted(t *testing.T, path string, sorted []byte) string {
+func writeSorted(t *testing.T, path string, sorted string) string {
 	dir, file := filepath.Split(path)
 	dir = filepath.Dir(dir)
 	dest := filepath.Join("/tmp", dir, file)
@@ -1351,7 +1347,7 @@ func writeSorted(t *testing.T, path string, sorted []byte) string {
 		t.Errorf("failed to create file=%s err=%v", dest, err)
 		return ""
 	}
-	_, err = create.Write(sorted)
+	_, err = create.WriteString(sorted)
 	if err != nil {
 		t.Errorf("failed to write sorted json to file=%s err=%v", dest, err)
 		return ""

diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go
@@ -10,14 +10,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	goversion "github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/cmd/harvest/version"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/requests"
 	"github.com/netapp/harvest/v2/pkg/util"
+	goversion "github.com/netapp/harvest/v2/third_party/go-version"
+	"github.com/netapp/harvest/v2/third_party/tidwall/sjson"
 	"github.com/spf13/cobra"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 	"io"

diff --git a/cmd/tools/rest/rest.go b/cmd/tools/rest/rest.go
@@ -11,9 +11,9 @@ import (
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/logging"
 	"github.com/netapp/harvest/v2/pkg/util"
+	"github.com/netapp/harvest/v2/third_party/tidwall/sjson"
 	"github.com/spf13/cobra"
 	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 	"log"
 	"net/url"
 	"os"

diff --git a/cmd/tools/rest/swag.go b/cmd/tools/rest/swag.go
@@ -6,10 +6,10 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/bbrks/wrap/v2"
 	"github.com/go-openapi/spec"
 	"github.com/netapp/harvest/v2/pkg/errs"
-	tw "github.com/olekukonko/tablewriter"
+	"github.com/netapp/harvest/v2/third_party/bbrks/wrap/v2"
+	tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter"
 	"golang.org/x/exp/maps"
 	"gopkg.in/yaml.v3"
 	"html"

diff --git a/container/onePollerPerContainer/Dockerfile b/container/onePollerPerContainer/Dockerfile
@@ -1,4 +1,6 @@
-FROM golang:1.22.2 as builder
+# GO_VERSION should be overridden by the build script via --build-arg GO_VERSION=$value
+ARG GO_VERSION
+FROM golang:${GO_VERSION} as builder
 
 SHELL ["/bin/bash", "-c"]
 

diff --git a/deb/build-deb.sh b/deb/build-deb.sh
@@ -23,7 +23,7 @@ mkdir -p "$BUILD/opt/harvest/bin/"
 cp -r "$SRC/grafana" "$SRC/conf" "$BUILD/opt/harvest/"
 cp "$SRC/harvest.yml" "$SRC/prom-stack.tmpl" "$SRC/harvest.cue" "$BUILD/opt/harvest/"
 cp -r "$SRC/.github/" "$BUILD/opt/harvest/"
-cp -r "$SRC/pkg/" "$SRC/cmd/" "$SRC/container/" "$BUILD/opt/harvest/"
+cp -r "$SRC/pkg/" "$SRC/cmd/" "$SRC/container/" "$SRC/third_party/" "$BUILD/opt/harvest/"
 cp -r "$SRC/rpm/" "$SRC/deb/" "$SRC/service/" "$SRC/cert/" "$SRC/autosupport/" "$SRC/.git" "$BUILD/opt/harvest/"
 cp "$SRC/Makefile" "$SRC/README.md" "$SRC/LICENSE" "$SRC/go.mod" "$SRC/go.sum" "$BUILD/opt/harvest/"
 if [ -d "$SRC/vendor" ]; then

diff --git a/docs/install/containerd.md b/docs/install/containerd.md
@@ -35,7 +35,8 @@ You only need to [create a new image](containers.md#building-harvest-docker-imag
 These are the same steps outline on [Building Harvest Docker Image](containers.md#building-harvest-docker-image) except we replace `docker build` with `nerdctl` like so:
 
 ```sh
-nerdctl build -f container/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache 
+source .harvest.env
+nerdctl build -f container/onePollerPerContainer/Dockerfile --build-arg GO_VERSION=${GO_VERSION} -t harvest:latest . --no-cache 
 ```
 
 ## Generate a Harvest compose file