perf: reduce allocs when reading credential files

cmd/poller/options/options.go

@@ -14,6 +14,7 @@ package options
 
 import (
 	"github.com/netapp/harvest/v2/pkg/conf"
+	"github.com/netapp/harvest/v2/pkg/logging"
 	"github.com/rs/zerolog"
 	"os"
 	"path/filepath"
@@ -81,7 +82,7 @@ func (o *Options) SetDefaults() *Options {
 	}
 
 	o.HomePath = conf.Path("")
-	o.LogPath = conf.GetHarvestLogPath()
+	o.LogPath = logging.GetLogPath()
 	o.SetConfPath(o.ConfPath)
 
 	// If both debug and loglevel are set, loglevel wins

pkg/conf/conf.go

@@ -5,10 +5,10 @@
 package conf
 
 import (
-	"cmp"
 	"errors"
 	"fmt"
 	"github.com/netapp/harvest/v2/pkg/errs"
+	"github.com/netapp/harvest/v2/pkg/logging"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/netapp/harvest/v2/third_party/mergo"
@@ -20,10 +20,16 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 )
 
-var Config = HarvestConfig{}
-var configRead = false
+var (
+	Config            = HarvestConfig{}
+	configRead        = false
+	readMu            = sync.Mutex{}
+	credentialModTime = int64(0)
+	credConfig        HarvestConfig
+)
 
 const (
 	DefaultAPIVersion = "1.3"
@@ -222,22 +228,28 @@ func DecodeConfig(contents []byte) error {
 }
 
 func ReadCredentialFile(credPath string, p *Poller) error {
-	contents, err := os.ReadFile(credPath)
+	fileChanged, err := hasFileChanged(credPath)
 	if err != nil {
-		abs, err2 := filepath.Abs(credPath)
-		if err2 != nil {
-			abs = credPath
+		return err
+	}
+	if fileChanged {
+		logging.Get().Info().Str("credPath", credPath).Msg("reading credentials")
+		contents, err := os.ReadFile(credPath)
+		if err != nil {
+			abs, err2 := filepath.Abs(credPath)
+			if err2 != nil {
+				abs = credPath
+			}
+			return fmt.Errorf("failed to read file=%s error: %w", abs, err)
+		}
+		err = yaml.Unmarshal(contents, &credConfig)
+		if err != nil {
+			return err
 		}
-		return fmt.Errorf("failed to read file=%s error: %w", abs, err)
 	}
 	if p == nil {
 		return nil
 	}
-	var credConfig HarvestConfig
-	err = yaml.Unmarshal(contents, &credConfig)
-	if err != nil {
-		return err
-	}
 
 	credPoller := credConfig.Pollers[p.Name]
 	if credPoller == nil {
@@ -272,6 +284,20 @@ func ReadCredentialFile(credPath string, p *Poller) error {
 	return nil
 }
 
+func hasFileChanged(path string) (bool, error) {
+	readMu.Lock()
+	defer readMu.Unlock()
+	stat, err := os.Stat(path)
+	if err != nil {
+		return false, fmt.Errorf("failed to stat file=%s error: %w", path, err)
+	}
+	if stat.ModTime().Unix() > credentialModTime {
+		credentialModTime = stat.ModTime().Unix()
+		return true, nil
+	}
+	return false, nil
+}
+
 func PollerNamed(name string) (*Poller, error) {
 	poller, ok := Config.Pollers[name]
 	if !ok {
@@ -299,10 +325,6 @@ func Path(aPath string) string {
 	return filepath.Join(confDir, aPath)
 }
 
-func GetHarvestLogPath() string {
-	return cmp.Or(os.Getenv("HARVEST_LOGS"), "/var/log/harvest/")
-}
-
 // GetLastPromPort returns the Prometheus port for the given poller
 // If multiple Prometheus exporters are configured for a poller, the port for the last exporter is returned.
 func GetLastPromPort(pollerName string, validatePortInUse bool) (int, error) {

pkg/logging/logger.go

@@ -1,7 +1,7 @@
 package logging
 
 import (
-	"github.com/netapp/harvest/v2/pkg/conf"
+	"cmp"
 	"io"
 	"os"
 	"path"
@@ -67,7 +67,7 @@ func Get() *Logger {
 				PrefixValue:        defaultPrefixValue,
 				LogLevel:           defaultLogLevel,
 				FileLoggingEnabled: defaultFileLoggingEnabled,
-				Directory:          conf.GetHarvestLogPath(),
+				Directory:          GetLogPath(),
 				Filename:           defaultLogFileName,
 				MaxSize:            DefaultLogMaxMegaBytes,
 				MaxBackups:         DefaultLogMaxBackups,
@@ -78,6 +78,10 @@ func Get() *Logger {
 	return logger
 }
 
+func GetLogPath() string {
+	return cmp.Or(os.Getenv("HARVEST_LOGS"), "/var/log/harvest/")
+}
+
 // SubLogger adds the field key with val as a string to the logger context and returns sublogger
 func (l *Logger) SubLogger(key string, value string) *Logger {
 	if l != nil {