diff --git a/base/log/log.go b/base/log/log.go index f6e890c46..6c684593c 100644 --- a/base/log/log.go +++ b/base/log/log.go @@ -17,22 +17,27 @@ package log import ( "net/url" "os" - "path/filepath" "runtime" "strings" "github.com/emicklei/go-restful/v3" "github.com/go-sql-driver/mysql" - "github.com/samber/lo" + "github.com/spf13/pflag" "go.opentelemetry.io/otel" "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" ) var logger *zap.Logger func init() { // setup default logger - SetProductionLogger() + var err error + logger, err = zap.NewProduction() + if err != nil { + panic(err) + } // Windows file sink support: https://github.com/uber-go/zap/issues/621 if runtime.GOOS == "windows" { if err := zap.RegisterSink("windows", func(u *url.URL) (zap.Sink, error) { @@ -53,22 +58,9 @@ func ResponseLogger(resp *restful.Response) *zap.Logger { return logger.With(zap.String("request_id", resp.Header().Get("X-Request-ID"))) } -// SetProductionLogger set current logger in production mode. -func SetProductionLogger(outputPaths ...string) { - for _, outputPath := range outputPaths { - err := os.MkdirAll(filepath.Dir(outputPath), os.ModePerm) - if err != nil { - panic(err) - } - } +func CloseLogger() { cfg := zap.NewProductionConfig() - if runtime.GOOS == "windows" { - cfg.OutputPaths = append(cfg.OutputPaths, lo.Map(outputPaths, func(path string, _ int) string { - return "windows:///" + path - })...) - } else { - cfg.OutputPaths = append(cfg.OutputPaths, outputPaths...) - } + cfg.Level = zap.NewAtomicLevelAt(zap.FatalLevel) var err error logger, err = cfg.Build() if err != nil { @@ -76,31 +68,44 @@ func SetProductionLogger(outputPaths ...string) { } } -// SetDevelopmentLogger set current logger in development mode. -func SetDevelopmentLogger(outputPaths ...string) { - for _, outputPath := range outputPaths { - err := os.MkdirAll(filepath.Dir(outputPath), os.ModePerm) - if err != nil { - panic(err) - } - } - cfg := zap.NewDevelopmentConfig() - cfg.OutputPaths = append(cfg.OutputPaths, outputPaths...) - var err error - logger, err = cfg.Build() - if err != nil { - panic(err) - } +func AddFlags(flagSet *pflag.FlagSet) { + flagSet.String("log-path", "", "path of log file") + flagSet.Int("log-max-size", 100, "maximum size in megabytes of the log file") + flagSet.Int("log-max-age", 0, "maximum number of days to retain old log files") + flagSet.Int("log-max-backups", 0, "maximum number of old log files to retain") } -func CloseLogger() { - cfg := zap.NewProductionConfig() - cfg.Level = zap.NewAtomicLevelAt(zap.FatalLevel) - var err error - logger, err = cfg.Build() - if err != nil { - panic(err) +func SetLogger(flagSet *pflag.FlagSet, debug bool) { + // enable or disable debug mode + var ( + encoder zapcore.Encoder + level zapcore.LevelEnabler + ) + if debug { + encoder = zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) + level = zap.DebugLevel + } else { + encoder = zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()) + level = zap.InfoLevel + } + // create lumberjack logger + writers := []zapcore.WriteSyncer{zapcore.AddSync(os.Stdout)} + if flagSet.Changed("log-path") { + path, _ := flagSet.GetString("log-path") + maxSize, _ := flagSet.GetInt("log-max-size") + maxAge, _ := flagSet.GetInt("log-max-age") + maxBackups, _ := flagSet.GetInt("log-max-backups") + writers = append(writers, zapcore.AddSync(&lumberjack.Logger{ + Filename: path, + MaxSize: maxSize, + MaxBackups: maxBackups, + MaxAge: maxAge, + Compress: false, + })) } + // create zap logger + core := zapcore.NewCore(encoder, zap.CombineWriteSyncers(writers...), level) + logger = zap.New(core) } const mysqlPrefix = "mysql://" diff --git a/base/log/log_test.go b/base/log/log_test.go index fee2c5015..3cef90d62 100644 --- a/base/log/log_test.go +++ b/base/log/log_test.go @@ -15,56 +15,46 @@ package log import ( - "github.com/stretchr/testify/assert" - "os" - "runtime" "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" ) func TestSetDevelopmentLogger(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - temp, err := os.MkdirTemp("", "test_gorse") - assert.NoError(t, err) + temp := t.TempDir() + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + AddFlags(flagSet) // set existed path - SetDevelopmentLogger(temp + "/gorse.log") - _, err = os.Stat(temp + "/gorse.log") + err := flagSet.Set("log-path", temp+"/gorse.log") assert.NoError(t, err) + SetLogger(flagSet, true) + Logger().Debug("test") + assert.FileExists(t, temp+"/gorse.log") // set non-existed path - SetDevelopmentLogger(temp + "/gorse/gorse.log") - _, err = os.Stat(temp + "/gorse/gorse.log") + err = flagSet.Set("log-path", temp+"/gorse/gorse.log") assert.NoError(t, err) - // permission denied - assert.Panics(t, func() { - SetDevelopmentLogger("/gorse.log") - }) - assert.Panics(t, func() { - SetDevelopmentLogger("/gorse/gorse.log") - }) + SetLogger(flagSet, true) + Logger().Debug("test") + assert.FileExists(t, temp+"/gorse/gorse.log") } func TestSetProductionLogger(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - temp, err := os.MkdirTemp("", "test_gorse") - assert.NoError(t, err) + temp := t.TempDir() + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + AddFlags(flagSet) // set existed path - SetProductionLogger(temp + "/gorse.log") - _, err = os.Stat(temp + "/gorse.log") + err := flagSet.Set("log-path", temp+"/gorse.log") assert.NoError(t, err) + SetLogger(flagSet, false) + Logger().Info("test") + assert.FileExists(t, temp+"/gorse.log") // set non-existed path - SetProductionLogger(temp + "/gorse/gorse.log") - _, err = os.Stat(temp + "/gorse/gorse.log") + err = flagSet.Set("log-path", temp+"/gorse/gorse.log") assert.NoError(t, err) - // permission denied - assert.Panics(t, func() { - SetProductionLogger("/gorse.log") - }) - assert.Panics(t, func() { - SetProductionLogger("/gorse/gorse.log") - }) + SetLogger(flagSet, false) + Logger().Info("test") + assert.FileExists(t, temp+"/gorse/gorse.log") } func TestRedactDBURL(t *testing.T) { diff --git a/cmd/gorse-in-one/main.go b/cmd/gorse-in-one/main.go index 195011ac1..ea9cd9297 100644 --- a/cmd/gorse-in-one/main.go +++ b/cmd/gorse-in-one/main.go @@ -58,17 +58,8 @@ var oneCommand = &cobra.Command{ } // setup logger - var outputPaths []string - if cmd.PersistentFlags().Changed("log-path") { - outputPath, _ := cmd.PersistentFlags().GetString("log-path") - outputPaths = append(outputPaths, outputPath) - } - debugMode, _ := cmd.PersistentFlags().GetBool("debug") - if debugMode { - log.SetDevelopmentLogger(outputPaths...) - } else { - log.SetProductionLogger(outputPaths...) - } + debug, _ := cmd.PersistentFlags().GetBool("debug") + log.SetLogger(cmd.PersistentFlags(), debug) // load config var conf *config.Config @@ -135,10 +126,10 @@ var oneCommand = &cobra.Command{ } func init() { + log.AddFlags(oneCommand.PersistentFlags()) oneCommand.PersistentFlags().Bool("debug", false, "use debug log mode") oneCommand.PersistentFlags().Bool("managed", false, "enable managed mode") oneCommand.PersistentFlags().BoolP("version", "v", false, "gorse version") - oneCommand.PersistentFlags().String("log-path", "", "path of log file") oneCommand.PersistentFlags().Bool("playground", false, "playground mode (setup a recommender system for GitHub repositories)") oneCommand.PersistentFlags().StringP("config", "c", "", "configuration file path") oneCommand.PersistentFlags().String("cache-path", "one_cache.data", "path of cache file") diff --git a/cmd/gorse-master/main.go b/cmd/gorse-master/main.go index 97fc47d28..2f3cbbe66 100644 --- a/cmd/gorse-master/main.go +++ b/cmd/gorse-master/main.go @@ -37,17 +37,9 @@ var masterCommand = &cobra.Command{ return } // setup logger - var outputPaths []string - if cmd.PersistentFlags().Changed("log-path") { - outputPath, _ := cmd.PersistentFlags().GetString("log-path") - outputPaths = append(outputPaths, outputPath) - } - debugMode, _ := cmd.PersistentFlags().GetBool("debug") - if debugMode { - log.SetDevelopmentLogger(outputPaths...) - } else { - log.SetProductionLogger(outputPaths...) - } + debug, _ := cmd.PersistentFlags().GetBool("debug") + log.SetLogger(cmd.PersistentFlags(), debug) + // Create master configPath, _ := cmd.PersistentFlags().GetString("config") log.Logger().Info("load config", zap.String("config", configPath)) @@ -75,11 +67,11 @@ var masterCommand = &cobra.Command{ } func init() { + log.AddFlags(masterCommand.PersistentFlags()) masterCommand.PersistentFlags().Bool("debug", false, "use debug log mode") masterCommand.PersistentFlags().Bool("managed", false, "enable managed mode") masterCommand.PersistentFlags().StringP("config", "c", "", "configuration file path") masterCommand.PersistentFlags().BoolP("version", "v", false, "gorse version") - masterCommand.PersistentFlags().String("log-path", "", "path of log file") masterCommand.PersistentFlags().String("cache-path", "master_cache.data", "path of cache file") } diff --git a/cmd/gorse-server/main.go b/cmd/gorse-server/main.go index d7054d2fe..736ad434b 100644 --- a/cmd/gorse-server/main.go +++ b/cmd/gorse-server/main.go @@ -37,17 +37,8 @@ var serverCommand = &cobra.Command{ } // setup logger - var outputPaths []string - if cmd.PersistentFlags().Changed("log-path") { - outputPath, _ := cmd.PersistentFlags().GetString("log-path") - outputPaths = append(outputPaths, outputPath) - } - debugMode, _ := cmd.PersistentFlags().GetBool("debug") - if debugMode { - log.SetDevelopmentLogger(outputPaths...) - } else { - log.SetProductionLogger(outputPaths...) - } + debug, _ := cmd.PersistentFlags().GetBool("debug") + log.SetLogger(cmd.PersistentFlags(), debug) // create server masterPort, _ := cmd.PersistentFlags().GetInt("master-port") @@ -75,13 +66,13 @@ var serverCommand = &cobra.Command{ } func init() { + log.AddFlags(serverCommand.PersistentFlags()) serverCommand.PersistentFlags().BoolP("version", "v", false, "gorse version") serverCommand.PersistentFlags().Int("master-port", 8086, "port of master node") serverCommand.PersistentFlags().String("master-host", "127.0.0.1", "host of master node") serverCommand.PersistentFlags().Int("http-port", 8087, "host for RESTful APIs and Prometheus metrics export") serverCommand.PersistentFlags().String("http-host", "127.0.0.1", "port for RESTful APIs and Prometheus metrics export") serverCommand.PersistentFlags().Bool("debug", false, "use debug log mode") - serverCommand.PersistentFlags().String("log-path", "", "path of log file") serverCommand.PersistentFlags().String("cache-path", "server_cache.data", "path of cache file") } diff --git a/cmd/gorse-worker/main.go b/cmd/gorse-worker/main.go index fac3d2d36..9769138e6 100644 --- a/cmd/gorse-worker/main.go +++ b/cmd/gorse-worker/main.go @@ -41,17 +41,8 @@ var workerCommand = &cobra.Command{ workingJobs, _ := cmd.PersistentFlags().GetInt("jobs") managedModel, _ := cmd.PersistentFlags().GetBool("managed") // setup logger - var outputPaths []string - if cmd.PersistentFlags().Changed("log-path") { - outputPath, _ := cmd.PersistentFlags().GetString("log-path") - outputPaths = append(outputPaths, outputPath) - } - debugMode, _ := cmd.PersistentFlags().GetBool("debug") - if debugMode { - log.SetDevelopmentLogger(outputPaths...) - } else { - log.SetProductionLogger(outputPaths...) - } + debug, _ := cmd.PersistentFlags().GetBool("debug") + log.SetLogger(cmd.PersistentFlags(), debug) // create worker cachePath, _ := cmd.PersistentFlags().GetString("cache-path") w := worker.NewWorker(masterHost, masterPort, httpHost, httpPort, workingJobs, cachePath, managedModel) @@ -60,6 +51,7 @@ var workerCommand = &cobra.Command{ } func init() { + log.AddFlags(workerCommand.PersistentFlags()) workerCommand.PersistentFlags().BoolP("version", "v", false, "gorse version") workerCommand.PersistentFlags().String("master-host", "127.0.0.1", "host of master node") workerCommand.PersistentFlags().Int("master-port", 8086, "port of master node") @@ -68,7 +60,6 @@ func init() { workerCommand.PersistentFlags().Bool("debug", false, "use debug log mode") workerCommand.PersistentFlags().Bool("managed", false, "enable managed mode") workerCommand.PersistentFlags().IntP("jobs", "j", 1, "number of working jobs.") - workerCommand.PersistentFlags().String("log-path", "", "path of log file") workerCommand.PersistentFlags().String("cache-path", "worker_cache.data", "path of cache file") } diff --git a/go.mod b/go.mod index f19bb81aa..4566194aa 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/scylladb/go-set v1.0.2 github.com/sijms/go-ora/v2 v2.4.27 github.com/spf13/cobra v1.5.0 + github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 github.com/steinfletcher/apitest v1.5.14 github.com/stretchr/testify v1.8.0 @@ -58,10 +59,11 @@ require ( go.opentelemetry.io/otel/sdk v1.11.1 go.opentelemetry.io/otel/trace v1.11.1 go.uber.org/atomic v1.10.0 - go.uber.org/zap v1.22.0 + go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 gorm.io/driver/clickhouse v0.4.2 gorm.io/driver/mysql v1.3.4 @@ -136,7 +138,6 @@ require ( github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect @@ -146,7 +147,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect go.opentelemetry.io/otel/metric v0.33.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.8.0 // indirect + go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.7.0 // indirect diff --git a/go.sum b/go.sum index 6be5e818f..cc6b67d08 100644 --- a/go.sum +++ b/go.sum @@ -640,15 +640,15 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0= -go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -1056,6 +1056,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=