support log files rotation (#659)

(cherry picked from commit a2f2c5a)

base/log/log.go

@@ -17,22 +17,27 @@ package log
 import (
 	"net/url"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 
 	"github.com/emicklei/go-restful/v3"
 	"github.com/go-sql-driver/mysql"
-	"github.com/samber/lo"
+	"github.com/spf13/pflag"
 	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 var logger *zap.Logger
 
 func init() {
 	// setup default logger
-	SetProductionLogger()
+	var err error
+	logger, err = zap.NewProduction()
+	if err != nil {
+		panic(err)
+	}
 	// Windows file sink support: https://github.com/uber-go/zap/issues/621
 	if runtime.GOOS == "windows" {
 		if err := zap.RegisterSink("windows", func(u *url.URL) (zap.Sink, error) {
@@ -53,54 +58,54 @@ func ResponseLogger(resp *restful.Response) *zap.Logger {
 	return logger.With(zap.String("request_id", resp.Header().Get("X-Request-ID")))
 }
 
-// SetProductionLogger set current logger in production mode.
-func SetProductionLogger(outputPaths ...string) {
-	for _, outputPath := range outputPaths {
-		err := os.MkdirAll(filepath.Dir(outputPath), os.ModePerm)
-		if err != nil {
-			panic(err)
-		}
-	}
+func CloseLogger() {
 	cfg := zap.NewProductionConfig()
-	if runtime.GOOS == "windows" {
-		cfg.OutputPaths = append(cfg.OutputPaths, lo.Map(outputPaths, func(path string, _ int) string {
-			return "windows:///" + path
-		})...)
-	} else {
-		cfg.OutputPaths = append(cfg.OutputPaths, outputPaths...)
-	}
+	cfg.Level = zap.NewAtomicLevelAt(zap.FatalLevel)
 	var err error
 	logger, err = cfg.Build()
 	if err != nil {
 		panic(err)
 	}
 }
 
-// SetDevelopmentLogger set current logger in development mode.
-func SetDevelopmentLogger(outputPaths ...string) {
-	for _, outputPath := range outputPaths {
-		err := os.MkdirAll(filepath.Dir(outputPath), os.ModePerm)
-		if err != nil {
-			panic(err)
-		}
-	}
-	cfg := zap.NewDevelopmentConfig()
-	cfg.OutputPaths = append(cfg.OutputPaths, outputPaths...)
-	var err error
-	logger, err = cfg.Build()
-	if err != nil {
-		panic(err)
-	}
+func AddFlags(flagSet *pflag.FlagSet) {
+	flagSet.String("log-path", "", "path of log file")
+	flagSet.Int("log-max-size", 100, "maximum size in megabytes of the log file")
+	flagSet.Int("log-max-age", 0, "maximum number of days to retain old log files")
+	flagSet.Int("log-max-backups", 0, "maximum number of old log files to retain")
 }
 
-func CloseLogger() {
-	cfg := zap.NewProductionConfig()
-	cfg.Level = zap.NewAtomicLevelAt(zap.FatalLevel)
-	var err error
-	logger, err = cfg.Build()
-	if err != nil {
-		panic(err)
+func SetLogger(flagSet *pflag.FlagSet, debug bool) {
+	// enable or disable debug mode
+	var (
+		encoder zapcore.Encoder
+		level   zapcore.LevelEnabler
+	)
+	if debug {
+		encoder = zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
+		level = zap.DebugLevel
+	} else {
+		encoder = zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
+		level = zap.InfoLevel
+	}
+	// create lumberjack logger
+	writers := []zapcore.WriteSyncer{zapcore.AddSync(os.Stdout)}
+	if flagSet.Changed("log-path") {
+		path, _ := flagSet.GetString("log-path")
+		maxSize, _ := flagSet.GetInt("log-max-size")
+		maxAge, _ := flagSet.GetInt("log-max-age")
+		maxBackups, _ := flagSet.GetInt("log-max-backups")
+		writers = append(writers, zapcore.AddSync(&lumberjack.Logger{
+			Filename:   path,
+			MaxSize:    maxSize,
+			MaxBackups: maxBackups,
+			MaxAge:     maxAge,
+			Compress:   false,
+		}))
 	}
+	// create zap logger
+	core := zapcore.NewCore(encoder, zap.CombineWriteSyncers(writers...), level)
+	logger = zap.New(core)
 }
 
 const mysqlPrefix = "mysql://"

base/log/log_test.go

@@ -15,56 +15,46 @@
 package log
 
 import (
-	"github.com/stretchr/testify/assert"
-	"os"
-	"runtime"
 	"testing"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestSetDevelopmentLogger(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	temp, err := os.MkdirTemp("", "test_gorse")
-	assert.NoError(t, err)
+	temp := t.TempDir()
+	flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError)
+	AddFlags(flagSet)
 	// set existed path
-	SetDevelopmentLogger(temp + "/gorse.log")
-	_, err = os.Stat(temp + "/gorse.log")
+	err := flagSet.Set("log-path", temp+"/gorse.log")
 	assert.NoError(t, err)
+	SetLogger(flagSet, true)
+	Logger().Debug("test")
+	assert.FileExists(t, temp+"/gorse.log")
 	// set non-existed path
-	SetDevelopmentLogger(temp + "/gorse/gorse.log")
-	_, err = os.Stat(temp + "/gorse/gorse.log")
+	err = flagSet.Set("log-path", temp+"/gorse/gorse.log")
 	assert.NoError(t, err)
-	// permission denied
-	assert.Panics(t, func() {
-		SetDevelopmentLogger("/gorse.log")
-	})
-	assert.Panics(t, func() {
-		SetDevelopmentLogger("/gorse/gorse.log")
-	})
+	SetLogger(flagSet, true)
+	Logger().Debug("test")
+	assert.FileExists(t, temp+"/gorse/gorse.log")
 }
 
 func TestSetProductionLogger(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	temp, err := os.MkdirTemp("", "test_gorse")
-	assert.NoError(t, err)
+	temp := t.TempDir()
+	flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError)
+	AddFlags(flagSet)
 	// set existed path
-	SetProductionLogger(temp + "/gorse.log")
-	_, err = os.Stat(temp + "/gorse.log")
+	err := flagSet.Set("log-path", temp+"/gorse.log")
 	assert.NoError(t, err)
+	SetLogger(flagSet, false)
+	Logger().Info("test")
+	assert.FileExists(t, temp+"/gorse.log")
 	// set non-existed path
-	SetProductionLogger(temp + "/gorse/gorse.log")
-	_, err = os.Stat(temp + "/gorse/gorse.log")
+	err = flagSet.Set("log-path", temp+"/gorse/gorse.log")
 	assert.NoError(t, err)
-	// permission denied
-	assert.Panics(t, func() {
-		SetProductionLogger("/gorse.log")
-	})
-	assert.Panics(t, func() {
-		SetProductionLogger("/gorse/gorse.log")
-	})
+	SetLogger(flagSet, false)
+	Logger().Info("test")
+	assert.FileExists(t, temp+"/gorse/gorse.log")
 }
 
 func TestRedactDBURL(t *testing.T) {

cmd/gorse-in-one/main.go

@@ -58,17 +58,8 @@ var oneCommand = &cobra.Command{
 		}
 
 		// setup logger
-		var outputPaths []string
-		if cmd.PersistentFlags().Changed("log-path") {
-			outputPath, _ := cmd.PersistentFlags().GetString("log-path")
-			outputPaths = append(outputPaths, outputPath)
-		}
-		debugMode, _ := cmd.PersistentFlags().GetBool("debug")
-		if debugMode {
-			log.SetDevelopmentLogger(outputPaths...)
-		} else {
-			log.SetProductionLogger(outputPaths...)
-		}
+		debug, _ := cmd.PersistentFlags().GetBool("debug")
+		log.SetLogger(cmd.PersistentFlags(), debug)
 
 		// load config
 		var conf *config.Config
@@ -135,10 +126,10 @@ var oneCommand = &cobra.Command{
 }
 
 func init() {
+	log.AddFlags(oneCommand.PersistentFlags())
 	oneCommand.PersistentFlags().Bool("debug", false, "use debug log mode")
 	oneCommand.PersistentFlags().Bool("managed", false, "enable managed mode")
 	oneCommand.PersistentFlags().BoolP("version", "v", false, "gorse version")
-	oneCommand.PersistentFlags().String("log-path", "", "path of log file")
 	oneCommand.PersistentFlags().Bool("playground", false, "playground mode (setup a recommender system for GitHub repositories)")
 	oneCommand.PersistentFlags().StringP("config", "c", "", "configuration file path")
 	oneCommand.PersistentFlags().String("cache-path", "one_cache.data", "path of cache file")

cmd/gorse-master/main.go

@@ -37,17 +37,9 @@ var masterCommand = &cobra.Command{
 			return
 		}
 		// setup logger
-		var outputPaths []string
-		if cmd.PersistentFlags().Changed("log-path") {
-			outputPath, _ := cmd.PersistentFlags().GetString("log-path")
-			outputPaths = append(outputPaths, outputPath)
-		}
-		debugMode, _ := cmd.PersistentFlags().GetBool("debug")
-		if debugMode {
-			log.SetDevelopmentLogger(outputPaths...)
-		} else {
-			log.SetProductionLogger(outputPaths...)
-		}
+		debug, _ := cmd.PersistentFlags().GetBool("debug")
+		log.SetLogger(cmd.PersistentFlags(), debug)
+
 		// Create master
 		configPath, _ := cmd.PersistentFlags().GetString("config")
 		log.Logger().Info("load config", zap.String("config", configPath))
@@ -75,11 +67,11 @@ var masterCommand = &cobra.Command{
 }
 
 func init() {
+	log.AddFlags(masterCommand.PersistentFlags())
 	masterCommand.PersistentFlags().Bool("debug", false, "use debug log mode")
 	masterCommand.PersistentFlags().Bool("managed", false, "enable managed mode")
 	masterCommand.PersistentFlags().StringP("config", "c", "", "configuration file path")
 	masterCommand.PersistentFlags().BoolP("version", "v", false, "gorse version")
-	masterCommand.PersistentFlags().String("log-path", "", "path of log file")
 	masterCommand.PersistentFlags().String("cache-path", "master_cache.data", "path of cache file")
 }
 

cmd/gorse-server/main.go

@@ -37,17 +37,8 @@ var serverCommand = &cobra.Command{
 		}
 
 		// setup logger
-		var outputPaths []string
-		if cmd.PersistentFlags().Changed("log-path") {
-			outputPath, _ := cmd.PersistentFlags().GetString("log-path")
-			outputPaths = append(outputPaths, outputPath)
-		}
-		debugMode, _ := cmd.PersistentFlags().GetBool("debug")
-		if debugMode {
-			log.SetDevelopmentLogger(outputPaths...)
-		} else {
-			log.SetProductionLogger(outputPaths...)
-		}
+		debug, _ := cmd.PersistentFlags().GetBool("debug")
+		log.SetLogger(cmd.PersistentFlags(), debug)
 
 		// create server
 		masterPort, _ := cmd.PersistentFlags().GetInt("master-port")
@@ -75,13 +66,13 @@ var serverCommand = &cobra.Command{
 }
 
 func init() {
+	log.AddFlags(serverCommand.PersistentFlags())
 	serverCommand.PersistentFlags().BoolP("version", "v", false, "gorse version")
 	serverCommand.PersistentFlags().Int("master-port", 8086, "port of master node")
 	serverCommand.PersistentFlags().String("master-host", "127.0.0.1", "host of master node")
 	serverCommand.PersistentFlags().Int("http-port", 8087, "host for RESTful APIs and Prometheus metrics export")
 	serverCommand.PersistentFlags().String("http-host", "127.0.0.1", "port for RESTful APIs and Prometheus metrics export")
 	serverCommand.PersistentFlags().Bool("debug", false, "use debug log mode")
-	serverCommand.PersistentFlags().String("log-path", "", "path of log file")
 	serverCommand.PersistentFlags().String("cache-path", "server_cache.data", "path of cache file")
 }
 

cmd/gorse-worker/main.go

@@ -41,17 +41,8 @@ var workerCommand = &cobra.Command{
 		workingJobs, _ := cmd.PersistentFlags().GetInt("jobs")
 		managedModel, _ := cmd.PersistentFlags().GetBool("managed")
 		// setup logger
-		var outputPaths []string
-		if cmd.PersistentFlags().Changed("log-path") {
-			outputPath, _ := cmd.PersistentFlags().GetString("log-path")
-			outputPaths = append(outputPaths, outputPath)
-		}
-		debugMode, _ := cmd.PersistentFlags().GetBool("debug")
-		if debugMode {
-			log.SetDevelopmentLogger(outputPaths...)
-		} else {
-			log.SetProductionLogger(outputPaths...)
-		}
+		debug, _ := cmd.PersistentFlags().GetBool("debug")
+		log.SetLogger(cmd.PersistentFlags(), debug)
 		// create worker
 		cachePath, _ := cmd.PersistentFlags().GetString("cache-path")
 		w := worker.NewWorker(masterHost, masterPort, httpHost, httpPort, workingJobs, cachePath, managedModel)
@@ -60,6 +51,7 @@ var workerCommand = &cobra.Command{
 }
 
 func init() {
+	log.AddFlags(workerCommand.PersistentFlags())
 	workerCommand.PersistentFlags().BoolP("version", "v", false, "gorse version")
 	workerCommand.PersistentFlags().String("master-host", "127.0.0.1", "host of master node")
 	workerCommand.PersistentFlags().Int("master-port", 8086, "port of master node")
@@ -68,7 +60,6 @@ func init() {
 	workerCommand.PersistentFlags().Bool("debug", false, "use debug log mode")
 	workerCommand.PersistentFlags().Bool("managed", false, "enable managed mode")
 	workerCommand.PersistentFlags().IntP("jobs", "j", 1, "number of working jobs.")
-	workerCommand.PersistentFlags().String("log-path", "", "path of log file")
 	workerCommand.PersistentFlags().String("cache-path", "worker_cache.data", "path of cache file")
 }
 

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/scylladb/go-set v1.0.2
 	github.com/sijms/go-ora/v2 v2.4.27
 	github.com/spf13/cobra v1.5.0
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
 	github.com/steinfletcher/apitest v1.5.14
 	github.com/stretchr/testify v1.8.0
@@ -58,10 +59,11 @@ require (
 	go.opentelemetry.io/otel/sdk v1.11.1
 	go.opentelemetry.io/otel/trace v1.11.1
 	go.uber.org/atomic v1.10.0
-	go.uber.org/zap v1.22.0
+	go.uber.org/zap v1.24.0
 	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
 	google.golang.org/grpc v1.50.1
 	google.golang.org/protobuf v1.28.1
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
 	gorm.io/driver/clickhouse v0.4.2
 	gorm.io/driver/mysql v1.3.4
@@ -136,7 +138,6 @@ require (
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
@@ -146,7 +147,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect
 	go.opentelemetry.io/otel/metric v0.33.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.7.0 // indirect