Add 'log-format' CLI flag, along with associated config flag, for 'va…

…ult server' command. (#6840)

* Read config before creating logger when booting vault server

* Allow for specifying log output in JSON format in a config file, via a 'log_level' flag

* Create parser for log format flag

* Allow for specifying log format in a config file, via a 'log_format' flag. Also, get rid of 'log_json' flag.

* Add 'log-format' command line flag

* Update documentation to include description of log_format setting

* Tweak comment for VAULT_LOG_FORMAT environment variable

* add test for ParseEnvLogFormat()

* clarify how log format is set

* fix typos in documentation

command/server.go

@@ -89,6 +89,7 @@ type ServerCommand struct {
 	// new stuff
 	flagConfigs        []string
 	flagLogLevel       string
+	flagLogFormat      string
 	flagDev            bool
 	flagDevRootTokenID string
 	flagDevListenAddr  string
@@ -175,6 +176,17 @@ func (c *ServerCommand) Flags() *FlagSets {
 			"\"trace\", \"debug\", \"info\", \"warn\", and \"err\".",
 	})
 
+	f.StringVar(&StringVar{
+		Name:    "log-format",
+		Target:  &c.flagLogFormat,
+		Default: notSetValue,
+		// EnvVar can't be just "VAULT_LOG_FORMAT", because more than one env var name is supported
+		// for backwards compatibility reasons.
+		// See github.com/hashicorp/vault/sdk/helper/logging.ParseEnvLogFormat()
+		Completion: complete.PredictSet("standard", "json"),
+		Usage:      `Log format. Supported values are "standard" and "json".`,
+	})
+
 	f = set.NewFlagSet("Dev Options")
 
 	f.BoolVar(&BoolVar{
@@ -343,49 +355,6 @@ func (c *ServerCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Create a logger. We wrap it in a gated writer so that it doesn't
-	// start logging too early.
-	c.logGate = &gatedwriter.Writer{Writer: os.Stderr}
-	c.logWriter = c.logGate
-	if c.flagCombineLogs {
-		c.logWriter = os.Stdout
-	}
-	var level log.Level
-	var logLevelWasNotSet bool
-	logLevelString := c.flagLogLevel
-	c.flagLogLevel = strings.ToLower(strings.TrimSpace(c.flagLogLevel))
-	switch c.flagLogLevel {
-	case notSetValue, "":
-		logLevelWasNotSet = true
-		logLevelString = "info"
-		level = log.Info
-	case "trace":
-		level = log.Trace
-	case "debug":
-		level = log.Debug
-	case "notice", "info":
-		level = log.Info
-	case "warn", "warning":
-		level = log.Warn
-	case "err", "error":
-		level = log.Error
-	default:
-		c.UI.Error(fmt.Sprintf("Unknown log level: %s", c.flagLogLevel))
-		return 1
-	}
-
-	if c.flagDevThreeNode || c.flagDevFourCluster {
-		c.logger = log.New(&log.LoggerOptions{
-			Mutex:  &sync.Mutex{},
-			Output: c.logWriter,
-			Level:  log.Trace,
-		})
-	} else {
-		c.logger = logging.NewVaultLoggerWithWriter(c.logWriter, level)
-	}
-
-	allLoggers := []log.Logger{c.logger}
-
 	// Automatically enable dev mode if other dev flags are provided.
 	if c.flagDevHA || c.flagDevTransactional || c.flagDevLeasedKV || c.flagDevThreeNode || c.flagDevFourCluster || c.flagDevAutoSeal || c.flagDevKVV1 {
 		c.flagDev = true
@@ -427,7 +396,7 @@ func (c *ServerCommand) Run(args []string) int {
 		}
 	}
 	for _, path := range c.flagConfigs {
-		current, err := server.LoadConfig(path, c.logger)
+		current, err := server.LoadConfig(path)
 		if err != nil {
 			c.UI.Error(fmt.Sprintf("Error loading configuration from %s: %s", path, err))
 			return 1
@@ -450,6 +419,77 @@ func (c *ServerCommand) Run(args []string) int {
 		return 1
 	}
 
+	// Create a logger. We wrap it in a gated writer so that it doesn't
+	// start logging too early.
+	c.logGate = &gatedwriter.Writer{Writer: os.Stderr}
+	c.logWriter = c.logGate
+	if c.flagCombineLogs {
+		c.logWriter = os.Stdout
+	}
+	var level log.Level
+	var logLevelWasNotSet bool
+	logLevelString := c.flagLogLevel
+	c.flagLogLevel = strings.ToLower(strings.TrimSpace(c.flagLogLevel))
+	switch c.flagLogLevel {
+	case notSetValue, "":
+		logLevelWasNotSet = true
+		logLevelString = "info"
+		level = log.Info
+	case "trace":
+		level = log.Trace
+	case "debug":
+		level = log.Debug
+	case "notice", "info":
+		level = log.Info
+	case "warn", "warning":
+		level = log.Warn
+	case "err", "error":
+		level = log.Error
+	default:
+		c.UI.Error(fmt.Sprintf("Unknown log level: %s", c.flagLogLevel))
+		return 1
+	}
+
+	logFormat := logging.UnspecifiedFormat
+	if c.flagLogFormat != notSetValue {
+		var err error
+		logFormat, err = logging.ParseLogFormat(c.flagLogFormat)
+		if err != nil {
+			c.UI.Error(err.Error())
+			return 1
+		}
+	}
+	if logFormat == logging.UnspecifiedFormat {
+		logFormat = logging.ParseEnvLogFormat()
+	}
+	if logFormat == logging.UnspecifiedFormat {
+		var err error
+		logFormat, err = logging.ParseLogFormat(config.LogFormat)
+		if err != nil {
+			c.UI.Error(err.Error())
+			return 1
+		}
+	}
+
+	if c.flagDevThreeNode || c.flagDevFourCluster {
+		c.logger = log.New(&log.LoggerOptions{
+			Mutex:  &sync.Mutex{},
+			Output: c.logWriter,
+			Level:  log.Trace,
+		})
+	} else {
+		c.logger = log.New(&log.LoggerOptions{
+			Output: c.logWriter,
+			Level:  level,
+			// Note that if logFormat is either unspecified or standard, then
+			// the resulting logger's format will be standard.
+			JSONFormat: logFormat == logging.JSONFormat,
+		})
+	}
+
+	allLoggers := []log.Logger{c.logger}
+
+	// adjust log level based on config setting
 	if config.LogLevel != "" && logLevelWasNotSet {
 		configLogLevel := strings.ToLower(strings.TrimSpace(config.LogLevel))
 		logLevelString = configLogLevel
@@ -470,6 +510,7 @@ func (c *ServerCommand) Run(args []string) int {
 		}
 	}
 
+	// create GRPC logger
 	namedGRPCLogFaker := c.logger.Named("grpclogfaker")
 	allLoggers = append(allLoggers, namedGRPCLogFaker)
 	grpclog.SetLogger(&grpclogFaker{
@@ -1310,7 +1351,7 @@ CLUSTER_SYNTHESIS_COMPLETE:
 			var config *server.Config
 			var level log.Level
 			for _, path := range c.flagConfigs {
-				current, err := server.LoadConfig(path, c.logger)
+				current, err := server.LoadConfig(path)
 				if err != nil {
 					c.logger.Error("could not reload config", "path", path, "error", err)
 					goto RUNRELOADFUNCS

command/server/config.go

@@ -12,9 +12,8 @@ import (
 	"time"
 
 	"github.com/hashicorp/errwrap"
-	log "github.com/hashicorp/go-hclog"
 
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/hcl/hcl/ast"
 	"github.com/hashicorp/vault/sdk/helper/parseutil"
@@ -60,6 +59,10 @@ type Config struct {
 
 	LogLevel string `hcl:"log_level"`
 
+	// LogFormat specifies the log format.  Valid values are "standard" and "json".  The values are case-insenstive.
+	// If no log format is specified, then standard format will be used.
+	LogFormat string `hcl:"log_format"`
+
 	PidFile              string      `hcl:"pid_file"`
 	EnableRawEndpoint    bool        `hcl:"-"`
 	EnableRawEndpointRaw interface{} `hcl:"raw_storage_endpoint"`
@@ -322,6 +325,11 @@ func (c *Config) Merge(c2 *Config) *Config {
 		result.LogLevel = c2.LogLevel
 	}
 
+	result.LogFormat = c.LogFormat
+	if c2.LogFormat != "" {
+		result.LogFormat = c2.LogFormat
+	}
+
 	result.ClusterName = c.ClusterName
 	if c2.ClusterName != "" {
 		result.ClusterName = c2.ClusterName
@@ -415,29 +423,29 @@ func (c *Config) Merge(c2 *Config) *Config {
 
 // LoadConfig loads the configuration at the given path, regardless if
 // its a file or directory.
-func LoadConfig(path string, logger log.Logger) (*Config, error) {
+func LoadConfig(path string) (*Config, error) {
 	fi, err := os.Stat(path)
 	if err != nil {
 		return nil, err
 	}
 
 	if fi.IsDir() {
-		return LoadConfigDir(path, logger)
+		return LoadConfigDir(path)
 	}
-	return LoadConfigFile(path, logger)
+	return LoadConfigFile(path)
 }
 
 // LoadConfigFile loads the configuration from the given file.
-func LoadConfigFile(path string, logger log.Logger) (*Config, error) {
+func LoadConfigFile(path string) (*Config, error) {
 	// Read the file
 	d, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
-	return ParseConfig(string(d), logger)
+	return ParseConfig(string(d))
 }
 
-func ParseConfig(d string, logger log.Logger) (*Config, error) {
+func ParseConfig(d string) (*Config, error) {
 	// Parse!
 	obj, err := hcl.Parse(d)
 	if err != nil {
@@ -580,7 +588,7 @@ func ParseConfig(d string, logger log.Logger) (*Config, error) {
 
 // LoadConfigDir loads all the configurations in the given directory
 // in alphabetical order.
-func LoadConfigDir(dir string, logger log.Logger) (*Config, error) {
+func LoadConfigDir(dir string) (*Config, error) {
 	f, err := os.Open(dir)
 	if err != nil {
 		return nil, err
@@ -629,7 +637,7 @@ func LoadConfigDir(dir string, logger log.Logger) (*Config, error) {
 
 	var result *Config
 	for _, f := range files {
-		config, err := LoadConfigFile(f, logger)
+		config, err := LoadConfigFile(f)
 		if err != nil {
 			return nil, errwrap.Wrapf(fmt.Sprintf("error loading %q: {{err}}", f), err)
 		}

sdk/helper/logging/logging.go

@@ -0,0 +1,81 @@
+package logging
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	log "github.com/hashicorp/go-hclog"
+)
+
+type LogFormat int
+
+const (
+	UnspecifiedFormat LogFormat = iota
+	StandardFormat
+	JSONFormat
+)
+
+// Stringer implementation
+func (l LogFormat) String() string {
+	switch l {
+	case UnspecifiedFormat:
+		return "unspecified"
+	case StandardFormat:
+		return "standard"
+	case JSONFormat:
+		return "json"
+	}
+
+	// unreachable
+	return "unknown"
+}
+
+// NewVaultLogger creates a new logger with the specified level and a Vault
+// formatter
+func NewVaultLogger(level log.Level) log.Logger {
+	return NewVaultLoggerWithWriter(log.DefaultOutput, level)
+}
+
+// NewVaultLoggerWithWriter creates a new logger with the specified level and
+// writer and a Vault formatter
+func NewVaultLoggerWithWriter(w io.Writer, level log.Level) log.Logger {
+	opts := &log.LoggerOptions{
+		Level:      level,
+		Output:     w,
+		JSONFormat: ParseEnvLogFormat() == JSONFormat,
+	}
+	return log.New(opts)
+}
+
+// ParseLogFormat parses the log format from the provided string.
+func ParseLogFormat(format string) (LogFormat, error) {
+
+	switch strings.ToLower(strings.TrimSpace(format)) {
+	case "":
+		return UnspecifiedFormat, nil
+	case "standard":
+		return StandardFormat, nil
+	case "json":
+		return JSONFormat, nil
+	default:
+		return UnspecifiedFormat, fmt.Errorf("Unknown log format: %s", format)
+	}
+}
+
+// ParseEnvLogFormat parses the log format from an environment variable.
+func ParseEnvLogFormat() LogFormat {
+	logFormat := os.Getenv("VAULT_LOG_FORMAT")
+	if logFormat == "" {
+		logFormat = os.Getenv("LOGXI_FORMAT")
+	}
+	switch strings.ToLower(logFormat) {
+	case "json", "vault_json", "vault-json", "vaultjson":
+		return JSONFormat
+	case "standard":
+		return StandardFormat
+	default:
+		return UnspecifiedFormat
+	}
+}