From fb3f2d7cf56535b5644ee94cb681c87aad6c0a4b Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Fri, 24 Jan 2020 16:35:42 +0100 Subject: [PATCH] Introduce CLI flag -environment (#15422) * Introduce CLI flag -environment Introduce CLI flags -environment to control the default logging settings the Beat should use if no logging is configured. The behavior of -e does not change. By replacing `-e` with `-environment system` in the system unit file we continue to log to stdout/stderr by default, but users are still able to overwrite settings. * Add more environment types Add windows_service, macos_service, and container envionment types. If beats are installed using our scripts the `-environment` flag will always be set. Replace `-e` with `-environment docker` CLI flag in the Dockerfile. Although it's uncommon, users can not overwrite the logging configuration in docker containers as well. The docker(container) environment and systemd environment will default to stdout/stderr logging. All other environments continue to use file rotation as default. (cherry picked from commit 1d3d4d1ab0b29a707d605bab3270664c91397ead) --- CHANGELOG.next.asciidoc | 3 + .../darwin/launchd-daemon.plist.tmpl | 36 ++++---- .../templates/docker/Dockerfile.tmpl | 2 +- .../templates/linux/systemd.unit.tmpl | 4 +- .../windows/install-service.ps1.tmpl | 2 +- libbeat/cmd/root.go | 1 + libbeat/docs/command-reference.asciidoc | 7 ++ libbeat/docs/shared-systemd.asciidoc | 17 +--- libbeat/logp/config.go | 51 ++++++++---- libbeat/logp/configure/logging.go | 21 ++++- libbeat/logp/environment.go | 82 +++++++++++++++++++ 11 files changed, 174 insertions(+), 52 deletions(-) create mode 100644 libbeat/logp/environment.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3e8f5954c032..5871bfc145f6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,8 +60,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix spooling to disk blocking infinitely if the lock file can not be acquired. {pull}15338[15338] - Fix `metricbeat test output` with an ipv6 ES host in the output.hosts. {pull}15368[15368] - Fix `convert` processor conversion of string to integer with leading zeros. {issue}15513[15513] {pull}15557[15557] +- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568] - Fix missing output in dockerlogbeat {pull}15719[15719] +- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442] *Auditbeat* diff --git a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl index 683188534a3b..b98e9081172e 100644 --- a/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl +++ b/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl @@ -2,23 +2,25 @@ - Label + Label {{.identifier}} - ProgramArguments - - {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} - -c - /etc/{{.BeatName}}/{{.BeatName}}.yml - --path.home - {{.install_path}}/{{.BeatVendor}}/{{.BeatName}} - --path.config - /etc/{{.BeatName}} - --path.data - /var/lib/{{.BeatName}} - --path.logs - /var/log/{{.BeatName}} - - RunAtLoad - + ProgramArguments + + {{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}} + -environment + macOS_service + -c + /etc/{{.BeatName}}/{{.BeatName}}.yml + --path.home + {{.install_path}}/{{.BeatVendor}}/{{.BeatName}} + --path.config + /etc/{{.BeatName}} + --path.data + /var/lib/{{.BeatName}} + --path.logs + /var/log/{{.BeatName}} + + RunAtLoad + diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 0fae61cdbdfc..bd5eb0f3c1a2 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -50,4 +50,4 @@ EXPOSE {{ $port }} WORKDIR {{ $beatHome }} ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] -CMD ["-e"] +CMD ["-environment", "container"] diff --git a/dev-tools/packaging/templates/linux/systemd.unit.tmpl b/dev-tools/packaging/templates/linux/systemd.unit.tmpl index 7f27459b4654..10a56c0b5900 100644 --- a/dev-tools/packaging/templates/linux/systemd.unit.tmpl +++ b/dev-tools/packaging/templates/linux/systemd.unit.tmpl @@ -9,10 +9,10 @@ After=network-online.target User={{ .BeatUser }} Group={{ .BeatUser }} {{- end }} -Environment="BEAT_LOG_OPTS=-e" +Environment="BEAT_LOG_OPTS=" Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml" Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}" -ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS +ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} -environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS Restart=always [Install] diff --git a/dev-tools/packaging/templates/windows/install-service.ps1.tmpl b/dev-tools/packaging/templates/windows/install-service.ps1.tmpl index cab1373ef15f..c398e18d3333 100644 --- a/dev-tools/packaging/templates/windows/install-service.ps1.tmpl +++ b/dev-tools/packaging/templates/windows/install-service.ps1.tmpl @@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path # Create the new service. New-Service -name {{.BeatName}} ` -displayName {{.BeatName | title}} ` - -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true" + -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true" # Attempt to set the service to delayed start using sc config. Try { diff --git a/libbeat/cmd/root.go b/libbeat/cmd/root.go index e76a7ae9d909..94382eed5bae 100644 --- a/libbeat/cmd/root.go +++ b/libbeat/cmd/root.go @@ -88,6 +88,7 @@ func GenRootCmdWithSettings(beatCreator beat.Creator, settings instance.Settings rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("e")) + rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("environment")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data")) rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs")) diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index 30af1d7f44d1..6675c35588cc 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -918,6 +918,13 @@ messages. *`-e, --e`*:: Logs to stderr and disables syslog/file output. +*`-environment`*:: +For logging purposes, specifies the environment that {beatname_uc} is running in. +This setting is used to select a default log output when no log output is configured. +Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`. +If `systemd` or `container` is specified, {beatname_uc} will log to stdout and stderr +by default. + *`--path.config`*:: Sets the path for configuration files. See the <> section for details. diff --git a/libbeat/docs/shared-systemd.asciidoc b/libbeat/docs/shared-systemd.asciidoc index cf028b9573f2..252018cb11c4 100644 --- a/libbeat/docs/shared-systemd.asciidoc +++ b/libbeat/docs/shared-systemd.asciidoc @@ -49,10 +49,6 @@ Logs are stored by default in journald. To view the Logs, use `journalctl`: journalctl -u {beatname_lc}.service ------------------------------------------------ -NOTE: The unit file included in the packages sets the `-e` flag by default. -This flag makes {beatname_uc} log to stderr and disables other log outputs. -Systemd stores all output sent to stderr in journald. - [float] === Customize systemd unit for {beatname_uc} @@ -62,7 +58,7 @@ override to change the default options. [cols="" + } +} + +// ParseEnvironment returns the environment type by name. +// The parse is case insensitive. +// InvalidEnvironment is returned if the environment type is unknown. +func ParseEnvironment(in string) Environment { + switch strings.ToLower(in) { + case "default": + return DefaultEnvironment + case "systemd": + return SystemdEnvironment + case "container": + return ContainerEnvironment + case "macos_service": + return MacOSServiceEnvironment + case "windows_service": + return WindowsServiceEnvironment + default: + return InvalidEnvironment + } +}