Improve shutdown logs (#1618)

(cherry picked from commit 72a4f61)

NOTICE.txt

@@ -1061,11 +1061,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.2.6
+Version: v0.2.14
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].6/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].14/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

changelog/fragments/1666789812-Improve-shutdown-logs.yaml

@@ -0,0 +1,31 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Improve shutdown logs
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: cmd, handler, upgrade
+
+# PR number; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: 1618
+
+# Issue number; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: 1358

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4
 	github.com/elastic/elastic-agent-autodiscover v0.2.1
 	github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6
-	github.com/elastic/elastic-agent-libs v0.2.6
+	github.com/elastic/elastic-agent-libs v0.2.14
 	github.com/elastic/elastic-agent-system-metrics v0.3.0
 	github.com/elastic/go-licenser v0.4.0
 	github.com/elastic/go-sysinfo v1.7.1

go.sum

@@ -384,8 +384,8 @@ github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+i
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
 github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
-github.com/elastic/elastic-agent-libs v0.2.6 h1:DpcUcCVYZ7lNtHLUlyT1u/GtGAh49wpL15DTH7+8O5o=
-github.com/elastic/elastic-agent-libs v0.2.6/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
+github.com/elastic/elastic-agent-libs v0.2.14 h1:o1agY/37TKl5kjhv3ur5M9d127wzQPRxwA4Xoh0jUEo=
+github.com/elastic/elastic-agent-libs v0.2.14/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss=
 github.com/elastic/elastic-agent-system-metrics v0.3.0 h1:W8L0E8lWJmdguH+oIR7OzuFgopvw8ucZAE9w6iqVlpE=
 github.com/elastic/elastic-agent-system-metrics v0.3.0/go.mod h1:RIYhJOS7mUeyIthfOSqmmbEILYSzaDWLi5zQ70bQo+o=
 github.com/elastic/elastic-package v0.32.1/go.mod h1:l1fEnF52XRBL6a5h6uAemtdViz2bjtjUtgdQcuRhEAY=

internal/pkg/agent/application/pipeline/actions/handlers/handler_action_settings.go

@@ -62,6 +62,7 @@ func (h *Settings) Handle(ctx context.Context, a fleetapi.Action, acker store.Fl
 		h.log.Errorf("failed to commit acker after acknowledging action with id '%s'", action.ActionID)
 	}
 
+	h.log.Info("SETTINGS action done, triggering agent restart")
 	h.reexec.ReExec(nil)
 	return nil
 }

internal/pkg/agent/application/upgrade/upgrade.go

@@ -200,8 +200,12 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree
 		if err != nil {
 			u.log.Errorw("Unable to clean downloads after update", "error.message", err, "downloads.path", paths.Downloads())
 		}
-		u.log.Infow("Restarting after upgrade", "new_version", release.Version(), "prev_version", a.Version(),
-			"hash", trimmedNewHash, "home", paths.Home())
+
+		u.log.Infow("Restarting after upgrade",
+			"new_version", release.Version(),
+			"prev_version", a.Version(),
+			"hash", trimmedNewHash,
+			"home", paths.Home())
 		u.reexec.ReExec(cb)
 		return nil, nil
 	}

internal/pkg/agent/cmd/common.go

@@ -68,7 +68,7 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 	cmd.AddCommand(newStatusCommand(args, streams))
 	cmd.AddCommand(newDiagnosticsCommand(args, streams))
 
-	// windows special hidden sub-command (only added on windows)
+	// windows special hidden sub-command (only added on Windows)
 	reexec := newReExecWindowsCommand(args, streams)
 	if reexec != nil {
 		cmd.AddCommand(reexec)

internal/pkg/agent/cmd/run.go

@@ -63,7 +63,14 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 		Short: "Start the elastic-agent.",
 		Run: func(_ *cobra.Command, _ []string) {
 			if err := run(nil); err != nil {
+				logp.NewLogger("cmd_run").
+					Errorw("run command finished with error",
+						"error.message", err)
 				fmt.Fprintf(streams.Err, "Error: %v\n%s\n", err, troubleshootMessage())
+
+				// TODO: remove it. os.Exit will be called on main and if it's called
+				// too early some goroutines with deferred functions related
+				// to the shutdown process might not run.
 				os.Exit(1)
 			}
 		},
@@ -237,13 +244,16 @@ func run(override cfgOverrider) error {
 		breakout := false
 		select {
 		case <-stop:
+			logger.Info("service.HandleSignals invoked stop function. Shutting down")
 			breakout = true
 		case <-rex.ShutdownChan():
+			logger.Info("reexec Shutdown channel triggered")
 			reexecing = true
 			breakout = true
 		case sig := <-signals:
+			logger.Infof("signal %q received", sig)
 			if sig == syscall.SIGHUP {
-				rexLogger.Infof("SIGHUP triggered re-exec")
+				logger.Infof("signals syscall.SIGHUP received, triggering agent restart")
 				rex.ReExec(nil)
 			} else {
 				breakout = true
@@ -252,6 +262,8 @@ func run(override cfgOverrider) error {
 		if breakout {
 			if !reexecing {
 				logger.Info("Shutting down Elastic Agent and sending last events...")
+			} else {
+				logger.Info("Restarting Elastic Agent")
 			}
 			break
 		}
@@ -407,7 +419,7 @@ func tryDelayEnroll(ctx context.Context, logger *logger.Logger, cfg *configurati
 	enrollPath := paths.AgentEnrollFile()
 	if _, err := os.Stat(enrollPath); err != nil {
 		// no enrollment file exists or failed to stat it; nothing to do
-		return cfg, nil //nolint:nilerr // there is nothing to do
+		return cfg, nil
 	}
 	contents, err := ioutil.ReadFile(enrollPath)
 	if err != nil {
@@ -461,7 +473,7 @@ func initTracer(agentName, version string, mcfg *monitoringCfg.MonitoringConfig)
 
 	cfg := mcfg.APM
 
-	// nolint:godox // the TODO is intentional
+	//nolint:godox // the TODO is intentional
 	// TODO(stn): Ideally, we'd use apmtransport.NewHTTPTransportOptions()
 	// but it doesn't exist today. Update this code once we have something
 	// available via the APM Go agent.