Merge remote-tracking branch 'upstream/master' into feature/fix-warnings-archive

* upstream/master:
  Fix minor spelling error in Jenkinsfile (elastic#19153)
  [CI] fail if not possible to install python3 (elastic#19164)
  [Elastic Agent] Improved mage demo experience (elastic#18445)
  [yum] Clear cached data and add retry loop (elastic#19182)
  fix lint job by updating NOTICE (elastic#19161)
  Fix tags for coredns/envoyproxy (elastic#19134)
  Disable host.* fields by default for CrowdStrike module (elastic#19132)
  Allow host.* fields to be disabled in Zeek module (elastic#19113)
  Rename to management.Manager, add UpdateStatus to Manager interface. (elastic#19114)
  Edit Elastic Agent docs (elastic#19146)
  [JJBB] create job definition for the golang-crossbuild project (elastic#19162)
  Fix incorrect usage of hints builder when exposed port is a substring of the hint (elastic#19052)
  Add basic cloudfoundry integration tests (elastic#19018)

Author: v1v

.ci/jobs/golang-crossbuild-mbp.yml

@@ -0,0 +1,43 @@
+---
+- job:
+    name: Beats/golang-crossbuild-mbp
+    display-name: Pipeline for golang-crossbuild
+    description: Jenkins pipeline for the golang-crossbuild project.
+    view: Beats
+    project-type: multibranch
+    script-path: Jenkinsfile
+    scm:
+      - github:
+          branch-discovery: no-pr
+          discover-pr-forks-strategy: merge-current
+          discover-pr-forks-trust: permission
+          discover-pr-origin: merge-current          
+          discover-tags: true
+          notification-context: 'beats-ci'
+          repo: golang-crossbuild
+          repo-owner: elastic
+          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          ssh-checkout:
+            credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          build-strategies:
+            - tags:
+                ignore-tags-older-than: -1
+                ignore-tags-newer-than: -1
+            - regular-branches: true
+            - change-request:
+                ignore-target-only-changes: false
+          clean:
+            after: true
+            before: true
+          prune: true
+          shallow-clone: true
+          depth: 4
+          do-not-fetch-tags: true
+          submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+          timeout: '15'
+          use-author: true
+          wipe-workspace: 'True'

.ci/scripts/install-tools.bat

@@ -20,7 +20,7 @@ where mage
 
 if not exist C:\Python38\python.exe (
     REM Install python 3.8.
-    choco install python -y -r --no-progress --version 3.8.2
+    choco install python -y -r --no-progress --version 3.8.2 || echo ERROR && exit /b
 )
 python --version
 where python

CHANGELOG-developer.next.asciidoc

@@ -46,6 +46,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
   Your magefile.go will require a change to adapt the devtool API. See the pull request for
   more details. {pull}18148[18148]
 - The Elasticsearch client settings expect the API key to be raw (not base64-encoded). {issue}18939[18939] {pull}18945[18945]
+- `management.ConfigManager` has been renamed to `management.Manager`. {pull}19114[19114]
+- `UpdateStatus` has been added to the `management.Manager` interface. {pull}19114[19114]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -39,10 +39,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 * CEF {pull}18223[18223]
 * PANW {pull}18223[18223]
 * Cisco {pull}18753[18753]
+* CrowdStrike {pull}19132[19132]
 * iptables {pull}18756[18756]
 * Checkpoint {pull}18754[18754]
 * Netflow {pull}19087[19087]
+* Zeek {pull}19113[19113] (`forwarded` tag is not included by default)
 * Suricata {pull}19107[19107] (`forwarded` tag is not included by default)
+* CoreDNS {pull}19134[19134] (`forwarded` tag is not included by default)
+* Envoy Proxy {pull}19134[19134] (`forwarded` tag is not included by default)
 - Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
 - Adds check on `<no value>` config option value for the azure input `resource_manager_endpoint`. {pull}18890[18890]
 - Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953]
@@ -241,6 +245,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add missing network.sent_packets_count metric into compute metricset in googlecloud module. {pull}18802[18802]
 - Fix compute and pubsub dashboard for googlecloud module. {issue}18962[18962] {pull}18980[18980]
 - Fix crash on vsphere module when Host information is not available. {issue}18996[18996] {pull}19078[19078]
+- Fix incorrect usage of hints builder when exposed port is a substring of the hint {pull}19052[19052]
 
 *Packetbeat*
 

Jenkinsfile

@@ -704,7 +704,7 @@ pipeline {
             stage('Generators Metricbeat Linux'){
               steps {
                 // FIXME see https://github.com/elastic/beats/issues/18132
-                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') {
+                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') {
                   makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test")
                   makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test-package")
                 }
@@ -713,7 +713,7 @@ pipeline {
             stage('Generators Beat Linux'){
               steps {
                 // FIXME see https://github.com/elastic/beats/issues/18132
-                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') {
+                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') {
                   makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test")
                   makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test-package")
                 }
@@ -730,7 +730,7 @@ pipeline {
               }
               steps {
                 // FIXME see https://github.com/elastic/beats/issues/18132
-                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') {
+                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') {
                   makeTarget(context: "Generators Metricbeat Mac OS X", target: "-C generator/_templates/metricbeat test")
                 }
               }
@@ -751,7 +751,7 @@ pipeline {
               }
               steps {
                 // FIXME see https://github.com/elastic/beats/issues/18132
-                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') {
+                catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') {
                   makeTarget(context: "Generators Beat Mac OS X", target: "-C generator/_templates/beat test")
                 }
               }

NOTICE.txt

@@ -7843,8 +7843,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: go.elastic.co/ecszap
-Version: v0.1.1
-Revision: cdd95a104193
+Version: v0.2.0
 License type (autodetected): Apache-2.0
 ./vendor/go.elastic.co/ecszap/LICENSE:
 --------------------------------------------------------------------

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -5,10 +5,8 @@
 FROM {{ .from }}
 
 # Installing jq needs to be installed after epel-release and cannot be in the same yum install command.
-RUN yum -y --setopt=tsflags=nodocs update && \
-    yum install epel-release -y && \
-    yum install jq -y && \
-    yum clean all
+RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release &&  yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
+RUN for iter in {1..10}; do yum update -y && yum install -y jq &&  yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
 
 LABEL \
   org.label-schema.build-date="{{ date }}" \

dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl

@@ -3,7 +3,6 @@
 set -eo pipefail
 
 # Environment variables used
-# FLEET_CONFIG_ID - config related to new token [defaul]
 # FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll
 # FLEET_ENROLL - if set to 1 enroll will be performed
 # FLEET_SETUP - if  set to 1 fleet setup will be performed
@@ -49,8 +48,9 @@ function enroll(){
       if [ $exitCode -ne 0 ]; then
         exit $exitCode
       fi
+
+      apikey=$(echo $enrollResp | jq -r '.item.api_key')
     fi
-    apikey=$(echo $enrollResp | jq -r '.item.api_key')
     echo $apikey
 
     ./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f

filebeat/beater/filebeat.go

@@ -114,7 +114,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 		haveEnabledInputs = true
 	}
 
-	if !config.ConfigInput.Enabled() && !config.ConfigModules.Enabled() && !haveEnabledInputs && config.Autodiscover == nil && !b.ConfigManager.Enabled() {
+	if !config.ConfigInput.Enabled() && !config.ConfigModules.Enabled() && !haveEnabledInputs && config.Autodiscover == nil && !b.Manager.Enabled() {
 		if !b.InSetupCmd {
 			return nil, errors.New("no modules or inputs enabled and configuration reloading disabled. What files do you want me to watch?")
 		}

heartbeat/beater/heartbeat.go

@@ -86,7 +86,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 		return err
 	}
 
-	if b.ConfigManager.Enabled() {
+	if b.Manager.Enabled() {
 		bt.RunCentralMgmtMonitors(b)
 	}
 

libbeat/beat/beat.go

@@ -66,7 +66,7 @@ type Beat struct {
 
 	Fields []byte // Data from fields.yml
 
-	ConfigManager management.ConfigManager // config manager
+	Manager management.Manager // manager
 
 	Keystore keystore.Keystore
 }

libbeat/cmd/instance/beat.go

@@ -330,12 +330,12 @@ func (b *Beat) createBeater(bt beat.Creator) (beat.Beater, error) {
 
 	// Report central management state
 	mgmt := monitoring.GetNamespace("state").GetRegistry().NewRegistry("management")
-	monitoring.NewBool(mgmt, "enabled").Set(b.ConfigManager.Enabled())
+	monitoring.NewBool(mgmt, "enabled").Set(b.Manager.Enabled())
 
 	debugf("Initializing output plugins")
 	outputEnabled := b.Config.Output.IsSet() && b.Config.Output.Config().Enabled()
 	if !outputEnabled {
-		if b.ConfigManager.Enabled() {
+		if b.Manager.Enabled() {
 			logp.Info("Output is configured through Central Management")
 		} else {
 			msg := "No outputs are defined. Please define one under the output section."
@@ -462,8 +462,8 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 	logp.Info("%s start running.", b.Info.Beat)
 
 	// Launch config manager
-	b.ConfigManager.Start(beater.Stop)
-	defer b.ConfigManager.Stop()
+	b.Manager.Start(beater.Stop)
+	defer b.Manager.Stop()
 
 	return beater.Run(&b.Beat)
 }
@@ -643,12 +643,12 @@ func (b *Beat) configure(settings Settings) error {
 	logp.Info("Beat ID: %v", b.Info.ID)
 
 	// initialize config manager
-	b.ConfigManager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID)
+	b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID)
 	if err != nil {
 		return err
 	}
 
-	if err := b.ConfigManager.CheckRawConfig(b.RawConfig); err != nil {
+	if err := b.Manager.CheckRawConfig(b.RawConfig); err != nil {
 		return err
 	}
 

libbeat/management/management.go

@@ -18,11 +18,36 @@
 package management
 
 import (
+	"sync"
+
 	"github.com/gofrs/uuid"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/reload"
 	"github.com/elastic/beats/v7/libbeat/feature"
+	"github.com/elastic/beats/v7/libbeat/logp"
+)
+
+// Status describes the current status of the beat.
+type Status int
+
+const (
+	// Unknown is initial status when none has been reported.
+	Unknown Status = iota
+	// Starting is status describing application is starting.
+	Starting
+	// Configuring is status describing application is configuring.
+	Configuring
+	// Running is status describing application is running.
+	Running
+	// Degraded is status describing application is degraded.
+	Degraded
+	// Failed is status describing application is failed. This status should
+	// only be used in the case the beat should stop running as the failure
+	// cannot be recovered.
+	Failed
+	// Stopping is status describing application is stopping.
+	Stopping
 )
 
 // Namespace is the feature namespace for queue definition.
@@ -33,27 +58,36 @@ var DebugK = "centralmgmt"
 
 var centralMgmtKey = "x-pack-cm"
 
-// ConfigManager interacts with the beat to update configurations
-// from an external source
-type ConfigManager interface {
-	// Enabled returns true if config manager is enabled
+// StatusReporter provides a method to update current status of the beat.
+type StatusReporter interface {
+	// UpdateStatus called when the status of the beat has changed.
+	UpdateStatus(status Status, msg string)
+}
+
+// Manager interacts with the beat to provide status updates and to receive
+// configurations.
+type Manager interface {
+	StatusReporter
+
+	// Enabled returns true if manager is enabled.
 	Enabled() bool
 
-	// Start the config manager
-	Start(func())
+	// Start the config manager giving it a stopFunc callback
+	// so the beat can be told when to stop.
+	Start(stopFunc func())
 
-	// Stop the config manager
+	// Stop the config manager.
 	Stop()
 
-	// CheckRawConfig check settings are correct before launching the beat
+	// CheckRawConfig check settings are correct before launching the beat.
 	CheckRawConfig(cfg *common.Config) error
 }
 
 // PluginFunc for creating FactoryFunc if it matches a config
 type PluginFunc func(*common.Config) FactoryFunc
 
 // FactoryFunc for creating a config manager
-type FactoryFunc func(*common.Config, *reload.Registry, uuid.UUID) (ConfigManager, error)
+type FactoryFunc func(*common.Config, *reload.Registry, uuid.UUID) (Manager, error)
 
 // Register a config manager
 func Register(name string, fn PluginFunc, stability feature.Stability) {
@@ -91,13 +125,32 @@ func defaultModeConfig() *modeConfig {
 }
 
 // nilManager, fallback when no manager is present
-type nilManager struct{}
+type nilManager struct {
+	logger *logp.Logger
+	lock   sync.Mutex
+	status Status
+	msg    string
+}
 
-func nilFactory(*common.Config, *reload.Registry, uuid.UUID) (ConfigManager, error) {
-	return nilManager{}, nil
+func nilFactory(*common.Config, *reload.Registry, uuid.UUID) (Manager, error) {
+	log := logp.NewLogger("mgmt")
+	return &nilManager{
+		logger: log,
+		status: Unknown,
+		msg:    "",
+	}, nil
 }
 
-func (nilManager) Enabled() bool                           { return false }
-func (nilManager) Start(_ func())                          {}
-func (nilManager) Stop()                                   {}
-func (nilManager) CheckRawConfig(cfg *common.Config) error { return nil }
+func (*nilManager) Enabled() bool                           { return false }
+func (*nilManager) Start(_ func())                          {}
+func (*nilManager) Stop()                                   {}
+func (*nilManager) CheckRawConfig(cfg *common.Config) error { return nil }
+func (n *nilManager) UpdateStatus(status Status, msg string) {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+	if n.status != status || n.msg != msg {
+		n.status = status
+		n.msg = msg
+		n.logger.Infof("Status change to %s: %s", status, msg)
+	}
+}

metricbeat/autodiscover/builder/hints/metrics.go

@@ -19,6 +19,7 @@ package hints
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/elastic/go-ucfg"
@@ -187,7 +188,7 @@ func (m *metricHints) getHostsWithPort(hints common.MapStr, port int) ([]string,
 	// Only pick hosts that have ${data.port} or the port on current event. This will make
 	// sure that incorrect meta mapping doesn't happen
 	for _, h := range thosts {
-		if strings.Contains(h, "data.port") || strings.Contains(h, fmt.Sprintf(":%d", port)) ||
+		if strings.Contains(h, "data.port") || m.checkHostPort(h, port) ||
 			// Use the event that has no port config if there is a ${data.host}:9090 like input
 			(port == 0 && strings.Contains(h, "data.host")) {
 			result = append(result, h)
@@ -202,6 +203,27 @@ func (m *metricHints) getHostsWithPort(hints common.MapStr, port int) ([]string,
 	return result, true
 }
 
+func (m *metricHints) checkHostPort(h string, p int) bool {
+	port := strconv.Itoa(p)
+
+	index := strings.LastIndex(h, ":"+port)
+	// Check if host contains :port. If not then return false
+	if index == -1 {
+		return false
+	}
+
+	// Check if the host ends with :port. Return true if yes
+	end := index + len(port) + 1
+	if end == len(h) {
+		return true
+	}
+
+	// Check if the character immediately after :port. If its not a number then return true.
+	// This is to avoid adding :80 as a valid host for an event that has port=8080
+	// Also ensure that port=3306 and hint="tcp(${data.host}:3306)/" is valid
+	return h[end] < '0' || h[end] > '9'
+}
+
 func (m *metricHints) getNamespace(hints common.MapStr) string {
 	return builder.GetHintString(hints, m.Key, namespace)
 }