Merge remote-tracking branch 'upstream/8.0' into mergify/bp/8.0/pr-28963

CHANGELOG.next.asciidoc

@@ -179,12 +179,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
 - Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
+- Upgrade azure-eventhub sdk reference, contains potential checkpoint fixes. {pull}28919[28919]
 
 *Heartbeat*
 
 - Fix broken seccomp filtering and improve security via `setcap` and `setuid` when running as root on linux in containers. {pull}27878[27878]
 - Log browser `zip_url` download failures as `warn` instead of as `info`. {pull}28440[28440]
 - Properly locate base stream in fleet configs. {pull}28455[28455]
+- Stop logging params values. {pull}28774[28774]
+- Remove accidentally included cups library in docker images. {pull}28853[pull]
+- Fix broken monitors with newer versions of image relying on dup3. {pull}28938[pull]
 
 *Journalbeat*
 
@@ -230,6 +234,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
 - Add ECS 1.9 new users fields {pull}26509[26509]
+- Don't split hyphenated tokens {pull}28483[28483]
 
 *Functionbeat*
 
@@ -267,6 +272,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update kubernetes scheduler and controllermanager endpoints in elastic-agent-standalone-kubernetes.yaml with secure ports {pull}28675[28675]
 - Add options to configure k8s client qps/burst. {pull}28151[28151]
 - Update to ECS 8.0 fields. {pull}28620[28620]
+- Add http.pprof.enabled option to libbeat to allow http/pprof endpoints on the socket that libbeat creates for metrics. {issue}21965[21965]
+- Support custom analyzers in fields.yml. {issue}28540[28540] {pull}28926[28926]
 
 *Auditbeat*
 
@@ -345,6 +352,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
 - Preliminary AIX support {pull}27954[27954]
 - Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
+- Update reference to gosigar pacakge for filesystem windows fix. {pull}28909[28909]
 
 *Packetbeat*
 

NOTICE.txt

@@ -832,11 +832,11 @@ Contents of probable licence file $GOMODCACHE/code.cloudfoundry.org/go-loggregat
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-event-hubs-go/v3
-Version: v3.1.2
+Version: v3.3.15
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs-go/v3@v3.1.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs-go/v3@v3.3.15/LICENSE:
 
     MIT License
 
@@ -923,11 +923,11 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-storage-bl
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/go-autorest/autorest
-Version: v0.11.16
+Version: v0.11.18
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/[email protected].16/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/[email protected].18/LICENSE:
 
 
                                  Apache License
@@ -8056,11 +8056,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.14.1
+Version: v0.14.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004
@@ -20030,11 +20030,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-amqp-common-go/v3
-Version: v3.0.0
+Version: v3.2.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-amqp-common-go/v3@v3.0.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-amqp-common-go/v3@v3.2.1/LICENSE:
 
     MIT License
 
@@ -20091,15 +20091,16 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-pipeline-g
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/go-amqp
-Version: v0.12.6
+Version: v0.16.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.12.6/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.16.0/LICENSE:
 
     MIT License
 
-    Copyright (c) Microsoft Corporation.
+    Copyright (C) 2017 Kale Blankenship
+    Portions Copyright (C) Microsoft Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal
@@ -20756,11 +20757,11 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/auto
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/go-autorest/autorest/to
-Version: v0.3.0
+Version: v0.4.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/autorest/to@v0.3.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/autorest/to@v0.4.0/LICENSE:
 
 
                                  Apache License
@@ -20957,11 +20958,11 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/auto
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/go-autorest/autorest/validation
-Version: v0.2.0
+Version: v0.3.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/autorest/validation@v0.2.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/autorest/validation@v0.3.1/LICENSE:
 
 
                                  Apache License

auditbeat/auditbeat.reference.yml

@@ -1632,6 +1632,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.

dev-tools/mage/crossbuild.go

@@ -134,16 +134,37 @@ func CrossBuild(options ...CrossBuildOption) error {
 		opt(&params)
 	}
 
-	// Docker is required for this target.
-	if err := HaveDocker(); err != nil {
-		return err
-	}
-
 	if len(params.Platforms) == 0 {
 		log.Printf("Skipping cross-build of target=%v because platforms list is empty.", params.Target)
 		return nil
 	}
 
+	// AIX can't really be crossbuilt, due to cgo and various compiler shortcomings.
+	// If we have a singular AIX platform set, revert to a native build toolchain
+	if runtime.GOOS == "aix" {
+		for _, platform := range params.Platforms {
+			if platform.GOOS() == "aix" {
+				if len(params.Platforms) != 1 {
+					return errors.New("AIX cannot be crossbuilt with other platforms. Set PLATFORMS='aix/ppc64'")
+				} else {
+					// This is basically a short-out so we can attempt to build on AIX in a relatively generic way
+					log.Printf("Target is building for AIX, skipping normal crossbuild process")
+					args := DefaultBuildArgs()
+					args.OutputDir = filepath.Join("build", "golang-crossbuild")
+					args.Name += "-" + Platform.GOOS + "-" + Platform.Arch
+					return Build(args)
+				}
+			}
+		}
+		// If we're here, something isn't set.
+		return errors.New("Cannot crossbuild on AIX. Either run `mage build` or set PLATFORMS='aix/ppc64'")
+	}
+
+	// Docker is required for this target.
+	if err := HaveDocker(); err != nil {
+		return err
+	}
+
 	if CrossBuildMountModcache {
 		// Make sure the module dependencies are downloaded on the host,
 		// as they will be mounted into the container read-only.

dev-tools/mage/pkgtypes.go

@@ -176,6 +176,11 @@ var OSArchNames = map[string]map[PackageType]map[string]string{
 			"arm64": "arm64",
 		},
 	},
+	"aix": map[PackageType]map[string]string{
+		TarGz: map[string]string{
+			"ppc64": "ppc64",
+		},
+	},
 }
 
 // getOSArchName returns the architecture name to use in a package.

dev-tools/mage/platforms.go

@@ -27,6 +27,7 @@ import (
 // BuildPlatforms is a list of GOOS/GOARCH pairs supported by Go.
 // The list originated from 'go tool dist list -json'.
 var BuildPlatforms = BuildPlatformList{
+	{"aix/ppc64", CGOSupported},
 	{"android/386", CGOSupported},
 	{"android/amd64", CGOSupported},
 	{"android/arm", CGOSupported},
@@ -256,7 +257,6 @@ func (list BuildPlatformList) Remove(name string) BuildPlatformList {
 // Select returns a new list containing the platforms that match name.
 func (list BuildPlatformList) Select(name string) BuildPlatformList {
 	attrs := BuildPlatform{Name: name}.Attributes()
-
 	if attrs.Arch == "" {
 		// Filter by GOOS only.
 		return list.filter(func(bp BuildPlatform) bool {
@@ -353,8 +353,11 @@ func NewPlatformList(expr string) BuildPlatformList {
 
 	var out BuildPlatformList
 	if len(pe.Add) == 0 || (len(pe.Select) == 0 && len(pe.Remove) == 0) {
-		// Bootstrap list with default platforms when the expression is
+		// Bootstrap list with platforms when the expression is
 		// exclusively adds OR exclusively selects and removes.
+		out = BuildPlatforms
+	}
+	if len(pe.Remove) > 0 || len(pe.Add) > 0 {
 		out = BuildPlatforms.Defaults()
 	}
 
@@ -375,7 +378,6 @@ func NewPlatformList(expr string) BuildPlatformList {
 		}
 		out = selected
 	}
-
 	for _, name := range pe.Remove {
 		if name == "defaults" {
 			for _, defaultBP := range all.Defaults() {

dev-tools/mage/pytest.go

@@ -41,7 +41,8 @@ import (
 // to point to somewhere on C:\.
 
 const (
-	libbeatRequirements = "{{ elastic_beats_dir}}/libbeat/tests/system/requirements.txt"
+	libbeatRequirements    = "{{ elastic_beats_dir}}/libbeat/tests/system/requirements.txt"
+	aixLibbeatRequirements = "{{ elastic_beats_dir}}/libbeat/tests/system/requirements_aix.txt"
 )
 
 var (
@@ -199,11 +200,12 @@ func PythonVirtualenv() (string, error) {
 	pythonVirtualenvLock.Lock()
 	defer pythonVirtualenvLock.Unlock()
 
-	// When upgrading pip we might run into an error with the cryptography package
-	// (pip dependency) will not compile if no recent rust development environment is available.
-	// We set `CRYPTOGRAPHY_DONT_BUILD_RUST=1`, to disable the need for python.
-	// See: https://github.com/pyca/cryptography/issues/5771
-	os.Setenv("CRYPTOGRAPHY_DONT_BUILD_RUST", "1")
+	// Certain docker requirements simply won't build on AIX
+	// Skipping them here will obviously break the components that require docker-compose,
+	// But at least the components that don't require it will still run
+	if runtime.GOOS == "aix" {
+		VirtualenvReqs[0] = aixLibbeatRequirements
+	}
 
 	// Determine the location of the virtualenv.
 	ve, err := pythonVirtualenvPath()

dev-tools/packaging/packages.yml

@@ -712,6 +712,11 @@ specs:
       spec:
         <<: *docker_spec
 
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+
   # Elastic Beat with Apache License (OSS) and binary taken the current
   # directory.
   elastic_beat_oss:
@@ -758,6 +763,13 @@ specs:
         <<: *apache_license_for_binaries
         name: '{{.BeatName}}-oss'
 
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
   # Elastic Beat with Elastic License and binary taken the current directory.
   elastic_beat_xpack:
     ###
@@ -827,6 +839,12 @@ specs:
         <<: *elastic_docker_spec
         <<: *elastic_license_for_binaries
 
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
   # Elastic Beat with Elastic License and binary taken the current directory.
   elastic_beat_xpack_reduced:
     ###
@@ -850,6 +868,12 @@ specs:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
 
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_xpack_separate_binaries:
     ###
@@ -947,6 +971,15 @@ specs:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_agent_binaries:
     ###
@@ -1105,6 +1138,17 @@ specs:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *agent_binary_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}
+            symlink: true
+            mode: 0755
+
 
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_agent_demo_binaries:
@@ -1130,3 +1174,12 @@ specs:
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *agent_binary_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -57,8 +57,8 @@ RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \
 
 {{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }}
 RUN for iter in {1..10}; do \
-        yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
-        libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \
+        yum -y install atk gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
+        libXext libXi libXtst libXScrnSaver libXrandr GConf2 \
         alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \
         xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \
         yum clean all && \

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -38,8 +38,8 @@ RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \
     yum -y update $YUM_FLAGS \
   {{- if (eq .BeatName "heartbeat") }}
     && yum -y install epel-release \
-    && yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
-        libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \
+    && yum -y install atk gtk gdk xrandr pango libXcomposite libXcursor libXdamage \
+        libXext libXi libXtst libXScrnSaver libXrandr GConf2 \
         alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \
         xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \
   {{- end }}