Merge remote-tracking branch 'upstream/master' into feature/windows-2012

* upstream/master:
  Add `docker logs` support to the Elastic Log Driver (elastic#19531)
  [Elastic Agent] Fix saving of agent configuration on Windows to have proper ACLs (elastic#19793)
  Send the config revision down to the endpoint application. (elastic#19759)
  [Elastic Agent] Add support for multiple hosts in connection to kibana (elastic#19628)
  Remove the downloadConfig and retryConfig from plugin/process.Application and plugin/service.Application. (elastic#19603)
  Update go version to 1.14.4 (elastic#19753)
  ci: set builds as skipped when they do not match the trigger (elastic#19750)
  [Auditbeat] Fix up socket dataset runaway CPU usage (elastic#19764)
  Convert cloudfoundry input to v2 (elastic#19717)

.ci/apm-beats-update.groovy

@@ -35,7 +35,14 @@ pipeline {
       when {
         beforeAgent true
         expression {
-          return isCommentTrigger() || isUserTrigger()
+          def ret = isCommentTrigger() || isUserTrigger()
+          if(!ret){
+            currentBuild.result = 'NOT_BUILT'
+            currentBuild.description = "The build has been skipped"
+            currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+            echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, and GitHub comment")
+          }
+          return ret
         }
       }
       /**

.ci/packaging.groovy

@@ -38,7 +38,14 @@ pipeline {
       when {
         beforeAgent true
         expression {
-          return isCommentTrigger() || isUserTrigger() || isUpstreamTrigger()
+          def ret = isCommentTrigger() || isUserTrigger() || isUpstreamTrigger()
+          if(!ret){
+            currentBuild.result = 'NOT_BUILT'
+            currentBuild.description = "The build has been skipped"
+            currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+            echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+          }
+          return ret
         }
       }
       stages {

.go-version

@@ -1 +1 @@
-1.13.10
+1.14.4

CHANGELOG-developer.next.asciidoc

@@ -95,3 +95,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add IP* fields to `fields.yml` generator script in Filebeat. {issue}17998[17998] {pull}18256[18256]
 - Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606]
 - Remove vendor folder from repository. {pull}18655[18655]
+- Update Go version to 1.14.4. {pull}19753[19753]

CHANGELOG.next.asciidoc

@@ -148,7 +148,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/package: Fix parsing of Installed-Size field of DEB packages. {issue}16661[16661] {pull}17188[17188]
 - system module: Fix panic during initialisation when /proc/stat can't be read. {pull}17569[17569]
 - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887]
-- system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033]
+- system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764]
 - system/socket: Fixed tracking of long-running connections. {pull}19033[19033]
 - system/package: Fix librpm loading on Fedora 31/32. {pull}NNNN[NNNN]
 - file_integrity: Create fsnotify watcher only when starting file_integrity module {pull}19505[19505]
@@ -560,6 +560,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add registry and code signature information and ECS categorization fields for sysmon module {pull}18058[18058]
 - Add new winlogbeat security dashboard {pull}18775[18775]
 
+*Elastic Log Driver*
+- Add support for `docker logs` command {pull}19531[19531]
+
 ==== Deprecated
 
 *Affecting all Beats*

Makefile

@@ -98,6 +98,8 @@ check: python-env
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
 	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
 	@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
+	# check if vendor folder does not exists
+	[ ! -d vendor ]
 	@# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers

NOTICE.txt

@@ -11014,6 +11014,25 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/hectane/go-acl
+Version: v0.0.0-20190604041725-da78bae5fc95
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/hectane/[email protected]/LICENSE.txt:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Nathan Osman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/dhcp
 Version: v0.0.0-20200227161230-57ec251c7eb3
@@ -15626,11 +15645,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/tools
-Version: v0.0.0-20200615191743-991d59a616de
+Version: v0.0.0-20200602230032-c00d67ef29d0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200615191743-991d59a616de/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200602230032-c00d67ef29d0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.10
+FROM golang:1.14.4
 
 RUN \
     apt-get update \

dev-tools/mage/build.go

@@ -118,16 +118,6 @@ func Build(params BuildArgs) error {
 	}
 	env["CGO_ENABLED"] = cgoEnabled
 
-	if UseVendor {
-		var goFlags string
-		goFlags, ok := env["GOFLAGS"]
-		if !ok {
-			env["GOFLAGS"] = "-mod=vendor"
-		} else {
-			env["GOFLAGS"] = strings.Join([]string{goFlags, "-mod=vendor"}, " ")
-		}
-	}
-
 	// Spec
 	args := []string{
 		"build",

dev-tools/mage/common.go

@@ -49,8 +49,6 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/magefile/mage/target"
 	"github.com/pkg/errors"
-
-	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // Expand expands the given Go text/template string.
@@ -806,16 +804,6 @@ func ParseVersion(version string) (major, minor, patch int, err error) {
 	return
 }
 
-// listModuleDir calls gotool.ListModuleVendorDir or
-// gotool.ListModuleCacheDir, depending on the value of
-// UseVendor.
-func listModuleDir(modpath string) (string, error) {
-	if UseVendor {
-		return gotool.ListModuleVendorDir(modpath)
-	}
-	return gotool.ListModuleCacheDir(modpath)
-}
-
 // ListMatchingEnvVars returns all of the environment variables names that begin
 // with prefix.
 func ListMatchingEnvVars(prefixes ...string) []string {

dev-tools/mage/crossbuild.go

@@ -255,9 +255,6 @@ func (b GolangCrossBuilder) Build() error {
 	if versionQualified {
 		args = append(args, "--env", "VERSION_QUALIFIER="+versionQualifier)
 	}
-	if UseVendor {
-		args = append(args, "--env", "GOFLAGS=-mod=vendor")
-	}
 	if CrossBuildMountModcache {
 		// Mount $GOPATH/pkg/mod into the container, read-only.
 		hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod")

dev-tools/mage/fields.go

@@ -101,16 +101,12 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
 		return err
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, globalFieldsCmdPath),
 		"-es_beats_path", beatsDir,
 		"-beat_path", baseDir,
 		"-out", CreateDir(output),
-	)
+	}
 	globalFieldsCmd := sh.RunCmd("go", cmd...)
 
 	return globalFieldsCmd(moduleDirs...)
@@ -130,18 +126,14 @@ func GenerateFieldsGo(fieldsYML, out string) error {
 		return err
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, assetCmdPath),
 		"-pkg", "include",
 		"-in", fieldsYML,
 		"-out", CreateDir(out),
 		"-license", toLibbeatLicenseName(BeatLicense),
 		BeatName,
-	)
+	}
 	assetCmd := sh.RunCmd("go", cmd...)
 
 	return assetCmd()
@@ -162,16 +154,12 @@ func GenerateModuleFieldsGo(moduleDir string) error {
 		moduleDir = CWD(moduleDir)
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, moduleFieldsCmdPath),
 		"-beat", BeatName,
 		"-license", toLibbeatLicenseName(BeatLicense),
 		moduleDir,
-	)
+	}
 	moduleFieldsCmd := sh.RunCmd("go", cmd...)
 
 	return moduleFieldsCmd()
@@ -194,16 +182,12 @@ func GenerateIncludeListGo(options IncludeListOptions) error {
 		return err
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, moduleIncludeListCmdPath),
 		"-license", toLibbeatLicenseName(BeatLicense),
 		"-out", options.Outfile, "-buildTags", options.BuildTags,
 		"-pkg", options.Pkg,
-	)
+	}
 
 	includeListCmd := sh.RunCmd("go", cmd...)
 

dev-tools/mage/fmt.go

@@ -66,19 +66,10 @@ func GoImports() error {
 	}
 
 	fmt.Println(">> fmt - goimports: Formatting Go code")
-	if UseVendor {
-		if err := gotool.Install(
-			gotool.Install.Vendored(),
-			gotool.Install.Package(filepath.Join(GoImportsImportPath)),
-		); err != nil {
-			return err
-		}
-	} else {
-		if err := gotool.Install(
-			gotool.Install.Package(filepath.Join(GoImportsImportPath)),
-		); err != nil {
-			return err
-		}
+	if err := gotool.Install(
+		gotool.Install.Package(filepath.Join(GoImportsImportPath)),
+	); err != nil {
+		return err
 	}
 
 	args := append(

dev-tools/mage/godaemon.go

@@ -22,6 +22,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 var (
@@ -43,7 +45,7 @@ func BuildGoDaemon() error {
 	}
 
 	// Test if binaries are up-to-date.
-	godaemonDir, err := listModuleDir("github.com/tsg/go-daemon")
+	godaemonDir, err := gotool.ListModuleCacheDir("github.com/tsg/go-daemon")
 	if err != nil {
 		return err
 	}

dev-tools/mage/install.go

@@ -37,9 +37,6 @@ func InstallVendored(importPath string) error {
 
 // InstallGoLicenser target installs go-licenser
 func InstallGoLicenser() error {
-	if UseVendor {
-		return InstallVendored(GoLicenserImportPath)
-	}
 	return gotool.Get(
 		gotool.Get.Package(GoLicenserImportPath),
 	)

dev-tools/mage/integtest.go

@@ -245,9 +245,6 @@ func initRunner(tester IntegrationTester, dir string, passInEnv map[string]strin
 	if mg.Verbose() {
 		env["MAGEFILE_VERBOSE"] = "1"
 	}
-	if UseVendor {
-		env["GOFLAGS"] = "-mod=vendor"
-	}
 
 	runner := &IntegrationRunner{
 		steps:  runnerSteps,

dev-tools/mage/settings.go

@@ -60,7 +60,6 @@ var (
 	XPackDir     = "../x-pack"
 	RaceDetector = false
 	TestCoverage = false
-	UseVendor    = false
 
 	// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
 	// the crossbuild images at /go/pkg/mod, read-only.
@@ -110,10 +109,6 @@ func init() {
 	if err != nil {
 		panic(errors.Wrap(err, "failed to parse TEST_COVERAGE env value"))
 	}
-	UseVendor, err = strconv.ParseBool(EnvOr("USE_VENDOR", "false"))
-	if err != nil {
-		panic(errors.Wrap(err, "failed to parse USE_VENDOR env value"))
-	}
 
 	Snapshot, err = strconv.ParseBool(EnvOr("SNAPSHOT", "false"))
 	if err != nil {
@@ -286,7 +281,7 @@ func findElasticBeatsDir() (string, error) {
 	if repo.IsElasticBeats() {
 		return repo.RootDir, nil
 	}
-	return listModuleDir(elasticBeatsModulePath)
+	return gotool.ListModuleCacheDir(elasticBeatsModulePath)
 }
 
 var (

dev-tools/packaging/packages.yml

@@ -72,8 +72,8 @@ shared:
   - &macos_agent_pkg_spec
     <<: *common
     extra_vars:
-      # OS X 10.11 Mountain Lion is the oldest supported by Go 1.13.
-      # https://golang.org/doc/go1.13#ports
+      # OS X 10.11 El Capitan is the oldest supported by Go 1.14.
+      # https://golang.org/doc/go1.14#ports
       min_supported_osx_version: 10.11
       identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}'
       install_path: /Library/Application Support

docs/devguide/newbeat.asciidoc

@@ -515,9 +515,6 @@ To depend on the latest `master` of `github.com/elastic/beats` run the following
 go get github.com/elastic/beats@master
 ----------------------------------------------------------------------
 
-To move the dependencies to vendor, you need to manually fetch the new
-`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.
-
 We suggest you read the following section to learn about maintaining dependencies
 using go modules:
 * https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]
@@ -526,6 +523,5 @@ Please note that it is your choice whether you put dependencies in the folder ve
 
 If you choose to abandon the vendor folder, you have to adjust the following things in your Beat:
 
-* add `devtools.UseVendor = false` to the `magefile.go` of your Beat
 * add `devtools.CrossBuildMountModcache = true` to the `magefile.go` of your Beat
 * modify the path to `ES_BEATS` in the `Makefile` so it points to the folder under the module cache (`go list -m -f '{{.Dir}}' $ES_BEATS_IMPORT_PATH`)

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.10
+FROM golang:1.14.4
 
 RUN \
     apt-get update \

generator/_templates/beat/{beat}/Makefile

@@ -4,7 +4,7 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
-ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
 LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
 GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
 GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"

generator/_templates/beat/{beat}/magefile.go

@@ -14,7 +14,6 @@ import (
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	"github.com/elastic/beats/v7/generator/common/beatgen"
 )
 
 func init() {
@@ -23,11 +22,7 @@ func init() {
 	devtools.BeatDescription = "One sentence description of the Beat."
 	devtools.BeatVendor = "{full_name}"
 	devtools.BeatProjectType = devtools.CommunityProject
-}
-
-// VendorUpdate updates the vendor dir
-func VendorUpdate() error {
-	return beatgen.VendorUpdate()
+	devtools.CrossBuildMountModcache = true
 }
 
 // Package packages the Beat for distribution.