Update beats framework to 03f7e87 (#1281)

NOTICE.txt

@@ -292,7 +292,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: e30da2c174a48511ffc583696e03dcc3237aabe9
+Revision: 03f7e873b2df8ed71e40570976bc36eb9a0fda7c
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/dev-tools/mage/clean.go

@@ -35,6 +35,10 @@ var DefaultCleanPaths = []string{
 	"_meta/kibana.generated",
 	"_meta/kibana/5/index-pattern/{{.BeatName}}.json",
 	"_meta/kibana/6/index-pattern/{{.BeatName}}.json",
+
+	"../x-pack/{{.BeatName}}/build",
+	"../x-pack/{{.BeatName}}/{{.BeatName}}",
+	"../x-pack/{{.BeatName}}/{{.BeatName}}.exe",
 }
 
 // Clean clean generated build artifacts.

_beats/dev-tools/mage/crossbuild.go

@@ -63,6 +63,13 @@ func WithTarget(target string) func(params *crossBuildParams) {
 	}
 }
 
+// InDir specifies the base directory to use when cross-building.
+func InDir(path ...string) func(params *crossBuildParams) {
+	return func(params *crossBuildParams) {
+		params.InDir = filepath.Join(path...)
+	}
+}
+
 // Serially causes each cross-build target to be executed serially instead of
 // in parallel.
 func Serially() func(params *crossBuildParams) {
@@ -75,6 +82,7 @@ type crossBuildParams struct {
 	Platforms BuildPlatformList
 	Target    string
 	Serial    bool
+	InDir     string
 }
 
 // CrossBuild executes a given build target once for each target platform.
@@ -103,8 +111,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		if !buildPlatform.Flags.CanCrossBuild() {
 			return fmt.Errorf("unsupported cross build platform %v", buildPlatform.Name)
 		}
-
-		builder := GolangCrossBuilder{buildPlatform.Name, params.Target}
+		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
 				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v",
@@ -120,6 +127,15 @@ func CrossBuild(options ...CrossBuildOption) error {
 	return nil
 }
 
+// CrossBuildXPack executes the 'golangCrossBuild' target in the Beat's
+// associated x-pack directory to produce a version of the Beat that contains
+// Elastic licensed content.
+func CrossBuildXPack(options ...CrossBuildOption) error {
+	o := []CrossBuildOption{InDir("x-pack", BeatName)}
+	o = append(o, options...)
+	return CrossBuild(o...)
+}
+
 // buildMage pre-compiles the magefile to a binary using the native GOOS/GOARCH
 // values for Docker. This is required to so that we can later pass GOOS and
 // GOARCH to mage for the cross-build. It has the benefit of speeding up the
@@ -166,6 +182,7 @@ func crossBuildImage(platform string) (string, error) {
 type GolangCrossBuilder struct {
 	Platform string
 	Target   string
+	InDir    string
 }
 
 // Build executes the build inside of Docker.
@@ -178,9 +195,16 @@ func (b GolangCrossBuilder) Build() error {
 	}
 
 	mountPoint := filepath.ToSlash(filepath.Join("/go", "src", repoInfo.RootImportPath))
-	workDir := mountPoint
-	if repoInfo.SubDir != "" {
-		workDir = filepath.ToSlash(filepath.Join(workDir, repoInfo.SubDir))
+	// use custom dir for build if given, subdir if not:
+	cwd := repoInfo.SubDir
+	if b.InDir != "" {
+		cwd = b.InDir
+	}
+	workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd))
+
+	buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-amd64"))
+	if err != nil {
+		return errors.Wrap(err, "failed to determine mage-linux-amd64 relative path")
 	}
 
 	dockerRun := sh.RunCmd("docker", "run")
@@ -206,7 +230,7 @@ func (b GolangCrossBuilder) Build() error {
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,
-		"--build-cmd", "build/mage-linux-amd64 "+b.Target,
+		"--build-cmd", buildCmd+" "+b.Target,
 		"-p", b.Platform,
 	)
 
@@ -215,25 +239,27 @@ func (b GolangCrossBuilder) Build() error {
 
 // DockerChown chowns files generated during build. EXEC_UID and EXEC_GID must
 // be set in the containers environment otherwise this is a noop.
-func DockerChown(file string) {
+func DockerChown(path string) {
 	// Chown files generated during build that are root owned.
 	uid, _ := strconv.Atoi(EnvOr("EXEC_UID", "-1"))
 	gid, _ := strconv.Atoi(EnvOr("EXEC_GID", "-1"))
 	if uid > 0 && gid > 0 {
-		if err := chownPaths(uid, gid, file); err != nil {
+		if err := chownPaths(uid, gid, path); err != nil {
 			log.Println(err)
 		}
 	}
 }
 
 // chownPaths will chown the file and all of the dirs specified in the path.
-func chownPaths(uid, gid int, file string) error {
-	pathParts := strings.Split(file, string(filepath.Separator))
-	for i := range pathParts {
-		chownDir := filepath.Join(pathParts[:i+1]...)
-		if err := os.Chown(chownDir, uid, gid); err != nil {
-			return errors.Wrapf(err, "failed to chown path=%v", chownDir)
+func chownPaths(uid, gid int, path string) error {
+	return filepath.Walk(path, func(name string, _ os.FileInfo, err error) error {
+		if err != nil {
+			return err
 		}
-	}
-	return nil
+		log.Printf("chown line: %s\n", name)
+		if err := os.Chown(name, uid, gid); err != nil {
+			return errors.Wrapf(err, "failed to chown path=%v", name)
+		}
+		return err
+	})
 }

_beats/dev-tools/mage/pkg.go

@@ -37,7 +37,7 @@ func Package() error {
 
 	if len(Packages) == 0 {
 		return errors.New("no package specs are registered. Call " +
-			"UseCommunityBeatPackaging or UseElasticBeatPackaging first.")
+			"UseCommunityBeatPackaging, UseElasticBeatPackaging or USeElasticBeatWithoutXPackPackaging first.")
 	}
 
 	var tasks []interface{}

_beats/dev-tools/mage/pkgspecs.go

@@ -62,6 +62,26 @@ func UseElasticBeatPackaging() {
 	}
 }
 
+// UseElasticBeatWithoutXPackPackaging configures the package target to build packages for
+// an Elastic Beat. This means it will generate two sets of packages -- one
+// that is purely OSS under Apache 2.0 and one that is licensed under the
+// Elastic License and may contain additional X-Pack features.
+//
+// NOTE: This method doesn't use binaries produced in the x-pack folder, this is
+// a temporary packaging target for projects that depends on beat but do have concrete x-pack
+// binaries.
+func UseElasticBeatWithoutXPackPackaging() {
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		panic(err)
+	}
+
+	err = LoadNamedSpec("elastic_beat_without_xpack", filepath.Join(beatsDir, packageSpecFile))
+	if err != nil {
+		panic(err)
+	}
+}
+
 // LoadNamedSpec loads a packaging specification with the given name from the
 // specified YAML file. name should be a sub-key of 'specs'.
 func LoadNamedSpec(name, file string) error {

_beats/dev-tools/packaging/packages.yml

@@ -229,6 +229,78 @@ specs:
       spec:
         <<: *deb_rpm_spec
 
+  elastic_beat_without_xpack:
+    ###
+    # OSS Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *windows_binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
+    - os: darwin
+      types: [dmg]
+      spec:
+        <<: *macos_beat_pkg_spec
+        <<: *apache_license_for_macos_pkg
+        name: '{{.BeatName}}-oss'
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
+    - os: linux
+      types: [deb, rpm]
+      spec:
+        <<: *deb_rpm_spec
+        <<: *apache_license_for_deb_rpm
+        name: '{{.BeatName}}-oss'
+
+    ###
+    # Elastic Licensed Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *windows_binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: darwin
+      types: [dmg]
+      spec:
+        <<: *macos_beat_pkg_spec
+        <<: *elastic_license_for_macos_pkg
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: linux
+      types: [deb, rpm]
+      spec:
+        <<: *deb_rpm_spec
+        <<: *elastic_license_for_deb_rpm
+
   # Official Beats
   elastic_beat:
     ###
@@ -277,12 +349,17 @@ specs:
       spec:
         <<: *windows_binary_spec
         <<: *elastic_license_for_binaries
+        '{{.BeatName}}{{.BinaryExt}}':
+          source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: darwin
       types: [tgz]
       spec:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
+        files:
+          /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
+            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: darwin
       types: [dmg]
@@ -295,9 +372,15 @@ specs:
       spec:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: linux
       types: [deb, rpm]
       spec:
         <<: *deb_rpm_spec
         <<: *elastic_license_for_deb_rpm
+        files:
+          /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
+            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

_beats/libbeat/tests/system/beat/beat.py

@@ -584,8 +584,27 @@ def assert_fields_are_documented(self, evt):
         expected_fields, dict_fields = self.load_fields()
         flat = self.flatten_object(evt, dict_fields)
 
+        def field_pattern_match(pattern, key):
+            pattern_fields = pattern.split(".")
+            key_fields = key.split(".")
+            if len(pattern_fields) != len(key_fields):
+                return False
+            for i in range(len(pattern_fields)):
+                if pattern_fields[i] == "*":
+                    continue
+                if pattern_fields[i] != key_fields[i]:
+                    return False
+            return True
+
+        def is_documented(key):
+            if key in expected_fields:
+                return True
+            for pattern in (f for f in expected_fields if "*" in f):
+                if field_pattern_match(pattern, key):
+                    return True
+            return False
+
         for key in flat.keys():
-            documented = key in expected_fields
             metaKey = key.startswith('@metadata.')
-            if not(documented or metaKey):
+            if not(is_documented(key) or metaKey):
                 raise Exception("Key '{}' found in event is not documented!".format(key))

magefile.go

@@ -83,7 +83,7 @@ func Package() {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
-	mage.UseElasticBeatPackaging()
+	mage.UseElasticBeatWithoutXPackPackaging()
 	customizePackaging()
 
 	mg.Deps(Update, prepareIngestPackaging)