Port fields.yml collector to Golang

CHANGELOG-developer.asciidoc

@@ -21,6 +21,7 @@ The list below covers the major changes between 6.3.0 and master only.
 
 - The beat.Pipeline is now passed to cfgfile.RunnerFactory. Beats using libbeat for module reloading or autodiscovery need to be adapted. {pull}7018[7017]
 - Moving of TLS helper functions and structs from `output/tls` to `tlscommon`. {pull}7054[7054]
+- Port fields.yml collector to Golang {pull}6911[6911]
 
 ==== Bugfixes
 

auditbeat/Makefile

@@ -7,6 +7,7 @@ GOX_OS?=linux windows ## @Building List of all OS to be supported by "make cross
 DEV_OS?=linux
 TESTING_ENVIRONMENT?=snapshot-noxpack
 ES_BEATS?=..
+FIELDS_FILE_PATH=module
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile
@@ -58,13 +59,6 @@ before-build:
 .PHONY: collect
 collect: fields collect-docs configs kibana
 
-# Collects all module and metricset fields
-.PHONY: fields
-fields: python-env
-	@mkdir -p _meta
-	@cp _meta/fields.common.yml _meta/fields.generated.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
-
 # Collects all module configs
 .PHONY: configs
 configs: python-env

filebeat/Makefile

@@ -5,6 +5,8 @@ SYSTEM_TESTS?=true
 TEST_ENVIRONMENT?=true
 GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le"
 ES_BEATS?=..
+FIELDS_FILE_PATH=module
+
 
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
@@ -19,13 +21,6 @@ kibana:
 	@mkdir -p _meta/kibana
 	@-cp -r module/*/_meta/kibana _meta/
 
-# Collects all module and dataset fields
-.PHONY: fields
-fields: python-env
-	@mkdir -p _meta/
-	@cp ${ES_BEATS}/filebeat/_meta/fields.common.yml  _meta/fields.generated.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
-
 # Collects all modules files to be packaged in a temporary folder
 .PHONY: modules
 modules:

generator/beat/Makefile

@@ -1 +1,4 @@
+override FIELDS_FILE_PATH=
+export FIELDS_FILE_PATH
+
 include ../common/Makefile

generator/beat/{beat}/Makefile

@@ -46,4 +46,4 @@ before-build:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect:
+collect: fields

generator/common/Makefile

@@ -6,6 +6,7 @@ BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 PREPARE_COMMAND?=
 
+
 # Runs test build for mock beat
 .PHONY: test
 test: prepare-test

generator/metricbeat/Makefile

@@ -1,9 +1,16 @@
 BEAT_TYPE=metricbeat
-PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ;
+PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; FIELDS_FILE_PATH=module
+override FIELDS_FILE_PATH=
+export FIELDS_FILE_PATH
+
 
 include ../common/Makefile
 
 prepare-test:: python-env
 
 	mkdir -p ${BEAT_PATH}/scripts
 	rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts
+
+# Collects all dependencies and then calls update
+.PHONY: collect
+collect: fields

heartbeat/Makefile

@@ -4,6 +4,7 @@ BEAT_PACKAGE_NAME=heartbeat-elastic
 BEAT_DESCRIPTION?=Ping remote services for availability and log results to Elasticsearch or send to Logstash.
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT=false
+FIELDS_FILE_PATH=monitors/active
 
 # Path to the libbeat Makefile
 -include ../libbeat/scripts/Makefile
@@ -16,13 +17,6 @@ before-build:
 .PHONY: collect
 collect: fields imports
 
-# Collects all module and metricset fields
-.PHONY: fields
-fields:
-	@mkdir -p _meta/
-	@cp ${ES_BEATS}/heartbeat/_meta/fields.common.yml _meta/fields.generated.yml
-	@cat ${ES_BEATS}/heartbeat/monitors/active/*/_meta/fields.yml >> _meta/fields.generated.yml
-
 # Generate imports for all monitors
 .PHONY: imports
 imports: python-env

libbeat/Makefile

@@ -1,15 +1,10 @@
 BEAT_NAME=libbeat
 TEST_ENVIRONMENT?=true
 SYSTEM_TESTS=true
+FIELDS_FILE_PATH=processors
 
 include scripts/Makefile
 
-# Collects all fields from processors
-.PHONY: fields
-fields:
-	@cp _meta/fields.common.yml _meta/fields.generated.yml
-	@cat processors/*/_meta/fields.yml >> _meta/fields.generated.yml
-
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
+collect: libbeat_fields

libbeat/generator/fields/fields.go

@@ -0,0 +1,149 @@
+package fields
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml")
+)
+
+// YmlFile holds the info on files and how to write them into the global fields.yml
+type YmlFile struct {
+	Path   string
+	Indent int
+}
+
+func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
+	commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml")
+	_, err := os.Stat(commonFields)
+	if os.IsNotExist(err) {
+		return fieldFiles, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	files := []*YmlFile{
+		&YmlFile{
+			Path:   commonFields,
+			Indent: 0,
+		},
+	}
+
+	return append(files, fieldFiles...), nil
+}
+
+func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error {
+	outPath := path.Join(beatsPath, generatedFieldsYml)
+	f, err := os.Create(outPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	fw := bufio.NewWriter(f)
+	for _, p := range fieldFiles {
+		ff, err := os.Open(p.Path)
+		if err != nil {
+			return err
+		}
+		defer ff.Close()
+
+		fs := bufio.NewScanner(ff)
+		for fs.Scan() {
+			err = writeIndentedLine(fw, fs.Text()+"\n", p.Indent)
+			if err != nil {
+				return err
+			}
+
+		}
+		if err := fs.Err(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func writeIndentedLine(fw *bufio.Writer, l string, indent int) error {
+	ll := strings.Repeat(" ", indent) + l
+	fmt.Fprint(fw, ll)
+	return fw.Flush()
+}
+
+// Generate collects fields.yml files and concatenates them into one global file.
+func Generate(esBeatsPath, beatPath string, files []*YmlFile) error {
+	files, err := collectBeatFiles(beatPath, files)
+	if err != nil {
+		return err
+	}
+
+	err = writeGeneratedFieldsYml(beatPath, files)
+	if err != nil {
+		return err
+	}
+
+	return AppendFromLibbeat(esBeatsPath, beatPath)
+}
+
+// AppendFromLibbeat appends fields.yml of libbeat to the fields.yml
+func AppendFromLibbeat(esBeatsPath, beatPath string) error {
+	fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml")
+	generatedPath := path.Join(beatPath, generatedFieldsYml)
+
+	err := createIfNotExists(fieldsMetaPath, generatedPath)
+	if err != nil {
+		return err
+	}
+
+	if isLibbeat(beatPath) {
+		out := filepath.Join(esBeatsPath, "libbeat", "fields.yml")
+		return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+	}
+
+	libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml)
+	out := filepath.Join(beatPath, "fields.yml")
+	err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+	if err != nil {
+		return err
+	}
+	return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND)
+}
+
+func isLibbeat(beatPath string) bool {
+	return filepath.Base(beatPath) == "libbeat"
+}
+
+func createIfNotExists(inPath, outPath string) error {
+	_, err := os.Stat(outPath)
+	if os.IsNotExist(err) {
+		err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+		if err != nil {
+			fmt.Println("Cannot find _meta/fields.yml")
+		}
+		return nil
+	}
+	return err
+}
+
+func copyFileWithFlag(in, out string, flag int) error {
+	input, err := ioutil.ReadFile(in)
+	if err != nil {
+		return err
+	}
+
+	output, err := os.OpenFile(out, flag, 0644)
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+
+	_, err = output.Write(input)
+	return err
+
+}

libbeat/generator/fields/module_fields_collector.go

@@ -0,0 +1,74 @@
+package fields
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+var indentByModule = map[string]int{
+	"processors": 0,
+	"module":     8,
+	"active":     8,
+	"protos":     8,
+}
+
+// CollectModuleFiles looks for fields.yml files under the
+// specified root directory
+func CollectModuleFiles(root string) ([]*YmlFile, error) {
+	modules, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+
+	var files []*YmlFile
+	for _, m := range modules {
+		f, err := collectFiles(m, root)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, f...)
+	}
+
+	return files, nil
+}
+
+func collectFiles(module os.FileInfo, modulesPath string) ([]*YmlFile, error) {
+	if !module.IsDir() {
+		return nil, nil
+	}
+
+	var files []*YmlFile
+	fieldsYmlPath := filepath.Join(modulesPath, module.Name(), "_meta", "fields.yml")
+	if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
+		files = append(files, &YmlFile{
+			Path:   fieldsYmlPath,
+			Indent: 0,
+		})
+	} else if !os.IsNotExist(err) && err != nil {
+		return nil, err
+	}
+
+	modulesRoot := filepath.Base(modulesPath)
+	sets, err := ioutil.ReadDir(filepath.Join(modulesPath, module.Name()))
+	if err != nil {
+		return nil, err
+	}
+
+	for _, s := range sets {
+		if !s.IsDir() {
+			continue
+		}
+
+		fieldsYmlPath = filepath.Join(modulesPath, module.Name(), s.Name(), "_meta", "fields.yml")
+		if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
+			files = append(files, &YmlFile{
+				Path:   fieldsYmlPath,
+				Indent: indentByModule[modulesRoot],
+			})
+		} else if !os.IsNotExist(err) && err != nil {
+			return nil, err
+		}
+	}
+	return files, nil
+}

libbeat/scripts/Makefile

@@ -304,9 +304,19 @@ coverage-report:
 	test ! -s ${COVERAGE_DIR}/unit.cov   || go tool cover -html=${COVERAGE_DIR}/unit.cov   -o ${COVERAGE_DIR}/unit.html
 
 
+.PHONY: fields
+fields:
+	echo $(PWD)
+	@go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go --es_beats_path $(ES_BEATS) --beat_path $(PWD) $(FIELDS_FILE_PATH)
+
+
+.PHONY: libbeat_fields
+libbeat_fields:
+	@$(MAKE) -C ${ES_BEATS}/libbeat fields
+
 .PHONY: update
 update: ## @build Update expects the most recent version of libbeat in the GOPATH
-update: python-env collect
+update: python-env libbeat_fields collect
 	@echo "Updating generated files for ${BEAT_NAME}"
 
 	@# Update config files.
@@ -324,15 +334,7 @@ ifeq ($(BEAT_REF_YAML),true)
 	@chmod 0640 ${BEAT_NAME}.reference.yml
 endif
 
-	@# Update libbeat fields.generated.yml
-	@$(MAKE) -C ${ES_BEATS}/libbeat fields
-
-	@# Update fields.yml
-	@test -s  _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
-ifeq ($(BEAT_NAME), libbeat)
-	@cp ${ES_BEATS}/libbeat/_meta/fields.generated.yml fields.yml
-else
-	@cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > fields.yml
+ifneq ($(BEAT_NAME), libbeat)
 	mkdir -p include
 	go run  ${ES_BEATS}/dev-tools/cmd/asset/asset.go -pkg include fields.yml $(BEAT_NAME) > include/fields.go
 endif