From 6fa2cd40d017046bc80e41aa419bc129ff4b4e96 Mon Sep 17 00:00:00 2001
From: ruflin <spam@ruflin.com>
Date: Mon, 27 Feb 2017 12:00:00 +0100
Subject: [PATCH] Unify fields generated file across all beats

Each beat has its own way of generating the fields.yml file. Some have `_meta/fields.yml`, others `_meta/fields.generated.yml`. All scripts that used the fields.yml file had to do checks for these files and include the `libbeat/_meta/fields.generate.yml` file to get all data in. Now all beats will create `_meta/fields.generated.yml`. In case it does not exist, it is just a copy of `_meta/fields.yml` and will be automatically generated. In addition the file `_meta/fields.full.generated.yml` was introduced, which also contains the content from the `libbeat/_meta/fields.generate.yml`. This file can be used in most scripts to get all fields. The scripts were changed / simplified accordingly.

This simplifies the implementation of dynamic template loading as only 1 file has to be taken into account.

Part of https://github.com/elastic/beats/issues/3654
---
 libbeat/scripts/Makefile                  | 12 +++++++++---
 libbeat/scripts/generate_fields_docs.py   | 10 +---------
 libbeat/scripts/generate_index_pattern.py | 11 +----------
 3 files changed, 11 insertions(+), 22 deletions(-)

diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile
index d92f4c3889b..a9428694ad4 100755
--- a/libbeat/scripts/Makefile
+++ b/libbeat/scripts/Makefile
@@ -244,14 +244,20 @@ update: python-env collect
 	# make sure generated docs in libbeat are up-to-date
 	$(MAKE) -C ${ES_BEATS}/libbeat fields
 
+	# Generate full fields for each beat. Create fields.generated.yml in case it does not exist yet.
+	# Most beats have their own way of generating it.
+	test -s  _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
+	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > _meta/fields.full.generated.yml
+	# Revert changes to libbeat fields as it otherwise contains duplicates in full
+	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/_meta/fields.full.generated.yml
+
 	# Update docs
 	mkdir -p docs
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS}
 
 	# Generate index templates
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${PWD}/${ES_BEATS}/libbeat/_meta/fields.generated.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.generated.yml
-
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${PWD}/${ES_BEATS}/libbeat/_meta/fields.generated.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.generated.yml
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
 
 	# Generate index-pattern
 	echo "Generate index pattern"
diff --git a/libbeat/scripts/generate_fields_docs.py b/libbeat/scripts/generate_fields_docs.py
index bfa11e77bd0..7b9d82b5a15 100644
--- a/libbeat/scripts/generate_fields_docs.py
+++ b/libbeat/scripts/generate_fields_docs.py
@@ -111,20 +111,12 @@ def fields_to_asciidoc(input, output, beat):
     beat_name = args.beatname
     es_beats = args.es_beats
 
-    fields_yml = beat_path + "/_meta/fields.generated.yml"
-
-    # Not all beats have a fields.generated.yml. Fall back to fields.yml
-    if not os.path.isfile(fields_yml):
-        fields_yml = beat_path + "/_meta/fields.yml"
+    fields_yml = beat_path + "/_meta/fields.full.generated.yml"
 
     # Read fields.yml
     with open(fields_yml) as f:
         fields = f.read()
 
-    # Prepends beat fields from libbeat
-    with open(es_beats + "/libbeat/_meta/fields.generated.yml") as f:
-        fields = f.read() + fields
-
     output = open(beat_path + "/docs/fields.asciidoc", 'w')
 
     try:
diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py
index 95408180597..c7c2735de5a 100644
--- a/libbeat/scripts/generate_index_pattern.py
+++ b/libbeat/scripts/generate_index_pattern.py
@@ -139,21 +139,12 @@ def get_index_pattern_name(index):
 
     args = parser.parse_args()
 
-    fields_yml = args.beat + "/_meta/fields.generated.yml"
-
-    # Not all beats have a fields.generated.yml. Fall back to fields.yml
-    if not os.path.isfile(fields_yml):
-        fields_yml = args.beat + "/_meta/fields.yml"
+    fields_yml = args.beat + "/_meta/fields.full.generated.yml"
 
     # generate the index-pattern content
     with open(fields_yml, 'r') as f:
         fields = f.read()
 
-        if os.path.basename(args.beat) != "libbeat":
-            # Prepend beat fields from libbeat
-            with open(args.libbeat + "/_meta/fields.generated.yml") as f:
-                fields = f.read() + fields
-
         # with open(target, 'w') as output:
         output = fields_to_index_pattern(args, fields)