Cherrypick #10176 to 6.x branch: [Filebeat] Iptables / ubiquiti module

.gitattributes

@@ -1,5 +1,5 @@
-CHANGELOG.md  merge=union
-CHANGELOG.asciidoc  merge=union
+CHANGELOG.next.asciidoc  merge=union
+CHANGELOG-developer.next.asciidoc  merge=union
 
 # Keep these file types as CRLF (Windows).
 *.bat    text eol=crlf

.go-version

@@ -1 +1 @@
-1.10.3
+1.10.8

.travis.yml

@@ -12,7 +12,7 @@ env:
   global:
     # Cross-compile for amd64 only to speed up testing.
     - GOX_FLAGS="-arch amd64"
-    - DOCKER_COMPOSE_VERSION=1.11.1
+    - DOCKER_COMPOSE_VERSION=1.21.0
     - GO_VERSION="$(cat .go-version)"
     # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704
     - TRAVIS_MINIKUBE_VERSION=v0.25.2
@@ -34,6 +34,10 @@ jobs:
       env: TARGETS="TEST_ENVIRONMENT=0 -C filebeat testsuite"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/filebeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Heartbeat
     - os: linux
@@ -58,6 +62,10 @@ jobs:
       env: TARGETS="-C auditbeat crosscompile"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/auditbeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Libbeat
     - os: linux
@@ -72,6 +80,10 @@ jobs:
       env: STRESS_TEST_OPTIONS="-timeout=20m -race -v -parallel 1" TARGETS="-C libbeat stress-tests"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/libbeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Metricbeat
     - os: linux
@@ -99,6 +111,23 @@ jobs:
       go: $GO_VERSION
       stage: test
 
+    # Journalbeat
+    - os: linux
+      env: TARGETS="-C journalbeat testsuite"
+      go: $GO_VERSION
+      stage: test
+
+    # Functionbeat
+    - os: linux
+      env: TARGETS="-C x-pack/functionbeat testsuite"
+      go: $GO_VERSION
+      stage: test
+
+    - os: osx
+      env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite"
+      go: $GO_VERSION
+      stage: test
+
     # Generators
     - os: linux
       env: TARGETS="-C generator/metricbeat test"
@@ -109,6 +138,12 @@ jobs:
       go: $GO_VERSION
       stage: test
 
+    # Docs
+    - os: linux
+      env: TARGETS="docs"
+      go: $GO_VERSION
+      stage: test
+
     # Kubernetes
     - os: linux
       install: deploy/kubernetes/.travis/setup.sh
@@ -142,6 +177,9 @@ addons:
     packages:
       - python-virtualenv
       - libpcap-dev
+      - xsltproc
+      - libxml2-utils
+      - libsystemd-journal-dev
 
 before_install:
   - python --version
@@ -152,6 +190,8 @@ before_install:
   - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
   - chmod +x docker-compose
   - sudo mv docker-compose /usr/local/bin
+  - if [ $TRAVIS_OS_NAME = osx ]; then pip install virtualenv; fi
+
 
 # Skips installations step
 install: true

CHANGELOG-developer.asciidoc

@@ -13,9 +13,9 @@ other Beats should be migrated.
 Note: This changelog was only started after the 6.3 release.
 
 === Beats version HEAD
-https://github.com/elastic/beats/compare/v6.3.0..master[Check the HEAD diff]
+https://github.com/elastic/beats/compare/v6.3.0..6.x[Check the HEAD diff]
 
-The list below covers the major changes between 6.3.0 and master only.
+The list below covers the major changes between 6.3.0 and HEAD only.
 
 ==== Breaking changes
 
@@ -29,13 +29,37 @@ The list below covers the major changes between 6.3.0 and master only.
   the building a Beat more cross-platform friendly (e.g. Windows). This requires that your Beat
   has a magefile.go with a fields target. The `FIELDS_FILE_PATH` make variable is no longer
   used because the value is specified in magefile.go. {pull}7670[7670]
+- Renamed `-beat-name` CLI option used in `kibana_index_pattern.go` to `-beat` for consistency with other scripts in `dev-tools/cmd`. {pull}8615[8615]
 
 ==== Bugfixes
 
 - Fix permissions of generated Filebeat filesets. {pull}7140[7140]
+- Collect fields from _meta/fields.yml too. {pull}8397[8397]
+- Fix issue on asset generation that could lead to different results in Windows. {pull}8464[8464]
 
 ==== Added
 
 - Libbeat provides a global registry for beats developer that allow to register and retrieve plugin. {pull}7392[7392]
 - Added more options to control required and optional fields in schema.Apply(), error returned is a plain nil if no error happened {pull}7335[7335]
 - Packaging on MacOS now produces a .dmg file containing an installer (.pkg) and uninstaller for the Beat. {pull}7481[7481]
+- Added mage targets `goTestUnit` and `goTestIntegration` for executing
+  'go test'. This captures the log to a file, summarizes the result, produces a
+  coverage profile (.cov), and produces an HTML coverage report. See
+  `mage -h goTestUnit`. {pull}7766[7766]
+- Beats packaging now build non-oss binaries from code located in the x-pack folder. {issue}7783[7783]
+- New function `AddTagsWithKey` is added, so `common.MapStr` can be enriched with tags with an arbitrary key. {pull}7991[7991]
+- Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions
+  `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
+- Set current year in generator templates. {pull}8396[8396]
+- You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
+- Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942]
+- Add `-name` option to `asset.go` script to explicitly name the asset rather than using its filename. {pull}8693[8693]
+- Add `-out` option to `kibana_index_pattern.go` to control the output dir to make it possible to write the generated output to `build/kibana` instead of `_meta/kibana.generated` (but the output dir remains unchanged at this point). {pull}8615[8615]
+- Add `module_fields.go` for generated `fields.go` files for modules. {pull}8615[8615]
+- Add `mage.GenerateModuleReferenceConfig` for generating reference config files that include configuration sections from the module directory. {pull}8615[8615]
+- Add `mage.GenerateFieldsGo` for generating fields.go files. {pull}8615[8615]
+- Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615]
+- Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769]
+- Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889]
+- Add docker image building to `mage.Package`. {pull}8898[8898]
+- Filesets can now define multiple ingest pipelines, with the first one considered as the entry point pipeline. {pull}8914[8914]

CHANGELOG-developer.next.asciidoc

@@ -0,0 +1,27 @@
+// Use these for links to issue and pulls. Note issues and pulls redirect one to
+// each other on Github, so don't worry too much on using the right prefix.
+:issue: https://github.com/elastic/beats/issues/
+:pull: https://github.com/elastic/beats/pull/
+
+This changelog is intended for community Beat developers. It covers the major
+breaking changes to the internal APIs in the official Beats and changes related
+to developing a Beat like code generators or `fields.yml`. Only the major
+changes will be covered in this changelog that are expected to affect community
+developers. Each breaking change added here should have an explanation on how
+other Beats should be migrated.
+
+Note: This changelog documents the current changes which are not yet present in
+an actual release.
+
+=== Beats version HEAD
+https://github.com/elastic/beats/compare/v6.6.0..6.x[Check the HEAD diff]
+
+The list below covers the major changes between 6.6.0 and 6.x only.
+
+==== Breaking changes
+
+==== Bugfixes
+
+==== Added
+
+- Allow multiple object type configurations per field. {pull}9772[9772]