Add a disable_host option to Filebeat inputs (#18159) (#18188)

This adds a configuration option `publisher_pipeline.disable_host` to disable the addition
of `host.name` in events. By default Filebeat adds `host.name` to all events and we want
to be able to disable this for data sources that do not originate on the host (like cloud logs).

Relates #13920

(cherry picked from commit 6ea21a9)

CHANGELOG.next.asciidoc

@@ -363,6 +363,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for Google Application Default Credentials to the Google Pub/Sub input and Google Cloud modules. {pull}15668[15668]
 - Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
 - Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108]
+- Added an input option `publisher_pipeline.disable_host` to disable `host.name`
+  from being added to events by default. {pull}18159[18159]
 - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065]
 
 *Heartbeat*

filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl

@@ -65,6 +65,11 @@ filebeat.inputs:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # By default, all events contain `host.name`. This option can be set to true
+  # to disable the addition of this field to all events. The default value is
+  # false.
+  #publisher_pipeline.disable_host: false
+
   # Ignore files which were modified more then the defined timespan in the past.
   # ignore_older is disabled by default, so no files are ignored by setting it to 0.
   # Time strings like 2h (2 hours), 5m (5 minutes) can be used.

filebeat/channel/connector.go

@@ -96,6 +96,7 @@ func (c *pipelineConnector) ConnectWith(cfg *common.Config, clientCfg beat.Clien
 	clientCfg.Processing.Fields = fields
 	clientCfg.Processing.Processor = procs
 	clientCfg.Processing.KeepNull = config.KeepNull
+	clientCfg.Processing.DisableHost = config.PublisherPipeline.DisableHost
 	client, err := c.pipeline.ConnectWith(clientCfg)
 	if err != nil {
 		return nil, err

filebeat/channel/factory.go

@@ -50,6 +50,10 @@ type inputOutletConfig struct {
 	Processors           processors.PluginConfig `config:"processors"`
 	KeepNull             bool                    `config:"keep_null"`
 
+	PublisherPipeline struct {
+		DisableHost bool `config:"disable_host"` // Disable addition of host.name.
+	} `config:"publisher_pipeline"`
+
 	// implicit event fields
 	Type        string `config:"type"`         // input.type
 	ServiceType string `config:"service.type"` // service.type

filebeat/docs/inputs/input-common-options.asciidoc

@@ -101,3 +101,9 @@ version and the event timestamp; for access to dynamic fields, use
 
 Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might
 expand to `"filebeat-myindex-2019.11.01"`.
+
+[float]
+===== `publisher_pipeline.disable_host`
+
+By default, all events contain `host.name`. This option can be set to `true` to
+disable the addition of this field to all events. The default value is `false`.

filebeat/filebeat.reference.yml

@@ -452,6 +452,11 @@ filebeat.inputs:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # By default, all events contain `host.name`. This option can be set to true
+  # to disable the addition of this field to all events. The default value is
+  # false.
+  #publisher_pipeline.disable_host: false
+
   # Ignore files which were modified more then the defined timespan in the past.
   # ignore_older is disabled by default, so no files are ignored by setting it to 0.
   # Time strings like 2h (2 hours), 5m (5 minutes) can be used.

filebeat/tests/system/config/filebeat.yml.j2

@@ -52,6 +52,11 @@ filebeat.{{input_config | default("inputs")}}:
     {{k}}: {{v}}
   {% endfor %}
   {% endif %}
+  {%- if publisher_pipeline %}
+  {%- for name, value in publisher_pipeline.items() %}
+  publisher_pipeline.{{name}}: {{value | tojson}}
+  {%- endfor %}
+  {% endif %}
 
   fields_under_root: {{"true" if fieldsUnderRoot else "false"}}
 

filebeat/tests/system/test_input.py

@@ -662,3 +662,23 @@ def test_disable_recursive_glob(self):
                 "recursive glob disabled"),
             max_timeout=10)
         filebeat.check_kill_and_wait()
+
+    def test_input_processing_pipeline_disable_host(self):
+        """
+        Check processing_pipeline.disable_host in input config.
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/test.log",
+            publisher_pipeline={
+                "disable_host": True,
+            },
+        )
+        with open(self.working_dir + "/test.log", "w") as f:
+            f.write("test message\n")
+
+        filebeat = self.start_beat()
+        self.wait_until(lambda: self.output_has(lines=1))
+        filebeat.check_kill_and_wait()
+
+        output = self.read_output()
+        assert "host.name" not in output[0]

libbeat/beat/pipeline.go

@@ -110,6 +110,9 @@ type ProcessingConfig struct {
 	// KeepNull determines whether published events will keep null values or omit them.
 	KeepNull bool
 
+	// Disables the addition of host.name if it was enabled for the publisher.
+	DisableHost bool
+
 	// Private contains additional information to be passed to the processing
 	// pipeline builder.
 	Private interface{}

libbeat/publisher/processing/default.go

@@ -269,6 +269,12 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor,
 	needsCopy := b.alwaysCopy || localProcessors != nil || b.processors != nil
 
 	builtin := b.builtinMeta
+	if cfg.DisableHost {
+		tmp := builtin.Clone()
+		tmp.Delete("host")
+		builtin = tmp
+	}
+
 	var clientFields common.MapStr
 	for _, mod := range b.modifiers {
 		m := mod.ClientFields(b.info, cfg)

x-pack/filebeat/filebeat.reference.yml

@@ -1115,6 +1115,11 @@ filebeat.inputs:
   # Set to true to publish fields with null values in events.
   #keep_null: false
 
+  # By default, all events contain `host.name`. This option can be set to true
+  # to disable the addition of this field to all events. The default value is
+  # false.
+  #publisher_pipeline.disable_host: false
+
   # Ignore files which were modified more then the defined timespan in the past.
   # ignore_older is disabled by default, so no files are ignored by setting it to 0.
   # Time strings like 2h (2 hours), 5m (5 minutes) can be used.