From 1540360ee671f3c21032db2941d47eff5b7205ca Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 27 Aug 2018 12:06:39 -0700 Subject: [PATCH] Backport to 6.3: Doc changes (#7883 #7889 #7950) (#8013) * Add missing config options to Logstash section of reference.yml (#7883) * Add safeguard related statements for max_backoff setting (#7889) * Add recommendation to avoid harvesting symlinks when dealing with file rotation (#7950) --- auditbeat/auditbeat.reference.yml | 15 +++++++++++++++ .../inputs/input-common-file-options.asciidoc | 7 ++++--- filebeat/docs/inputs/input-log.asciidoc | 4 ++++ filebeat/filebeat.reference.yml | 15 +++++++++++++++ heartbeat/heartbeat.reference.yml | 15 +++++++++++++++ libbeat/_meta/config.reference.yml | 15 +++++++++++++++ libbeat/docs/outputconfig.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 15 +++++++++++++++ packetbeat/packetbeat.reference.yml | 15 +++++++++++++++ winlogbeat/winlogbeat.reference.yml | 15 +++++++++++++++ 10 files changed, 114 insertions(+), 4 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 08f6dc199484..854455eaf5b4 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -502,6 +502,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. diff --git a/filebeat/docs/inputs/input-common-file-options.asciidoc b/filebeat/docs/inputs/input-common-file-options.asciidoc index bebdbfca928e..ee663f7e47aa 100644 --- a/filebeat/docs/inputs/input-common-file-options.asciidoc +++ b/filebeat/docs/inputs/input-common-file-options.asciidoc @@ -348,9 +348,10 @@ specifying 10s for `max_backoff` means that, at the worst, a new line could be added to the log file if {beatname_uc} has backed off multiple times. The default is 10s. -Requirement: max_backoff should always be set to `max_backoff <= -scan_frequency`. In case `max_backoff` should be bigger, it is recommended to -close the file handler instead let the {beatname_uc} pick up the file again. +Requirement: Set `max_backoff` to be greater than or equal to `backoff` and +less than or equal to `scan_frequency` (`backoff <= max_backoff <= scan_frequency`). +If `max_backoff` needs to be higher, it is recommended to close the file handler +instead and let {beatname_uc} pick up the file again. [float] ===== `backoff_factor` diff --git a/filebeat/docs/inputs/input-log.asciidoc b/filebeat/docs/inputs/input-log.asciidoc index 7c73c63d396c..2e0fdf212fda 100644 --- a/filebeat/docs/inputs/input-log.asciidoc +++ b/filebeat/docs/inputs/input-log.asciidoc @@ -57,6 +57,10 @@ multiple input sections: IMPORTANT: Make sure a file is not defined more than once across all inputs because this can lead to unexpected behaviour. +NOTE: When dealing with file rotation, avoid harvesting symlinks. Instead +use the <> setting to point to the original file, and specify +a pattern that matches the file you want to harvest and all of its rotated +files. [id="{beatname_lc}-input-{type}-options"] ==== Configuration options diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 96f817e99b94..e237717cfdb9 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1044,6 +1044,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e6a948ba8f9d..0ce4944e93cd 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -611,6 +611,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index d283b8ec1749..1d792c7a0bb3 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -397,6 +397,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 82aedd38af54..e11ee42a3f32 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -514,7 +514,7 @@ configured. The default value is 2. deprecated[5.0.0] The default port to use if the port number is not given in <>. The default port number -is 10200. +is 5044. ===== `proxy_url` diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index a775103e902f..99c108b26b27 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -998,6 +998,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index e5060bef7494..cd149b5c9421 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -874,6 +874,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 46bde09bf3b2..d59c56b73b36 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -426,6 +426,21 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module.