From c42bf99cc047f3ee37a54b1bf823a7bccda3660a Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 10 Dec 2019 16:54:45 +0700 Subject: [PATCH 1/8] Add default_field option for Beats generator This updates the Beats generator to undestand the `default_field` option and add it to the generated yaml, as well as update the fields added by 1.3.0 to include the `default_field` tag. --- CHANGELOG.next.md | 1 + generated/beats/fields.ecs.yml | 16 ++++++++++++++++ generated/ecs/ecs_flat.yml | 16 ++++++++++++++++ generated/ecs/ecs_nested.yml | 18 ++++++++++++++++++ schemas/process.yml | 16 ++++++++++++++++ schemas/tls.yml | 1 + schemas/vulnerability.yml | 1 + scripts/generators/beats.py | 3 ++- 8 files changed, 71 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 90dfe91226..d2bf6a8f6e 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -22,6 +22,7 @@ Thanks, you're awesome :-) --> * Added `rule` fields. #665 * Added default `text` analyzer as a multi-field to around 25 more fields. #680 * Added `registry.*` fieldset for the Windows registry. #673 +* Added the `default_field` option for Beats. #NNN #### Improvements diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 8c39f9a22d..25250991d9 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -2329,6 +2329,7 @@ many arguments were provided to start a process. More arguments may be an indication of suspicious activity.' example: 4 + default_field: false - name: command_line level: extended type: keyword @@ -2342,6 +2343,7 @@ Some arguments may be filtered to protect sensitive information.' example: /usr/bin/ssh -l user 10.0.0.16 + default_field: false - name: executable level: extended type: keyword @@ -2360,6 +2362,7 @@ The field should be absent if there is no exit code for the event (e.g. process start).' example: 137 + default_field: false - name: hash.md5 level: extended type: keyword @@ -2404,6 +2407,7 @@ - -l - user - 10.0.0.16 + default_field: false - name: parent.args_count level: extended type: long @@ -2413,6 +2417,7 @@ many arguments were provided to start a process. More arguments may be an indication of suspicious activity.' example: 4 + default_field: false - name: parent.command_line level: extended type: keyword @@ -2426,6 +2431,7 @@ Some arguments may be filtered to protect sensitive information.' example: /usr/bin/ssh -l user 10.0.0.16 + default_field: false - name: parent.executable level: extended type: keyword @@ -2436,6 +2442,7 @@ norms: false description: Absolute path to the process executable. example: /usr/bin/ssh + default_field: false - name: parent.exit_code level: extended type: long @@ -2456,11 +2463,13 @@ Sometimes called program name or similar.' example: ssh + default_field: false - name: parent.pgid level: extended type: long format: string description: Identifier of the group of processes the process belongs to. + default_field: false - name: parent.pid level: core type: long @@ -2473,23 +2482,27 @@ format: string description: Parent process' pid. example: 4241 + default_field: false - name: parent.start level: extended type: date description: The time the process started. example: '2016-05-23T08:05:34.853Z' + default_field: false - name: parent.thread.id level: extended type: long format: string description: Thread ID. example: 4242 + default_field: false - name: parent.thread.name level: extended type: keyword ignore_above: 1024 description: Thread name. example: thread-0 + default_field: false - name: parent.title level: extended type: keyword @@ -2502,11 +2515,13 @@ The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.' + default_field: false - name: parent.uptime level: extended type: long description: Seconds the process has been up. example: 1325 + default_field: false - name: parent.working_directory level: extended type: keyword @@ -2517,6 +2532,7 @@ norms: false description: The working directory of the process. example: /home/alice + default_field: false - name: pgid level: extended type: long diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index d9ffea8e34..d0c9ade0b0 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -3610,6 +3610,7 @@ process.args: type: keyword process.args_count: dashed_name: process-args-count + default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many @@ -3624,6 +3625,7 @@ process.args_count: type: long process.command_line: dashed_name: process-command-line + default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -3659,6 +3661,7 @@ process.executable: type: keyword process.exit_code: dashed_name: process-exit-code + default_field: false description: 'The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process @@ -3734,6 +3737,7 @@ process.name: type: keyword process.parent.args: dashed_name: process-parent-args + default_field: false description: 'Array of process arguments. May be filtered to protect sensitive information.' @@ -3751,6 +3755,7 @@ process.parent.args: type: keyword process.parent.args_count: dashed_name: process-parent-args-count + default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many @@ -3765,6 +3770,7 @@ process.parent.args_count: type: long process.parent.command_line: dashed_name: process-parent-command-line + default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -3784,6 +3790,7 @@ process.parent.command_line: type: keyword process.parent.executable: dashed_name: process-parent-executable + default_field: false description: Absolute path to the process executable. example: /usr/bin/ssh flat_name: process.parent.executable @@ -3813,6 +3820,7 @@ process.parent.exit_code: type: long process.parent.name: dashed_name: process-parent-name + default_field: false description: 'Process name. Sometimes called program name or similar.' @@ -3831,6 +3839,7 @@ process.parent.name: type: keyword process.parent.pgid: dashed_name: process-parent-pgid + default_field: false description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid format: string @@ -3852,6 +3861,7 @@ process.parent.pid: type: long process.parent.ppid: dashed_name: process-parent-ppid + default_field: false description: Parent process' pid. example: 4241 flat_name: process.parent.ppid @@ -3863,6 +3873,7 @@ process.parent.ppid: type: long process.parent.start: dashed_name: process-parent-start + default_field: false description: The time the process started. example: '2016-05-23T08:05:34.853Z' flat_name: process.parent.start @@ -3873,6 +3884,7 @@ process.parent.start: type: date process.parent.thread.id: dashed_name: process-parent-thread-id + default_field: false description: Thread ID. example: 4242 flat_name: process.parent.thread.id @@ -3884,6 +3896,7 @@ process.parent.thread.id: type: long process.parent.thread.name: dashed_name: process-parent-thread-name + default_field: false description: Thread name. example: thread-0 flat_name: process.parent.thread.name @@ -3895,6 +3908,7 @@ process.parent.thread.name: type: keyword process.parent.title: dashed_name: process-parent-title + default_field: false description: 'Process title. The proctitle, some times the same as process name. Can also be different: for @@ -3913,6 +3927,7 @@ process.parent.title: type: keyword process.parent.uptime: dashed_name: process-parent-uptime + default_field: false description: Seconds the process has been up. example: 1325 flat_name: process.parent.uptime @@ -3923,6 +3938,7 @@ process.parent.uptime: type: long process.parent.working_directory: dashed_name: process-parent-working-directory + default_field: false description: The working directory of the process. example: /home/alice flat_name: process.parent.working_directory diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index d143132f2b..af81e689a8 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -4002,6 +4002,7 @@ process: type: keyword args_count: dashed_name: process-args-count + default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how @@ -4016,6 +4017,7 @@ process: type: long command_line: dashed_name: process-command-line + default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4051,6 +4053,7 @@ process: type: keyword exit_code: dashed_name: process-exit-code + default_field: false description: 'The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process @@ -4126,6 +4129,7 @@ process: type: keyword parent.args: dashed_name: process-parent-args + default_field: false description: 'Array of process arguments. May be filtered to protect sensitive information.' @@ -4143,6 +4147,7 @@ process: type: keyword parent.args_count: dashed_name: process-parent-args-count + default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how @@ -4157,6 +4162,7 @@ process: type: long parent.command_line: dashed_name: process-parent-command-line + default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4176,6 +4182,7 @@ process: type: keyword parent.executable: dashed_name: process-parent-executable + default_field: false description: Absolute path to the process executable. example: /usr/bin/ssh flat_name: process.parent.executable @@ -4205,6 +4212,7 @@ process: type: long parent.name: dashed_name: process-parent-name + default_field: false description: 'Process name. Sometimes called program name or similar.' @@ -4223,6 +4231,7 @@ process: type: keyword parent.pgid: dashed_name: process-parent-pgid + default_field: false description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid format: string @@ -4244,6 +4253,7 @@ process: type: long parent.ppid: dashed_name: process-parent-ppid + default_field: false description: Parent process' pid. example: 4241 flat_name: process.parent.ppid @@ -4255,6 +4265,7 @@ process: type: long parent.start: dashed_name: process-parent-start + default_field: false description: The time the process started. example: '2016-05-23T08:05:34.853Z' flat_name: process.parent.start @@ -4265,6 +4276,7 @@ process: type: date parent.thread.id: dashed_name: process-parent-thread-id + default_field: false description: Thread ID. example: 4242 flat_name: process.parent.thread.id @@ -4276,6 +4288,7 @@ process: type: long parent.thread.name: dashed_name: process-parent-thread-name + default_field: false description: Thread name. example: thread-0 flat_name: process.parent.thread.name @@ -4287,6 +4300,7 @@ process: type: keyword parent.title: dashed_name: process-parent-title + default_field: false description: 'Process title. The proctitle, some times the same as process name. Can also be different: @@ -4305,6 +4319,7 @@ process: type: keyword parent.uptime: dashed_name: process-parent-uptime + default_field: false description: Seconds the process has been up. example: 1325 flat_name: process.parent.uptime @@ -4315,6 +4330,7 @@ process: type: long parent.working_directory: dashed_name: process-parent-working-directory + default_field: false description: The working directory of the process. example: /home/alice flat_name: process.parent.working_directory @@ -5736,6 +5752,7 @@ threat: title: Threat type: group tls: + default_fields: false description: Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. @@ -6648,6 +6665,7 @@ user_agent: title: User agent type: group vulnerability: + default_field: false description: The vulnerability fields describe information about a vulnerability that is relevant to an event. fields: diff --git a/schemas/process.yml b/schemas/process.yml index bc9450f66d..e499554299 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -60,6 +60,7 @@ level: extended type: keyword short: Process name. + default_field: false description: > Process name. @@ -82,6 +83,7 @@ format: string level: extended type: long + default_field: false description: > Parent process' pid. example: 4241 @@ -98,6 +100,7 @@ format: string level: extended type: long + default_field: false description: > Identifier of the group of processes the process belongs to. @@ -106,6 +109,7 @@ level: extended type: keyword short: Full command line that started the process. + default_field: false description: > Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -120,6 +124,7 @@ level: extended type: keyword short: Full command line that started the process. + default_field: false description: > Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -145,6 +150,7 @@ level: extended type: keyword short: Array of process arguments. + default_field: false description: > Array of process arguments. @@ -155,6 +161,7 @@ level: extended type: long short: Length of the process.args array. + default_field: false description: > Length of the process.args array. @@ -167,6 +174,7 @@ level: extended type: long short: Length of the process.args array. + default_field: false description: > Length of the process.args array. @@ -189,6 +197,7 @@ - name: parent.executable level: extended type: keyword + default_field: false description: > Absolute path to the process executable. example: /usr/bin/ssh @@ -214,6 +223,7 @@ level: extended type: keyword short: Process title. + default_field: false description: > Process title. @@ -237,6 +247,7 @@ level: extended type: long example: 4242 + default_field: false description: > Thread ID. @@ -251,6 +262,7 @@ - name: parent.thread.name level: extended type: keyword + default_field: false example: 'thread-0' description: > Thread name. @@ -267,6 +279,7 @@ level: extended type: date example: "2016-05-23T08:05:34.853Z" + default_field: false description: > The time the process started. @@ -282,6 +295,7 @@ level: extended type: long example: 1325 + default_field: false description: > Seconds the process has been up. @@ -300,6 +314,7 @@ level: extended type: keyword example: /home/alice + default_field: false description: > The working directory of the process. multi_fields: @@ -312,6 +327,7 @@ type: long example: 137 short: The exit code of the process. + default_field: false description: > The exit code of the process, if this is a termination event. diff --git a/schemas/tls.yml b/schemas/tls.yml index 30d2f002fa..1645dff6b9 100644 --- a/schemas/tls.yml +++ b/schemas/tls.yml @@ -7,6 +7,7 @@ Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. type: group + default_fields: false fields: - name: version diff --git a/schemas/vulnerability.yml b/schemas/vulnerability.yml index d67f8e61e1..4f5d10b793 100644 --- a/schemas/vulnerability.yml +++ b/schemas/vulnerability.yml @@ -2,6 +2,7 @@ - name: vulnerability title: Vulnerability group: 2 + default_field: false short: Fields to describe the vulnerability relevant to an event. description: > The vulnerability fields describe information about a vulnerability that is diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index fcbc420bda..9864bbd33a 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -31,7 +31,8 @@ def generate(ecs_nested, ecs_version): def fieldset_field_array(source_fields): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', 'ignore_above', 'multi_fields', 'format', 'input_format', - 'output_format', 'output_precision', 'description', 'example'] + 'output_format', 'output_precision', 'description', + 'example', 'default_field'] multi_fields_allowed_keys = ['name', 'type', 'norms'] fields = [] From 6b9bfb0f660d777aff89e5b593a0db1482960b58 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 12 Dec 2019 16:53:25 +0100 Subject: [PATCH 2/8] Update CHANGELOG.next.md Co-Authored-By: Mathieu Martin --- CHANGELOG.next.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index d2bf6a8f6e..5011835016 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -22,7 +22,7 @@ Thanks, you're awesome :-) --> * Added `rule` fields. #665 * Added default `text` analyzer as a multi-field to around 25 more fields. #680 * Added `registry.*` fieldset for the Windows registry. #673 -* Added the `default_field` option for Beats. #NNN +* Added the `default_field` option for Beats. #687 #### Improvements From 433f8b1fceb6bf59f700b14e20d4c624fc12ae91 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 12 Dec 2019 17:17:22 +0100 Subject: [PATCH 3/8] Rename option to beats.default_field --- CHANGELOG.next.md | 2 +- generated/ecs/ecs_flat.yml | 32 +++++++++++++-------------- generated/ecs/ecs_nested.yml | 36 +++++++++++++++---------------- schemas/process.yml | 32 +++++++++++++-------------- schemas/tls.yml | 2 +- schemas/vulnerability.yml | 2 +- scripts/generators/beats.py | 6 +++++- scripts/generators/ecs_helpers.py | 8 +++++++ 8 files changed, 66 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 5011835016..d12f3962f0 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -22,7 +22,7 @@ Thanks, you're awesome :-) --> * Added `rule` fields. #665 * Added default `text` analyzer as a multi-field to around 25 more fields. #680 * Added `registry.*` fieldset for the Windows registry. #673 -* Added the `default_field` option for Beats. #687 +* Added the `beats.default_field` option for Beats. #687 #### Improvements diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index d0c9ade0b0..4a06b6a1d9 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -3609,8 +3609,8 @@ process.args: short: Array of process arguments. type: keyword process.args_count: + beats.default_field: false dashed_name: process-args-count - default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many @@ -3624,8 +3624,8 @@ process.args_count: short: Length of the process.args array. type: long process.command_line: + beats.default_field: false dashed_name: process-command-line - default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -3660,8 +3660,8 @@ process.executable: short: Absolute path to the process executable. type: keyword process.exit_code: + beats.default_field: false dashed_name: process-exit-code - default_field: false description: 'The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process @@ -3736,8 +3736,8 @@ process.name: short: Process name. type: keyword process.parent.args: + beats.default_field: false dashed_name: process-parent-args - default_field: false description: 'Array of process arguments. May be filtered to protect sensitive information.' @@ -3754,8 +3754,8 @@ process.parent.args: short: Array of process arguments. type: keyword process.parent.args_count: + beats.default_field: false dashed_name: process-parent-args-count - default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many @@ -3769,8 +3769,8 @@ process.parent.args_count: short: Length of the process.args array. type: long process.parent.command_line: + beats.default_field: false dashed_name: process-parent-command-line - default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -3789,8 +3789,8 @@ process.parent.command_line: short: Full command line that started the process. type: keyword process.parent.executable: + beats.default_field: false dashed_name: process-parent-executable - default_field: false description: Absolute path to the process executable. example: /usr/bin/ssh flat_name: process.parent.executable @@ -3819,8 +3819,8 @@ process.parent.exit_code: short: The exit code of the process. type: long process.parent.name: + beats.default_field: false dashed_name: process-parent-name - default_field: false description: 'Process name. Sometimes called program name or similar.' @@ -3838,8 +3838,8 @@ process.parent.name: short: Process name. type: keyword process.parent.pgid: + beats.default_field: false dashed_name: process-parent-pgid - default_field: false description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid format: string @@ -3860,8 +3860,8 @@ process.parent.pid: short: Process id. type: long process.parent.ppid: + beats.default_field: false dashed_name: process-parent-ppid - default_field: false description: Parent process' pid. example: 4241 flat_name: process.parent.ppid @@ -3872,8 +3872,8 @@ process.parent.ppid: short: Parent process' pid. type: long process.parent.start: + beats.default_field: false dashed_name: process-parent-start - default_field: false description: The time the process started. example: '2016-05-23T08:05:34.853Z' flat_name: process.parent.start @@ -3883,8 +3883,8 @@ process.parent.start: short: The time the process started. type: date process.parent.thread.id: + beats.default_field: false dashed_name: process-parent-thread-id - default_field: false description: Thread ID. example: 4242 flat_name: process.parent.thread.id @@ -3895,8 +3895,8 @@ process.parent.thread.id: short: Thread ID. type: long process.parent.thread.name: + beats.default_field: false dashed_name: process-parent-thread-name - default_field: false description: Thread name. example: thread-0 flat_name: process.parent.thread.name @@ -3907,8 +3907,8 @@ process.parent.thread.name: short: Thread name. type: keyword process.parent.title: + beats.default_field: false dashed_name: process-parent-title - default_field: false description: 'Process title. The proctitle, some times the same as process name. Can also be different: for @@ -3926,8 +3926,8 @@ process.parent.title: short: Process title. type: keyword process.parent.uptime: + beats.default_field: false dashed_name: process-parent-uptime - default_field: false description: Seconds the process has been up. example: 1325 flat_name: process.parent.uptime @@ -3937,8 +3937,8 @@ process.parent.uptime: short: Seconds the process has been up. type: long process.parent.working_directory: + beats.default_field: false dashed_name: process-parent-working-directory - default_field: false description: The working directory of the process. example: /home/alice flat_name: process.parent.working_directory diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index af81e689a8..c9f21129a4 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -4001,8 +4001,8 @@ process: short: Array of process arguments. type: keyword args_count: + beats.default_field: false dashed_name: process-args-count - default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how @@ -4016,8 +4016,8 @@ process: short: Length of the process.args array. type: long command_line: + beats.default_field: false dashed_name: process-command-line - default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4052,8 +4052,8 @@ process: short: Absolute path to the process executable. type: keyword exit_code: + beats.default_field: false dashed_name: process-exit-code - default_field: false description: 'The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process @@ -4128,8 +4128,8 @@ process: short: Process name. type: keyword parent.args: + beats.default_field: false dashed_name: process-parent-args - default_field: false description: 'Array of process arguments. May be filtered to protect sensitive information.' @@ -4146,8 +4146,8 @@ process: short: Array of process arguments. type: keyword parent.args_count: + beats.default_field: false dashed_name: process-parent-args-count - default_field: false description: 'Length of the process.args array. This field can be useful for querying or performing bucket analysis on how @@ -4161,8 +4161,8 @@ process: short: Length of the process.args array. type: long parent.command_line: + beats.default_field: false dashed_name: process-parent-command-line - default_field: false description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4181,8 +4181,8 @@ process: short: Full command line that started the process. type: keyword parent.executable: + beats.default_field: false dashed_name: process-parent-executable - default_field: false description: Absolute path to the process executable. example: /usr/bin/ssh flat_name: process.parent.executable @@ -4211,8 +4211,8 @@ process: short: The exit code of the process. type: long parent.name: + beats.default_field: false dashed_name: process-parent-name - default_field: false description: 'Process name. Sometimes called program name or similar.' @@ -4230,8 +4230,8 @@ process: short: Process name. type: keyword parent.pgid: + beats.default_field: false dashed_name: process-parent-pgid - default_field: false description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid format: string @@ -4252,8 +4252,8 @@ process: short: Process id. type: long parent.ppid: + beats.default_field: false dashed_name: process-parent-ppid - default_field: false description: Parent process' pid. example: 4241 flat_name: process.parent.ppid @@ -4264,8 +4264,8 @@ process: short: Parent process' pid. type: long parent.start: + beats.default_field: false dashed_name: process-parent-start - default_field: false description: The time the process started. example: '2016-05-23T08:05:34.853Z' flat_name: process.parent.start @@ -4275,8 +4275,8 @@ process: short: The time the process started. type: date parent.thread.id: + beats.default_field: false dashed_name: process-parent-thread-id - default_field: false description: Thread ID. example: 4242 flat_name: process.parent.thread.id @@ -4287,8 +4287,8 @@ process: short: Thread ID. type: long parent.thread.name: + beats.default_field: false dashed_name: process-parent-thread-name - default_field: false description: Thread name. example: thread-0 flat_name: process.parent.thread.name @@ -4299,8 +4299,8 @@ process: short: Thread name. type: keyword parent.title: + beats.default_field: false dashed_name: process-parent-title - default_field: false description: 'Process title. The proctitle, some times the same as process name. Can also be different: @@ -4318,8 +4318,8 @@ process: short: Process title. type: keyword parent.uptime: + beats.default_field: false dashed_name: process-parent-uptime - default_field: false description: Seconds the process has been up. example: 1325 flat_name: process.parent.uptime @@ -4329,8 +4329,8 @@ process: short: Seconds the process has been up. type: long parent.working_directory: + beats.default_field: false dashed_name: process-parent-working-directory - default_field: false description: The working directory of the process. example: /home/alice flat_name: process.parent.working_directory @@ -5752,7 +5752,7 @@ threat: title: Threat type: group tls: - default_fields: false + beats.default_field: false description: Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. @@ -6665,7 +6665,7 @@ user_agent: title: User agent type: group vulnerability: - default_field: false + beats.default_field: false description: The vulnerability fields describe information about a vulnerability that is relevant to an event. fields: diff --git a/schemas/process.yml b/schemas/process.yml index e499554299..210cdc7379 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -60,7 +60,7 @@ level: extended type: keyword short: Process name. - default_field: false + beats.default_field: false description: > Process name. @@ -83,7 +83,7 @@ format: string level: extended type: long - default_field: false + beats.default_field: false description: > Parent process' pid. example: 4241 @@ -100,7 +100,7 @@ format: string level: extended type: long - default_field: false + beats.default_field: false description: > Identifier of the group of processes the process belongs to. @@ -109,7 +109,7 @@ level: extended type: keyword short: Full command line that started the process. - default_field: false + beats.default_field: false description: > Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -124,7 +124,7 @@ level: extended type: keyword short: Full command line that started the process. - default_field: false + beats.default_field: false description: > Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -150,7 +150,7 @@ level: extended type: keyword short: Array of process arguments. - default_field: false + beats.default_field: false description: > Array of process arguments. @@ -161,7 +161,7 @@ level: extended type: long short: Length of the process.args array. - default_field: false + beats.default_field: false description: > Length of the process.args array. @@ -174,7 +174,7 @@ level: extended type: long short: Length of the process.args array. - default_field: false + beats.default_field: false description: > Length of the process.args array. @@ -197,7 +197,7 @@ - name: parent.executable level: extended type: keyword - default_field: false + beats.default_field: false description: > Absolute path to the process executable. example: /usr/bin/ssh @@ -223,7 +223,7 @@ level: extended type: keyword short: Process title. - default_field: false + beats.default_field: false description: > Process title. @@ -247,7 +247,7 @@ level: extended type: long example: 4242 - default_field: false + beats.default_field: false description: > Thread ID. @@ -262,7 +262,7 @@ - name: parent.thread.name level: extended type: keyword - default_field: false + beats.default_field: false example: 'thread-0' description: > Thread name. @@ -279,7 +279,7 @@ level: extended type: date example: "2016-05-23T08:05:34.853Z" - default_field: false + beats.default_field: false description: > The time the process started. @@ -295,7 +295,7 @@ level: extended type: long example: 1325 - default_field: false + beats.default_field: false description: > Seconds the process has been up. @@ -314,7 +314,7 @@ level: extended type: keyword example: /home/alice - default_field: false + beats.default_field: false description: > The working directory of the process. multi_fields: @@ -327,7 +327,7 @@ type: long example: 137 short: The exit code of the process. - default_field: false + beats.default_field: false description: > The exit code of the process, if this is a termination event. diff --git a/schemas/tls.yml b/schemas/tls.yml index 1645dff6b9..24aad83e16 100644 --- a/schemas/tls.yml +++ b/schemas/tls.yml @@ -7,7 +7,7 @@ Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. type: group - default_fields: false + beats.default_field: false fields: - name: version diff --git a/schemas/vulnerability.yml b/schemas/vulnerability.yml index 4f5d10b793..21d814b837 100644 --- a/schemas/vulnerability.yml +++ b/schemas/vulnerability.yml @@ -2,7 +2,7 @@ - name: vulnerability title: Vulnerability group: 2 - default_field: false + beats.default_field: false short: Fields to describe the vulnerability relevant to an event. description: > The vulnerability fields describe information about a vulnerability that is diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 9864bbd33a..4c44ddc8c2 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -32,8 +32,11 @@ def fieldset_field_array(source_fields): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', 'ignore_above', 'multi_fields', 'format', 'input_format', 'output_format', 'output_precision', 'description', - 'example', 'default_field'] + 'example', 'beats.default_field'] multi_fields_allowed_keys = ['name', 'type', 'norms'] + rename_keys = { + 'beats.default_field': 'default_field' + } fields = [] for nested_field_name in source_fields: @@ -48,6 +51,7 @@ def fieldset_field_array(source_fields): beats_field['multi_fields'] = cleaned_multi_fields beats_field['name'] = nested_field_name + ecs_helpers.dict_rename_keys(beats_field, rename_keys) fields.append(beats_field) return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index d04c3e282d..1cc9920085 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -61,6 +61,14 @@ def yaml_ordereddict(dumper, data): yaml.add_representer(OrderedDict, yaml_ordereddict) + +def dict_rename_keys(dict, renames): + for key, value in dict.iteritems(): + if key in renames: + del dict[key] + dict[renames[key]] = value + + # File helpers From ef2da989fe499d588592408ac2bec8b0a9955c3c Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 13 Dec 2019 10:59:09 +0100 Subject: [PATCH 4/8] Disable default for all fields introduced in ECS 1.3.x This is so that Beats' default_fields don't go above 1000 field limit. --- generated/beats/fields.ecs.yml | 7 +++++++ generated/ecs/ecs_flat.yml | 7 +++++++ generated/ecs/ecs_nested.yml | 7 +++++++ schemas/event.yml | 1 + schemas/host.yml | 1 + schemas/package.yml | 3 +++ schemas/process.yml | 2 ++ 7 files changed, 28 insertions(+) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 25250991d9..5960ba0005 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1031,6 +1031,7 @@ In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' example: 2016-05-23 08:05:35.101000 + default_field: false - name: kind level: extended type: keyword @@ -1446,6 +1447,7 @@ \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ \ name. For Linux this could be the domain of the host's LDAP provider." example: CONTOSO + default_field: false - name: geo.city_name level: core type: keyword @@ -2225,6 +2227,7 @@ For example use the commit SHA of a non-released package.' example: 36f4f7e89dd61b0988b12ee000b98966867710cd + default_field: false - name: checksum level: extended type: keyword @@ -2276,6 +2279,7 @@ description: Home page or reference URL of the software in this package, if available. example: https://golang.org + default_field: false - name: size level: extended type: long @@ -2291,6 +2295,7 @@ This should contain the package file type, rather than the package manager name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' example: rpm + default_field: false - name: version level: extended type: keyword @@ -2451,6 +2456,7 @@ The field should be absent if there is no exit code for the event (e.g. process start).' example: 137 + default_field: false - name: parent.name level: extended type: keyword @@ -2476,6 +2482,7 @@ format: string description: Process id. example: 4242 + default_field: false - name: parent.ppid level: extended type: long diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 4a06b6a1d9..759d3a6202 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1509,6 +1509,7 @@ event.id: short: Unique ID to describe the event. type: keyword event.ingested: + beats.default_field: false dashed_name: event-ingested description: 'Timestamp when an event arrived in the central data store. @@ -2185,6 +2186,7 @@ host.architecture: short: Operating system architecture. type: keyword host.domain: + beats.default_field: false dashed_name: host-domain description: "Name of the domain of which the host is a member. \nFor example, on\ \ Windows this could be the host's Active Directory domain or NetBIOS domain name.\ @@ -3453,6 +3455,7 @@ package.architecture: short: Package architecture. type: keyword package.build_version: + beats.default_field: false dashed_name: package-build-version description: 'Additional information about the build version of the installed package. @@ -3544,6 +3547,7 @@ package.path: short: Path where the package is installed. type: keyword package.reference: + beats.default_field: false dashed_name: package-reference description: Home page or reference URL of the software in this package, if available. example: https://golang.org @@ -3566,6 +3570,7 @@ package.size: short: Package size in bytes. type: long package.type: + beats.default_field: false dashed_name: package-type description: 'Type of package. @@ -3806,6 +3811,7 @@ process.parent.executable: short: Absolute path to the process executable. type: keyword process.parent.exit_code: + beats.default_field: false dashed_name: process-parent-exit-code description: 'The exit code of the process, if this is a termination event. @@ -3849,6 +3855,7 @@ process.parent.pgid: short: Identifier of the group of processes the process belongs to. type: long process.parent.pid: + beats.default_field: false dashed_name: process-parent-pid description: Process id. example: 4242 diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index c9f21129a4..0687e57207 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1724,6 +1724,7 @@ event: short: Unique ID to describe the event. type: keyword ingested: + beats.default_field: false dashed_name: event-ingested description: 'Timestamp when an event arrived in the central data store. @@ -2486,6 +2487,7 @@ host: short: Operating system architecture. type: keyword domain: + beats.default_field: false dashed_name: host-domain description: "Name of the domain of which the host is a member. \nFor example,\ \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ @@ -3829,6 +3831,7 @@ package: short: Package architecture. type: keyword build_version: + beats.default_field: false dashed_name: package-build-version description: 'Additional information about the build version of the installed package. @@ -3922,6 +3925,7 @@ package: short: Path where the package is installed. type: keyword reference: + beats.default_field: false dashed_name: package-reference description: Home page or reference URL of the software in this package, if available. @@ -3945,6 +3949,7 @@ package: short: Package size in bytes. type: long type: + beats.default_field: false dashed_name: package-type description: 'Type of package. @@ -4198,6 +4203,7 @@ process: short: Absolute path to the process executable. type: keyword parent.exit_code: + beats.default_field: false dashed_name: process-parent-exit-code description: 'The exit code of the process, if this is a termination event. @@ -4241,6 +4247,7 @@ process: short: Identifier of the group of processes the process belongs to. type: long parent.pid: + beats.default_field: false dashed_name: process-parent-pid description: Process id. example: 4242 diff --git a/schemas/event.yml b/schemas/event.yml index d757f6cd1e..d161504803 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -321,6 +321,7 @@ type: date short: Timestamp when an event arrived in the central data store. example: 2016-05-23T08:05:35.101Z + beats.default_field: false description: > Timestamp when an event arrived in the central data store. diff --git a/schemas/host.yml b/schemas/host.yml index c4eb751bf3..cb8d8cdeca 100644 --- a/schemas/host.yml +++ b/schemas/host.yml @@ -83,6 +83,7 @@ - name: domain level: extended type: keyword + beats.default_field: false short: Name of the directory the group is a member of. description: > Name of the domain of which the host is a member. diff --git a/schemas/package.yml b/schemas/package.yml index a1d5cbb00f..021c2646e8 100644 --- a/schemas/package.yml +++ b/schemas/package.yml @@ -24,6 +24,7 @@ - name: build_version level: extended type: keyword + beats.default_field: false short: Build version information description: > Additional information about the build version of the installed package. @@ -86,6 +87,7 @@ - name: reference level: extended type: keyword + beats.default_field: false short: Package home page or reference URL description: > Home page or reference URL of the software in this package, if available. @@ -94,6 +96,7 @@ - name: type level: extended type: keyword + beats.default_field: false short: Package type description: > Type of package. diff --git a/schemas/process.yml b/schemas/process.yml index 210cdc7379..4f23b133b0 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -38,6 +38,7 @@ format: string level: core type: long + beats.default_field: false description: > Process id. example: 4242 @@ -338,6 +339,7 @@ level: extended type: long example: 137 + beats.default_field: false short: The exit code of the process. description: > The exit code of the process, if this is a termination event. From 2bbebc0c1c5305a90c5239d58141e03c2543333a Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 13 Dec 2019 14:46:30 +0100 Subject: [PATCH 5/8] Unset default_field on more new fields --- generated/beats/fields.ecs.yml | 2 ++ generated/ecs/ecs_flat.yml | 2 ++ generated/ecs/ecs_nested.yml | 4 ++++ schemas/file.yml | 2 ++ schemas/registry.yml | 1 + schemas/rule.yml | 1 + 6 files changed, 12 insertions(+) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 5960ba0005..dc59c29e58 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1166,6 +1166,7 @@ that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write.' example: '["readonly", "system"]' + default_field: false - name: created level: extended type: date @@ -1201,6 +1202,7 @@ The value should be uppercase, and not include the colon.' example: C + default_field: false - name: extension level: extended type: keyword diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 759d3a6202..3932dadf5b 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1739,6 +1739,7 @@ file.accessed: short: Last time the file was accessed. type: date file.attributes: + beats.default_field: false dashed_name: file-attributes description: 'Array of file attributes. @@ -1800,6 +1801,7 @@ file.directory: short: Directory where the file is located. type: keyword file.drive_letter: + beats.default_field: false dashed_name: file-drive-letter description: 'Drive letter where the file is located. This field is only relevant on Windows. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 0687e57207..01e77d2e44 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1971,6 +1971,7 @@ file: short: Last time the file was accessed. type: date attributes: + beats.default_field: false dashed_name: file-attributes description: 'Array of file attributes. @@ -2032,6 +2033,7 @@ file: short: Directory where the file is located. type: keyword drive_letter: + beats.default_field: false dashed_name: file-drive-letter description: 'Drive letter where the file is located. This field is only relevant on Windows. @@ -4469,6 +4471,7 @@ process: title: Process type: group registry: + beats.default_field: false description: Fields related to Windows Registry operations. fields: data.bytes: @@ -4593,6 +4596,7 @@ related: title: Related type: group rule: + beats.default_field: false description: 'Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. diff --git a/schemas/file.yml b/schemas/file.yml index 1bb6794378..8c1925f8b5 100644 --- a/schemas/file.yml +++ b/schemas/file.yml @@ -21,6 +21,7 @@ - name: attributes level: extended type: keyword + beats.default_field: false short: Array of file attributes. description: > Array of file attributes. @@ -43,6 +44,7 @@ level: extended type: keyword ignore_above: 1 + beats.default_field: false short: Drive letter where the file is located. description: > Drive letter where the file is located. This field is only relevant on Windows. diff --git a/schemas/registry.yml b/schemas/registry.yml index 6642850396..9b26bb7ff0 100644 --- a/schemas/registry.yml +++ b/schemas/registry.yml @@ -2,6 +2,7 @@ - name: registry title: Registry group: 2 + beats.default_field: false description: Fields related to Windows Registry operations. type: group fields: diff --git a/schemas/rule.yml b/schemas/rule.yml index bd668fd60f..7cf82a5b06 100644 --- a/schemas/rule.yml +++ b/schemas/rule.yml @@ -2,6 +2,7 @@ - name: rule title: Rule group: 2 + beats.default_field: false short: Fields to capture details about rules used to generate alerts or other notable events. description: > Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. From a8d81fb1185cddd7e077e642d261f47d8a700f16 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 16 Dec 2019 19:29:04 +0100 Subject: [PATCH 6/8] Document beats keys --- schemas/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/schemas/README.md b/schemas/README.md index 275e973dc8..cd54878d13 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -58,3 +58,11 @@ Supported keys to describe expected values for a field - type (required): type of the multi\_fields - name (optional): defaults to multi\_fields type + + +### Beats-specific keys + +The following keys are available to tune the fields generated for Beats. These are grouped under `beats.*`: + +- beats.default\_field (optional): Allows to control wether the field is added to the + `index.query.default_field` index setting. Defaults to `true`. From 02b992e5ad9183dc84b8511ae97c1acd8280dcb1 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 20 Dec 2019 13:41:54 -0500 Subject: [PATCH 7/8] Move the Beats 'default_field' workaround to the Beats generator --- code/go/ecs/host.go | 4 +- code/go/ecs/rule.go | 2 +- docs/field-details.asciidoc | 6 +- generated/beats/fields.ecs.yml | 68 ++- generated/ecs/ecs_flat.yml | 32 +- generated/ecs/ecs_nested.yml | 42 +- schemas/README.md | 8 - schemas/event.yml | 1 - schemas/file.yml | 2 - schemas/host.yml | 5 +- schemas/package.yml | 3 - schemas/process.yml | 18 - schemas/registry.yml | 1 - schemas/rule.yml | 9 +- schemas/tls.yml | 1 - schemas/vulnerability.yml | 1 - scripts/generators/beats.py | 19 +- .../beats_default_fields_whitelist.yml | 401 ++++++++++++++++++ scripts/generators/ecs_helpers.py | 5 + 19 files changed, 504 insertions(+), 124 deletions(-) create mode 100644 scripts/generators/beats_default_fields_whitelist.yml diff --git a/code/go/ecs/host.go b/code/go/ecs/host.go index 44e52b6c8a..96925dcc33 100644 --- a/code/go/ecs/host.go +++ b/code/go/ecs/host.go @@ -61,7 +61,7 @@ type Host struct { // Name of the domain of which the host is a member. // For example, on Windows this could be the host's Active Directory domain - // or NetBIOS domain name. For Linux this could be the domain of the - // host's LDAP provider. + // or NetBIOS domain name. For Linux this could be the domain of the host's + // LDAP provider. Domain string `ecs:"domain"` } diff --git a/code/go/ecs/rule.go b/code/go/ecs/rule.go index 708c922fbd..2e2f1eec3a 100644 --- a/code/go/ecs/rule.go +++ b/code/go/ecs/rule.go @@ -22,7 +22,7 @@ package ecs // Rule fields are used to capture the specifics of any observer or agent rules // that generate alerts or other notable events. // Examples of data sources that would populate the rule fields include: -// network admission control platforms, network or host IDS/IPS, network +// network admission control platforms, network or host IDS/IPS, network // firewalls, web application firewalls, url filters, endpoint detection and // response (EDR) systems, etc. type Rule struct { diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index e8034c4b5c..beab45f6cf 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -2001,9 +2001,9 @@ example: `x86_64` // =============================================================== | host.domain -| Name of the domain of which the host is a member. +| Name of the domain of which the host is a member. -For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. +For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. type: keyword @@ -3671,7 +3671,7 @@ type: keyword Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. -Examples of data sources that would populate the rule fields include: network admission control platforms, network or host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. +Examples of data sources that would populate the rule fields include: network admission control platforms, network or host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. ==== Rule Field Details diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 28bebdfef3..2c74f21235 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1466,9 +1466,11 @@ level: extended type: keyword ignore_above: 1024 - description: "Name of the domain of which the host is a member. \nFor example,\ - \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ - \ name. For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain + or NetBIOS domain name. For Linux this could be the domain of the host''s + LDAP provider.' example: CONTOSO default_field: false - name: geo.city_name @@ -2640,6 +2642,7 @@ corresponds to the data pointed by `lp_data`. This is optional but provides better recoverability and should be populated for REG_BINARY encoded values.' example: ZQBuAC0AVQBTAAAAZQBuAAAAAAA= + default_field: false - name: data.strings level: core type: keyword @@ -2652,24 +2655,28 @@ variable length. For numeric data, such as REG_DWORD and REG_QWORD, this should be populated with the decimal representation (e.g `"1"`).' example: '["C:\rta\red_ttp\bin\myapp.exe"]' + default_field: false - name: data.type level: core type: keyword ignore_above: 1024 description: Standard registry type for encoding contents example: REG_SZ + default_field: false - name: hive level: core type: keyword ignore_above: 1024 description: Abbreviated name for the hive. example: HKLM + default_field: false - name: key level: core type: keyword ignore_above: 1024 description: Hive-relative path of keys. example: SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\winword.exe + default_field: false - name: path level: core type: keyword @@ -2677,12 +2684,14 @@ description: Full path, including hive, key and value example: HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\winword.exe\Debugger + default_field: false - name: value level: core type: keyword ignore_above: 1024 description: Name of the value written. example: Debugger + default_field: false - name: related title: Related group: 2 @@ -2708,6 +2717,7 @@ type: keyword ignore_above: 1024 description: All the user names seen on your event. + default_field: false - name: rule title: Rule group: 2 @@ -2715,7 +2725,7 @@ agent rules that generate alerts or other notable events. Examples of data sources that would populate the rule fields include: network - admission control platforms, network or host IDS/IPS, network firewalls, web + admission control platforms, network or host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc.' type: group @@ -2727,12 +2737,14 @@ description: A categorization value keyword used by the entity using the rule for detection of this event. example: Attempted Information Leak + default_field: false - name: description level: extended type: keyword ignore_above: 1024 description: The description of the rule generating the event. example: Block requests to public DNS over HTTPS / TLS protocols + default_field: false - name: id level: extended type: keyword @@ -2740,12 +2752,14 @@ description: A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. example: 101 + default_field: false - name: name level: extended type: keyword ignore_above: 1024 description: The name of the rule or signature generating the event. example: BLOCK_DNS_over_TLS + default_field: false - name: reference level: extended type: keyword @@ -2757,6 +2771,7 @@ not available, it can also be a link to a more general page describing this type of alert.' example: https://en.wikipedia.org/wiki/DNS_over_TLS + default_field: false - name: ruleset level: extended type: keyword @@ -2764,6 +2779,7 @@ description: Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. example: Standard_Protocol_Filters + default_field: false - name: uuid level: extended type: keyword @@ -2772,12 +2788,14 @@ agents, observers, or other entities using the rule for detection of this event. example: 1100110011 + default_field: false - name: version level: extended type: keyword ignore_above: 1024 description: The version / revision of the rule being used for analysis. example: 1.1 + default_field: false - name: server title: Server group: 2 @@ -3407,6 +3425,7 @@ ignore_above: 1024 description: String indicating the cipher used during the current connection. example: TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 + default_field: false - name: client.certificate level: extended type: keyword @@ -3415,6 +3434,7 @@ is usually mutually-exclusive of `client.certificate_chain` since this value also exists in that list. example: MII... + default_field: false - name: client.certificate_chain level: extended type: keyword @@ -3425,6 +3445,7 @@ example: - MII... - MII... + default_field: false - name: client.hash.md5 level: extended type: keyword @@ -3433,6 +3454,7 @@ of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC + default_field: false - name: client.hash.sha1 level: extended type: keyword @@ -3441,6 +3463,7 @@ of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 9E393D93138888D288266C2D915214D1D1CCEB2A + default_field: false - name: client.hash.sha256 level: extended type: keyword @@ -3449,6 +3472,7 @@ version of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0 + default_field: false - name: client.issuer level: extended type: keyword @@ -3456,6 +3480,7 @@ description: Distinguished name of subject of the issuer of the x.509 certificate presented by the client. example: CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: client.ja3 level: extended type: keyword @@ -3463,18 +3488,21 @@ description: A hash that identifies clients based on how they perform an SSL/TLS handshake. example: d4e5b18d6b55c71272893221c96ba240 + default_field: false - name: client.not_after level: extended type: date description: Date/Time indicating when client certificate is no longer considered valid. example: '2021-01-01T00:00:00.000Z' + default_field: false - name: client.not_before level: extended type: date description: Date/Time indicating when client certificate is first considered valid. example: '1970-01-01T00:00:00.000Z' + default_field: false - name: client.server_name level: extended type: keyword @@ -3483,6 +3511,7 @@ the client is attempting to connect. When this value is available, it should get copied to `destination.domain`. example: www.elastic.co + default_field: false - name: client.subject level: extended type: keyword @@ -3490,6 +3519,7 @@ description: Distinguished name of subject of the x.509 certificate presented by the client. example: CN=myclient, OU=Documentation Team, DC=mydomain, DC=com + default_field: false - name: client.supported_ciphers level: extended type: keyword @@ -3499,17 +3529,20 @@ - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - '...' + default_field: false - name: curve level: extended type: keyword ignore_above: 1024 description: String indicating the curve used for the given cipher, when applicable. example: secp256r1 + default_field: false - name: established level: extended type: boolean description: Boolean flag indicating if the TLS negotiation was successful and transitioned to an encrypted tunnel. + default_field: false - name: next_protocol level: extended type: keyword @@ -3518,11 +3551,13 @@ the IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids), this string should be lower case. example: http/1.1 + default_field: false - name: resumed level: extended type: boolean description: Boolean flag indicating if this TLS connection was resumed from an existing TLS negotiation. + default_field: false - name: server.certificate level: extended type: keyword @@ -3531,6 +3566,7 @@ is usually mutually-exclusive of `server.certificate_chain` since this value also exists in that list. example: MII... + default_field: false - name: server.certificate_chain level: extended type: keyword @@ -3541,6 +3577,7 @@ example: - MII... - MII... + default_field: false - name: server.hash.md5 level: extended type: keyword @@ -3549,6 +3586,7 @@ of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC + default_field: false - name: server.hash.sha1 level: extended type: keyword @@ -3557,6 +3595,7 @@ of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 9E393D93138888D288266C2D915214D1D1CCEB2A + default_field: false - name: server.hash.sha256 level: extended type: keyword @@ -3565,6 +3604,7 @@ version of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0 + default_field: false - name: server.issuer level: extended type: keyword @@ -3572,6 +3612,7 @@ description: Subject of the issuer of the x.509 certificate presented by the server. example: CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: server.ja3s level: extended type: keyword @@ -3579,36 +3620,42 @@ description: A hash that identifies servers based on how they perform an SSL/TLS handshake. example: 394441ab65754e2207b1e1b457b3641d + default_field: false - name: server.not_after level: extended type: date description: Timestamp indicating when server certificate is no longer considered valid. example: '2021-01-01T00:00:00.000Z' + default_field: false - name: server.not_before level: extended type: date description: Timestamp indicating when server certificate is first considered valid. example: '1970-01-01T00:00:00.000Z' + default_field: false - name: server.subject level: extended type: keyword ignore_above: 1024 description: Subject of the x.509 certificate presented by the server. example: CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: version level: extended type: keyword ignore_above: 1024 description: Numeric part of the version parsed from the original string. example: '1.2' + default_field: false - name: version_protocol level: extended type: keyword ignore_above: 1024 description: Normalized lowercase protocol name parsed from original string. example: tls + default_field: false - name: tracing title: Tracing group: 2 @@ -3934,6 +3981,7 @@ This field must be an array.' example: '["Firewall"]' + default_field: false - name: classification level: extended type: keyword @@ -3941,6 +3989,7 @@ description: The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) example: CVSS + default_field: false - name: description level: extended type: keyword @@ -3953,6 +4002,7 @@ of the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common Vulnerabilities and Exposure CVE description]) example: In macOS before 2.12.6, there is a vulnerability in the RPC... + default_field: false - name: enumeration level: extended type: keyword @@ -3960,6 +4010,7 @@ description: The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) example: CVE + default_field: false - name: id level: extended type: keyword @@ -3969,6 +4020,7 @@ example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] example: CVE-2019-00001 + default_field: false - name: reference level: extended type: keyword @@ -3976,18 +4028,21 @@ description: A resource that provides additional information, context, and mitigations for the identified vulnerability. example: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111 + default_field: false - name: report_id level: extended type: keyword ignore_above: 1024 description: The report or scan identification number. example: 20191018.0001 + default_field: false - name: scanner.vendor level: extended type: keyword ignore_above: 1024 description: The name of the vulnerability scanner vendor. example: Tenable + default_field: false - name: score.base level: extended type: float @@ -3997,6 +4052,7 @@ complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)' example: 5.5 + default_field: false - name: score.environmental level: extended type: float @@ -4005,6 +4061,7 @@ Environmental scores cover an assessment for any modified Base metrics, confidentiality, integrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)' example: 5.5 + default_field: false - name: score.temporal level: extended type: float @@ -4012,6 +4069,7 @@ Temporal scores cover an assessment for code maturity, remediation level, and confidence. For example (https://www.first.org/cvss/specification-document)' + default_field: false - name: score.version level: extended type: keyword @@ -4025,6 +4083,7 @@ organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)' example: 2.0 + default_field: false - name: severity level: extended type: keyword @@ -4032,3 +4091,4 @@ description: The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) example: Critical + default_field: false diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 09d24da320..3779dfe3e7 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1619,7 +1619,6 @@ event.id: short: Unique ID to describe the event. type: keyword event.ingested: - beats.default_field: false dashed_name: event-ingested description: 'Timestamp when an event arrived in the central data store. @@ -1968,7 +1967,6 @@ file.accessed: short: Last time the file was accessed. type: date file.attributes: - beats.default_field: false dashed_name: file-attributes description: 'Array of file attributes. @@ -2030,7 +2028,6 @@ file.directory: short: Directory where the file is located. type: keyword file.drive_letter: - beats.default_field: false dashed_name: file-drive-letter description: 'Drive letter where the file is located. This field is only relevant on Windows. @@ -2417,11 +2414,11 @@ host.architecture: short: Operating system architecture. type: keyword host.domain: - beats.default_field: false dashed_name: host-domain - description: "Name of the domain of which the host is a member. \nFor example, on\ - \ Windows this could be the host's Active Directory domain or NetBIOS domain name.\ - \ For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS + domain name. For Linux this could be the domain of the host''s LDAP provider.' example: CONTOSO flat_name: host.domain ignore_above: 1024 @@ -3686,7 +3683,6 @@ package.architecture: short: Package architecture. type: keyword package.build_version: - beats.default_field: false dashed_name: package-build-version description: 'Additional information about the build version of the installed package. @@ -3778,7 +3774,6 @@ package.path: short: Path where the package is installed. type: keyword package.reference: - beats.default_field: false dashed_name: package-reference description: Home page or reference URL of the software in this package, if available. example: https://golang.org @@ -3801,7 +3796,6 @@ package.size: short: Package size in bytes. type: long package.type: - beats.default_field: false dashed_name: package-type description: 'Type of package. @@ -3845,7 +3839,6 @@ process.args: short: Array of process arguments. type: keyword process.args_count: - beats.default_field: false dashed_name: process-args-count description: 'Length of the process.args array. @@ -3860,7 +3853,6 @@ process.args_count: short: Length of the process.args array. type: long process.command_line: - beats.default_field: false dashed_name: process-command-line description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -3896,7 +3888,6 @@ process.executable: short: Absolute path to the process executable. type: keyword process.exit_code: - beats.default_field: false dashed_name: process-exit-code description: 'The exit code of the process, if this is a termination event. @@ -3972,7 +3963,6 @@ process.name: short: Process name. type: keyword process.parent.args: - beats.default_field: false dashed_name: process-parent-args description: 'Array of process arguments. @@ -3990,7 +3980,6 @@ process.parent.args: short: Array of process arguments. type: keyword process.parent.args_count: - beats.default_field: false dashed_name: process-parent-args-count description: 'Length of the process.args array. @@ -4005,7 +3994,6 @@ process.parent.args_count: short: Length of the process.args array. type: long process.parent.command_line: - beats.default_field: false dashed_name: process-parent-command-line description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4025,7 +4013,6 @@ process.parent.command_line: short: Full command line that started the process. type: keyword process.parent.executable: - beats.default_field: false dashed_name: process-parent-executable description: Absolute path to the process executable. example: /usr/bin/ssh @@ -4042,7 +4029,6 @@ process.parent.executable: short: Absolute path to the process executable. type: keyword process.parent.exit_code: - beats.default_field: false dashed_name: process-parent-exit-code description: 'The exit code of the process, if this is a termination event. @@ -4056,7 +4042,6 @@ process.parent.exit_code: short: The exit code of the process. type: long process.parent.name: - beats.default_field: false dashed_name: process-parent-name description: 'Process name. @@ -4075,7 +4060,6 @@ process.parent.name: short: Process name. type: keyword process.parent.pgid: - beats.default_field: false dashed_name: process-parent-pgid description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid @@ -4086,7 +4070,6 @@ process.parent.pgid: short: Identifier of the group of processes the process belongs to. type: long process.parent.pid: - beats.default_field: false dashed_name: process-parent-pid description: Process id. example: 4242 @@ -4098,7 +4081,6 @@ process.parent.pid: short: Process id. type: long process.parent.ppid: - beats.default_field: false dashed_name: process-parent-ppid description: Parent process' pid. example: 4241 @@ -4110,7 +4092,6 @@ process.parent.ppid: short: Parent process' pid. type: long process.parent.start: - beats.default_field: false dashed_name: process-parent-start description: The time the process started. example: '2016-05-23T08:05:34.853Z' @@ -4121,7 +4102,6 @@ process.parent.start: short: The time the process started. type: date process.parent.thread.id: - beats.default_field: false dashed_name: process-parent-thread-id description: Thread ID. example: 4242 @@ -4133,7 +4113,6 @@ process.parent.thread.id: short: Thread ID. type: long process.parent.thread.name: - beats.default_field: false dashed_name: process-parent-thread-name description: Thread name. example: thread-0 @@ -4145,7 +4124,6 @@ process.parent.thread.name: short: Thread name. type: keyword process.parent.title: - beats.default_field: false dashed_name: process-parent-title description: 'Process title. @@ -4164,7 +4142,6 @@ process.parent.title: short: Process title. type: keyword process.parent.uptime: - beats.default_field: false dashed_name: process-parent-uptime description: Seconds the process has been up. example: 1325 @@ -4175,7 +4152,6 @@ process.parent.uptime: short: Seconds the process has been up. type: long process.parent.working_directory: - beats.default_field: false dashed_name: process-parent-working-directory description: The working directory of the process. example: /home/alice diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 483cca1c08..57cb4d8a90 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1836,7 +1836,6 @@ event: short: Unique ID to describe the event. type: keyword ingested: - beats.default_field: false dashed_name: event-ingested description: 'Timestamp when an event arrived in the central data store. @@ -2206,7 +2205,6 @@ file: short: Last time the file was accessed. type: date attributes: - beats.default_field: false dashed_name: file-attributes description: 'Array of file attributes. @@ -2268,7 +2266,6 @@ file: short: Directory where the file is located. type: keyword drive_letter: - beats.default_field: false dashed_name: file-drive-letter description: 'Drive letter where the file is located. This field is only relevant on Windows. @@ -2724,11 +2721,12 @@ host: short: Operating system architecture. type: keyword domain: - beats.default_field: false dashed_name: host-domain - description: "Name of the domain of which the host is a member. \nFor example,\ - \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ - \ name. For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain + or NetBIOS domain name. For Linux this could be the domain of the host''s + LDAP provider.' example: CONTOSO flat_name: host.domain ignore_above: 1024 @@ -4068,7 +4066,6 @@ package: short: Package architecture. type: keyword build_version: - beats.default_field: false dashed_name: package-build-version description: 'Additional information about the build version of the installed package. @@ -4162,7 +4159,6 @@ package: short: Path where the package is installed. type: keyword reference: - beats.default_field: false dashed_name: package-reference description: Home page or reference URL of the software in this package, if available. @@ -4186,7 +4182,6 @@ package: short: Package size in bytes. type: long type: - beats.default_field: false dashed_name: package-type description: 'Type of package. @@ -4243,7 +4238,6 @@ process: short: Array of process arguments. type: keyword args_count: - beats.default_field: false dashed_name: process-args-count description: 'Length of the process.args array. @@ -4258,7 +4252,6 @@ process: short: Length of the process.args array. type: long command_line: - beats.default_field: false dashed_name: process-command-line description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4294,7 +4287,6 @@ process: short: Absolute path to the process executable. type: keyword exit_code: - beats.default_field: false dashed_name: process-exit-code description: 'The exit code of the process, if this is a termination event. @@ -4370,7 +4362,6 @@ process: short: Process name. type: keyword parent.args: - beats.default_field: false dashed_name: process-parent-args description: 'Array of process arguments. @@ -4388,7 +4379,6 @@ process: short: Array of process arguments. type: keyword parent.args_count: - beats.default_field: false dashed_name: process-parent-args-count description: 'Length of the process.args array. @@ -4403,7 +4393,6 @@ process: short: Length of the process.args array. type: long parent.command_line: - beats.default_field: false dashed_name: process-parent-command-line description: 'Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -4423,7 +4412,6 @@ process: short: Full command line that started the process. type: keyword parent.executable: - beats.default_field: false dashed_name: process-parent-executable description: Absolute path to the process executable. example: /usr/bin/ssh @@ -4440,7 +4428,6 @@ process: short: Absolute path to the process executable. type: keyword parent.exit_code: - beats.default_field: false dashed_name: process-parent-exit-code description: 'The exit code of the process, if this is a termination event. @@ -4454,7 +4441,6 @@ process: short: The exit code of the process. type: long parent.name: - beats.default_field: false dashed_name: process-parent-name description: 'Process name. @@ -4473,7 +4459,6 @@ process: short: Process name. type: keyword parent.pgid: - beats.default_field: false dashed_name: process-parent-pgid description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid @@ -4484,7 +4469,6 @@ process: short: Identifier of the group of processes the process belongs to. type: long parent.pid: - beats.default_field: false dashed_name: process-parent-pid description: Process id. example: 4242 @@ -4496,7 +4480,6 @@ process: short: Process id. type: long parent.ppid: - beats.default_field: false dashed_name: process-parent-ppid description: Parent process' pid. example: 4241 @@ -4508,7 +4491,6 @@ process: short: Parent process' pid. type: long parent.start: - beats.default_field: false dashed_name: process-parent-start description: The time the process started. example: '2016-05-23T08:05:34.853Z' @@ -4519,7 +4501,6 @@ process: short: The time the process started. type: date parent.thread.id: - beats.default_field: false dashed_name: process-parent-thread-id description: Thread ID. example: 4242 @@ -4531,7 +4512,6 @@ process: short: Thread ID. type: long parent.thread.name: - beats.default_field: false dashed_name: process-parent-thread-name description: Thread name. example: thread-0 @@ -4543,7 +4523,6 @@ process: short: Thread name. type: keyword parent.title: - beats.default_field: false dashed_name: process-parent-title description: 'Process title. @@ -4562,7 +4541,6 @@ process: short: Process title. type: keyword parent.uptime: - beats.default_field: false dashed_name: process-parent-uptime description: Seconds the process has been up. example: 1325 @@ -4573,7 +4551,6 @@ process: short: Seconds the process has been up. type: long parent.working_directory: - beats.default_field: false dashed_name: process-parent-working-directory description: The working directory of the process. example: /home/alice @@ -4706,7 +4683,6 @@ process: title: Process type: group registry: - beats.default_field: false description: Fields related to Windows Registry operations. fields: data.bytes: @@ -4841,14 +4817,12 @@ related: title: Related type: group rule: - beats.default_field: false description: 'Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. Examples of data sources that would populate the rule fields include: network - admission control platforms, network or host IDS/IPS, network firewalls, web - application firewalls, url filters, endpoint detection and response (EDR) systems, - etc.' + admission control platforms, network or host IDS/IPS, network firewalls, web application + firewalls, url filters, endpoint detection and response (EDR) systems, etc.' fields: category: dashed_name: rule-category @@ -6008,7 +5982,6 @@ threat: title: Threat type: group tls: - beats.default_field: false description: Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. @@ -6921,7 +6894,6 @@ user_agent: title: User agent type: group vulnerability: - beats.default_field: false description: The vulnerability fields describe information about a vulnerability that is relevant to an event. fields: diff --git a/schemas/README.md b/schemas/README.md index cd54878d13..275e973dc8 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -58,11 +58,3 @@ Supported keys to describe expected values for a field - type (required): type of the multi\_fields - name (optional): defaults to multi\_fields type - - -### Beats-specific keys - -The following keys are available to tune the fields generated for Beats. These are grouped under `beats.*`: - -- beats.default\_field (optional): Allows to control wether the field is added to the - `index.query.default_field` index setting. Defaults to `true`. diff --git a/schemas/event.yml b/schemas/event.yml index 06c66d3d1a..323e99f509 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -528,7 +528,6 @@ type: date short: Timestamp when an event arrived in the central data store. example: 2016-05-23T08:05:35.101Z - beats.default_field: false description: > Timestamp when an event arrived in the central data store. diff --git a/schemas/file.yml b/schemas/file.yml index 8c1925f8b5..1bb6794378 100644 --- a/schemas/file.yml +++ b/schemas/file.yml @@ -21,7 +21,6 @@ - name: attributes level: extended type: keyword - beats.default_field: false short: Array of file attributes. description: > Array of file attributes. @@ -44,7 +43,6 @@ level: extended type: keyword ignore_above: 1 - beats.default_field: false short: Drive letter where the file is located. description: > Drive letter where the file is located. This field is only relevant on Windows. diff --git a/schemas/host.yml b/schemas/host.yml index cb8d8cdeca..bdfe42fc5c 100644 --- a/schemas/host.yml +++ b/schemas/host.yml @@ -83,12 +83,11 @@ - name: domain level: extended type: keyword - beats.default_field: false short: Name of the directory the group is a member of. description: > - Name of the domain of which the host is a member. + Name of the domain of which the host is a member. - For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. + For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. example: CONTOSO diff --git a/schemas/package.yml b/schemas/package.yml index 021c2646e8..a1d5cbb00f 100644 --- a/schemas/package.yml +++ b/schemas/package.yml @@ -24,7 +24,6 @@ - name: build_version level: extended type: keyword - beats.default_field: false short: Build version information description: > Additional information about the build version of the installed package. @@ -87,7 +86,6 @@ - name: reference level: extended type: keyword - beats.default_field: false short: Package home page or reference URL description: > Home page or reference URL of the software in this package, if available. @@ -96,7 +94,6 @@ - name: type level: extended type: keyword - beats.default_field: false short: Package type description: > Type of package. diff --git a/schemas/process.yml b/schemas/process.yml index 4f23b133b0..bc9450f66d 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -38,7 +38,6 @@ format: string level: core type: long - beats.default_field: false description: > Process id. example: 4242 @@ -61,7 +60,6 @@ level: extended type: keyword short: Process name. - beats.default_field: false description: > Process name. @@ -84,7 +82,6 @@ format: string level: extended type: long - beats.default_field: false description: > Parent process' pid. example: 4241 @@ -101,7 +98,6 @@ format: string level: extended type: long - beats.default_field: false description: > Identifier of the group of processes the process belongs to. @@ -110,7 +106,6 @@ level: extended type: keyword short: Full command line that started the process. - beats.default_field: false description: > Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -125,7 +120,6 @@ level: extended type: keyword short: Full command line that started the process. - beats.default_field: false description: > Full command line that started the process, including the absolute path to the executable, and all arguments. @@ -151,7 +145,6 @@ level: extended type: keyword short: Array of process arguments. - beats.default_field: false description: > Array of process arguments. @@ -162,7 +155,6 @@ level: extended type: long short: Length of the process.args array. - beats.default_field: false description: > Length of the process.args array. @@ -175,7 +167,6 @@ level: extended type: long short: Length of the process.args array. - beats.default_field: false description: > Length of the process.args array. @@ -198,7 +189,6 @@ - name: parent.executable level: extended type: keyword - beats.default_field: false description: > Absolute path to the process executable. example: /usr/bin/ssh @@ -224,7 +214,6 @@ level: extended type: keyword short: Process title. - beats.default_field: false description: > Process title. @@ -248,7 +237,6 @@ level: extended type: long example: 4242 - beats.default_field: false description: > Thread ID. @@ -263,7 +251,6 @@ - name: parent.thread.name level: extended type: keyword - beats.default_field: false example: 'thread-0' description: > Thread name. @@ -280,7 +267,6 @@ level: extended type: date example: "2016-05-23T08:05:34.853Z" - beats.default_field: false description: > The time the process started. @@ -296,7 +282,6 @@ level: extended type: long example: 1325 - beats.default_field: false description: > Seconds the process has been up. @@ -315,7 +300,6 @@ level: extended type: keyword example: /home/alice - beats.default_field: false description: > The working directory of the process. multi_fields: @@ -328,7 +312,6 @@ type: long example: 137 short: The exit code of the process. - beats.default_field: false description: > The exit code of the process, if this is a termination event. @@ -339,7 +322,6 @@ level: extended type: long example: 137 - beats.default_field: false short: The exit code of the process. description: > The exit code of the process, if this is a termination event. diff --git a/schemas/registry.yml b/schemas/registry.yml index 9b26bb7ff0..6642850396 100644 --- a/schemas/registry.yml +++ b/schemas/registry.yml @@ -2,7 +2,6 @@ - name: registry title: Registry group: 2 - beats.default_field: false description: Fields related to Windows Registry operations. type: group fields: diff --git a/schemas/rule.yml b/schemas/rule.yml index 7cf82a5b06..34e901839f 100644 --- a/schemas/rule.yml +++ b/schemas/rule.yml @@ -2,14 +2,13 @@ - name: rule title: Rule group: 2 - beats.default_field: false short: Fields to capture details about rules used to generate alerts or other notable events. description: > Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. - Examples of data sources that would populate the rule fields include: network admission control platforms, network or - host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. - + Examples of data sources that would populate the rule fields include: network admission control platforms, network or + host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. + type: group fields: @@ -20,7 +19,7 @@ description: > A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. - example: 101 + example: 101 - name: uuid level: extended diff --git a/schemas/tls.yml b/schemas/tls.yml index 24aad83e16..30d2f002fa 100644 --- a/schemas/tls.yml +++ b/schemas/tls.yml @@ -7,7 +7,6 @@ Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. type: group - beats.default_field: false fields: - name: version diff --git a/schemas/vulnerability.yml b/schemas/vulnerability.yml index 21d814b837..d67f8e61e1 100644 --- a/schemas/vulnerability.yml +++ b/schemas/vulnerability.yml @@ -2,7 +2,6 @@ - name: vulnerability title: Vulnerability group: 2 - beats.default_field: false short: Fields to describe the vulnerability relevant to an event. description: > The vulnerability fields describe information about a vulnerability that is diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 4c44ddc8c2..2926fe5570 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -5,8 +5,11 @@ def generate(ecs_nested, ecs_version): + # Load temporary whitelist for default_fields workaround. + df_whitelist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_whitelist.yml') + # base first - beats_fields = fieldset_field_array(ecs_nested['base']['fields']) + beats_fields = fieldset_field_array(ecs_nested['base']['fields'], df_whitelist) allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'footnote', 'type'] # other fieldsets @@ -16,7 +19,7 @@ def generate(ecs_nested, ecs_version): fieldset = ecs_nested[fieldset_name] beats_field = ecs_helpers.dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) - beats_field['fields'] = fieldset_field_array(fieldset['fields']) + beats_field['fields'] = fieldset_field_array(fieldset['fields'], df_whitelist) beats_fields.append(beats_field) beats_file = OrderedDict() @@ -28,15 +31,12 @@ def generate(ecs_nested, ecs_version): write_beats_yaml(beats_file, ecs_version) -def fieldset_field_array(source_fields): +def fieldset_field_array(source_fields, df_whitelist): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', 'ignore_above', 'multi_fields', 'format', 'input_format', 'output_format', 'output_precision', 'description', - 'example', 'beats.default_field'] + 'example'] multi_fields_allowed_keys = ['name', 'type', 'norms'] - rename_keys = { - 'beats.default_field': 'default_field' - } fields = [] for nested_field_name in source_fields: @@ -51,7 +51,10 @@ def fieldset_field_array(source_fields): beats_field['multi_fields'] = cleaned_multi_fields beats_field['name'] = nested_field_name - ecs_helpers.dict_rename_keys(beats_field, rename_keys) + + if not ecs_field['flat_name'] in df_whitelist: + beats_field['default_field'] = False + fields.append(beats_field) return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) diff --git a/scripts/generators/beats_default_fields_whitelist.yml b/scripts/generators/beats_default_fields_whitelist.yml new file mode 100644 index 0000000000..a7fa30b9c8 --- /dev/null +++ b/scripts/generators/beats_default_fields_whitelist.yml @@ -0,0 +1,401 @@ +--- +!!set +# Note: other fields can be inserted as needed + +# ECS 1.2 fields +'@timestamp': null +agent.ephemeral_id: null +agent.id: null +agent.name: null +agent.type: null +agent.version: null +as.number: null +as.organization.name: null +client.address: null +client.as.number: null +client.as.organization.name: null +client.bytes: null +client.domain: null +client.geo.city_name: null +client.geo.continent_name: null +client.geo.country_iso_code: null +client.geo.country_name: null +client.geo.location: null +client.geo.name: null +client.geo.region_iso_code: null +client.geo.region_name: null +client.ip: null +client.mac: null +client.nat.ip: null +client.nat.port: null +client.packets: null +client.port: null +client.registered_domain: null +client.top_level_domain: null +client.user.domain: null +client.user.email: null +client.user.full_name: null +client.user.group.domain: null +client.user.group.id: null +client.user.group.name: null +client.user.hash: null +client.user.id: null +client.user.name: null +cloud.account.id: null +cloud.availability_zone: null +cloud.instance.id: null +cloud.instance.name: null +cloud.machine.type: null +cloud.provider: null +cloud.region: null +container.id: null +container.image.name: null +container.image.tag: null +container.labels: null +container.name: null +container.runtime: null +destination.address: null +destination.as.number: null +destination.as.organization.name: null +destination.bytes: null +destination.domain: null +destination.geo.city_name: null +destination.geo.continent_name: null +destination.geo.country_iso_code: null +destination.geo.country_name: null +destination.geo.location: null +destination.geo.name: null +destination.geo.region_iso_code: null +destination.geo.region_name: null +destination.ip: null +destination.mac: null +destination.nat.ip: null +destination.nat.port: null +destination.packets: null +destination.port: null +destination.registered_domain: null +destination.top_level_domain: null +destination.user.domain: null +destination.user.email: null +destination.user.full_name: null +destination.user.group.domain: null +destination.user.group.id: null +destination.user.group.name: null +destination.user.hash: null +destination.user.id: null +destination.user.name: null +dns.answers: null +dns.answers.class: null +dns.answers.data: null +dns.answers.name: null +dns.answers.ttl: null +dns.answers.type: null +dns.header_flags: null +dns.id: null +dns.op_code: null +dns.question.class: null +dns.question.name: null +dns.question.registered_domain: null +dns.question.subdomain: null +dns.question.top_level_domain: null +dns.question.type: null +dns.resolved_ip: null +dns.response_code: null +dns.type: null +ecs.version: null +error.code: null +error.id: null +error.message: null +error.stack_trace: null +error.type: null +event.action: null +event.category: null +event.code: null +event.created: null +event.dataset: null +event.duration: null +event.end: null +event.hash: null +event.id: null +event.kind: null +event.module: null +event.original: null +event.outcome: null +event.provider: null +event.risk_score: null +event.risk_score_norm: null +event.sequence: null +event.severity: null +event.start: null +event.timezone: null +event.type: null +file.accessed: null +file.created: null +file.ctime: null +file.device: null +file.directory: null +file.extension: null +file.gid: null +file.group: null +file.hash.md5: null +file.hash.sha1: null +file.hash.sha256: null +file.hash.sha512: null +file.inode: null +file.mode: null +file.mtime: null +file.name: null +file.owner: null +file.path: null +file.size: null +file.target_path: null +file.type: null +file.uid: null +geo.city_name: null +geo.continent_name: null +geo.country_iso_code: null +geo.country_name: null +geo.location: null +geo.name: null +geo.region_iso_code: null +geo.region_name: null +group.domain: null +group.id: null +group.name: null +hash.md5: null +hash.sha1: null +hash.sha256: null +hash.sha512: null +host.architecture: null +host.geo.city_name: null +host.geo.continent_name: null +host.geo.country_iso_code: null +host.geo.country_name: null +host.geo.location: null +host.geo.name: null +host.geo.region_iso_code: null +host.geo.region_name: null +host.hostname: null +host.id: null +host.ip: null +host.mac: null +host.name: null +host.os.family: null +host.os.full: null +host.os.kernel: null +host.os.name: null +host.os.platform: null +host.os.version: null +host.type: null +host.uptime: null +host.user.domain: null +host.user.email: null +host.user.full_name: null +host.user.group.domain: null +host.user.group.id: null +host.user.group.name: null +host.user.hash: null +host.user.id: null +host.user.name: null +http.request.body.bytes: null +http.request.body.content: null +http.request.bytes: null +http.request.method: null +http.request.referrer: null +http.response.body.bytes: null +http.response.body.content: null +http.response.bytes: null +http.response.status_code: null +http.version: null +labels: null +log.level: null +log.logger: null +log.origin.file.line: null +log.origin.file.name: null +log.origin.function: null +log.original: null +log.syslog: null +log.syslog.facility.code: null +log.syslog.facility.name: null +log.syslog.priority: null +log.syslog.severity.code: null +log.syslog.severity.name: null +message: null +network.application: null +network.bytes: null +network.community_id: null +network.direction: null +network.forwarded_ip: null +network.iana_number: null +network.name: null +network.packets: null +network.protocol: null +network.transport: null +network.type: null +observer.geo.city_name: null +observer.geo.continent_name: null +observer.geo.country_iso_code: null +observer.geo.country_name: null +observer.geo.location: null +observer.geo.name: null +observer.geo.region_iso_code: null +observer.geo.region_name: null +observer.hostname: null +observer.ip: null +observer.mac: null +observer.name: null +observer.os.family: null +observer.os.full: null +observer.os.kernel: null +observer.os.name: null +observer.os.platform: null +observer.os.version: null +observer.product: null +observer.serial_number: null +observer.type: null +observer.vendor: null +observer.version: null +organization.id: null +organization.name: null +os.family: null +os.full: null +os.kernel: null +os.name: null +os.platform: null +os.version: null +package.architecture: null +package.checksum: null +package.description: null +package.install_scope: null +package.installed: null +package.license: null +package.name: null +package.path: null +package.size: null +package.version: null +process.args: null +process.executable: null +process.hash.md5: null +process.hash.sha1: null +process.hash.sha256: null +process.hash.sha512: null +process.name: null +process.pgid: null +process.pid: null +process.ppid: null +process.start: null +process.thread.id: null +process.thread.name: null +process.title: null +process.uptime: null +process.working_directory: null +related.ip: null +server.address: null +server.as.number: null +server.as.organization.name: null +server.bytes: null +server.domain: null +server.geo.city_name: null +server.geo.continent_name: null +server.geo.country_iso_code: null +server.geo.country_name: null +server.geo.location: null +server.geo.name: null +server.geo.region_iso_code: null +server.geo.region_name: null +server.ip: null +server.mac: null +server.nat.ip: null +server.nat.port: null +server.packets: null +server.port: null +server.registered_domain: null +server.top_level_domain: null +server.user.domain: null +server.user.email: null +server.user.full_name: null +server.user.group.domain: null +server.user.group.id: null +server.user.group.name: null +server.user.hash: null +server.user.id: null +server.user.name: null +service.ephemeral_id: null +service.id: null +service.name: null +service.node.name: null +service.state: null +service.type: null +service.version: null +source.address: null +source.as.number: null +source.as.organization.name: null +source.bytes: null +source.domain: null +source.geo.city_name: null +source.geo.continent_name: null +source.geo.country_iso_code: null +source.geo.country_name: null +source.geo.location: null +source.geo.name: null +source.geo.region_iso_code: null +source.geo.region_name: null +source.ip: null +source.mac: null +source.nat.ip: null +source.nat.port: null +source.packets: null +source.port: null +source.registered_domain: null +source.top_level_domain: null +source.user.domain: null +source.user.email: null +source.user.full_name: null +source.user.group.domain: null +source.user.group.id: null +source.user.group.name: null +source.user.hash: null +source.user.id: null +source.user.name: null +tags: null +threat.framework: null +threat.tactic.id: null +threat.tactic.name: null +threat.tactic.reference: null +threat.technique.id: null +threat.technique.name: null +threat.technique.reference: null +trace.id: null +transaction.id: null +url.domain: null +url.extension: null +url.fragment: null +url.full: null +url.original: null +url.password: null +url.path: null +url.port: null +url.query: null +url.registered_domain: null +url.scheme: null +url.top_level_domain: null +url.username: null +user.domain: null +user.email: null +user.full_name: null +user.group.domain: null +user.group.id: null +user.group.name: null +user.hash: null +user.id: null +user.name: null +user_agent.device.name: null +user_agent.name: null +user_agent.original: null +user_agent.os.family: null +user_agent.os.full: null +user_agent.os.kernel: null +user_agent.os.name: null +user_agent.os.platform: null +user_agent.os.version: null +user_agent.version: null diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index 1cc9920085..ff2f33baa1 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -78,6 +78,11 @@ def yaml_dump(filename, data, preamble=None): outfile.write(preamble) yaml.dump(data, outfile, default_flow_style=False) + +def yaml_load(filename): + with open(filename) as f: + return yaml.load(f.read()) + # List helpers From 97c9ef72c829f64e026d0fa966d2c0c27df25037 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 20 Dec 2019 13:47:40 -0500 Subject: [PATCH 8/8] Adjust the changelog --- CHANGELOG.next.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 9729c1deb5..88c48595f4 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -16,10 +16,10 @@ Thanks, you're awesome :-) --> #### Added -* Added the `beats.default_field` option for Beats. #687 - #### Improvements +* Temporary workaround for Beats templates' `default_field` growing too big. #687 + #### Deprecated