Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Platform Observability] Create initial PO package for ingesting kibana ECS formatted logs #3622

Merged
merged 18 commits into from
Jul 26, 2022
Merged

[Platform Observability] Create initial PO package for ingesting kibana ECS formatted logs #3622

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
ba764cb
create package for ingesting kibana ECS formatted logs
crespocarlos Jun 29, 2022
f91647a
changelog fix
crespocarlos Jun 29, 2022
2da545b
update test expected result
crespocarlos Jun 30, 2022
a8e4920
adds a filebeat processor alternative
crespocarlos Jul 4, 2022
b171f93
clean up poc code
crespocarlos Jul 6, 2022
d6cacfd
fix json identation
crespocarlos Jul 6, 2022
a3bd0c0
Align ingest pipeline approach and improve README.md
crespocarlos Jul 12, 2022
6e781a3
Improve README.md; Fix package folder name; Document ECS fields
crespocarlos Jul 14, 2022
0b03ec6
Fix supported kibana version on readme
crespocarlos Jul 18, 2022
33f32eb
Remove experimental attribute from manifest; Fix changelog PR; Fix mi…
crespocarlos Jul 18, 2022
8e47160
Support current and old model for license
crespocarlos Jul 18, 2022
3e74c73
Fix manifest
crespocarlos Jul 18, 2022
14dc44f
Add event.ingested field; Improve Kibana audit fields documentation
crespocarlos Jul 18, 2022
21524e8
Fix pipeline test
crespocarlos Jul 18, 2022
8cb34da
Extract ECS processor into a new file; Fix kibana logo; Add event.* f…
crespocarlos Jul 19, 2022
55c80e8
Simplify ingest pipeline and clean-ups
crespocarlos Jul 20, 2022
5dcc1fe
Fix json format
crespocarlos Jul 20, 2022
27a23aa
Align default log file name with kibana docs
crespocarlos Jul 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,3 +168,4 @@
/packages/zscaler @elastic/security-external-integrations
/packages/zscaler_zia @elastic/security-external-integrations
/packages/zscaler_zpa @elastic/security-external-integrations
/packages/platform-observability @elastic/obs-service-integrations @elastic/infra-monitoring-ui
6 changes: 6 additions & 0 deletions packages/platform-observability/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# newer versions go on top
- version: "0.0.1"
changes:
- description: Initial draft of the package
type: enhancement
link: https://github.com/elastic/integrations/pull/1
4 changes: 4 additions & 0 deletions ...es/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}}
{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/internal/bsearch","port":5601,"query":"compress=true","scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"abc8b4ad-5d96-42cf-9653-08aaeac0034e"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.178+00:00","message":"User is requesting [/internal/bsearch] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"3098796995e24283"}}
{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/log_entries/summary","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"f4181218-b2d3-480e-b9da-78aef88683ff"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.187+00:00","message":"User is requesting [/api/log_entries/summary] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1480039d6e6e321"}}
{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"83d80454-6b8a-4727-91ba-22e6ab27e476"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T13:14:30.142+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1f684ff5fcf7eaf"}}
48 changes: 48 additions & 0 deletions ...servability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
{
"expected": [
{
"data_stream": {
"dataset": "kibana-audit-log",
"namespace": "platform-observability",
"type": "logs"
},
"event": {
"dataset": "kibana-audit-log"
},
"message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"get\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/security/session\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"1c8c5808-d2d6-41fc-8cb7-998aa8996be9\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:03.742+00:00\",\"message\":\"User is requesting [/internal/security/session] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"f8863d86567119e6\"}}"
},
{
"data_stream": {
"dataset": "kibana-audit-log",
"namespace": "platform-observability",
"type": "logs"
},
"event": {
"dataset": "kibana-audit-log"
},
"message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"post\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/bsearch\",\"port\":5601,\"query\":\"compress=true\",\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"abc8b4ad-5d96-42cf-9653-08aaeac0034e\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:08.178+00:00\",\"message\":\"User is requesting [/internal/bsearch] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"3098796995e24283\"}}"
},
{
"data_stream": {
"dataset": "kibana-audit-log",
"namespace": "platform-observability",
"type": "logs"
},
"event": {
"dataset": "kibana-audit-log"
},
"message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"post\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/api/log_entries/summary\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"f4181218-b2d3-480e-b9da-78aef88683ff\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:08.187+00:00\",\"message\":\"User is requesting [/api/log_entries/summary] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"c1480039d6e6e321\"}}"
},
{
"data_stream": {
"dataset": "kibana-audit-log",
"namespace": "platform-observability",
"type": "logs"
},
"event": {
"dataset": "kibana-audit-log"
},
"message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"get\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/security/session\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"83d80454-6b8a-4727-91ba-22e6ab27e476\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T13:14:30.142+00:00\",\"message\":\"User is requesting [/internal/security/session] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"c1f684ff5fcf7eaf\"}}"
}
]
}
17 changes: 17 additions & 0 deletions packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
paths:
{{#each paths}}
- {{this}}
{{/each}}
processors:
- rename:
fields:
- from: "message"
to: "event.original"
- decode_json_fields:
fields: [event.original]
target: ""
add_error_key: true
overwrite_keys: true
{{#if processors}}
{{processors}}
Copy link
Contributor

@matschaffer matschaffer Jun 30, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm guessing we can add "filebeat" processors (like https://www.elastic.co/guide/en/beats/filebeat/current/decode-json-fields.html) to the manifest and have them show up here. Or maybe just put them in this file directly.

Not sure what's in-fashion for packages today regarding reader-vs-ES side processing but personally I like to push as many simple transformations as I can to the collection point. Maybe @mtojek can advise.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did a quick test and it seems to work. I'll explore it a bit more. It would simplify the pipeline config

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It works.
image

image

this is all I have for the pipeline:

[
  {
    "remove": {
      "field": "data_stream.dataset",
      "ignore_missing": true
    }
  },
  {
    "remove": {
      "field": "event.dataset",
      "ignore_missing": true
    }
  },
  {
    "set": {
      "description": "Uses event.dataset as a default for data_stream.dataset if the latter is not set.",
      "field": "data_stream.dataset",
      "copy_from": "event.dataset",
      "if": "ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1",
      "override": false
    }
  },
  {
    "script": {
      "source": "ctx.data_stream.dataset = /[\\/*?\"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_')\n",
      "if": "ctx.data_stream?.dataset != null"
    }
  },
  {
    "script": {
      "source": "ctx.data_stream.namespace = /[\\/*?\"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_')\n",
      "if": "ctx.data_stream?.namespace != null"
    }
  },
  {
    "set": {
      "field": "data_stream.type",
      "value": "logs"
    }
  },
  {
    "set": {
      "field": "data_stream.dataset",
      "value": "kibana-logs",
      "override": false
    }
  },
  {
    "set": {
      "field": "data_stream.namespace",
      "value": "platform-observability",
      "override": false
    }
  },
  {
    "set": {
      "field": "event.dataset",
      "copy_from": "data_stream.dataset"
    }
  },
  {
    "rename": {
      "field": "message",
      "target_field": "event.original",
      "ignore_missing": true
    }
  }
]

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A downside of this approach is that the tests are not able to use filebeat processor to parse the logs

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another surprise here :) Hoping we can have some integration experts weigh-in. Are we maybe not supposed to use processors anymore?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used to follow the rule of thumb. Are you collecting large observability data and is the network loaded? If so, limit those with processors.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We had a slack chat with @crespocarlos, and decided to go with the ingest pipeline due to code simplicity and convenient use of pipeline tests.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you collecting large observability data and is the network loaded?

Interesting. Does the end user get to set processors? I'd expect network load to be a factor of deployment size.

{{/if}}
37 changes: 37 additions & 0 deletions packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
---
description: Pipeline for parsing ECS logs
processors:
- remove:
field: data_stream.dataset
ignore_missing: true
- remove:
field: event.dataset
ignore_missing: true
- set:
description: Uses event.dataset as a default for data_stream.dataset if the latter is not set.
field: data_stream.dataset
copy_from: event.dataset
if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1
override: false
- script:
source: |
ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_')
if: ctx.data_stream?.dataset != null
- script:
source: |
ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_')
if: ctx.data_stream?.namespace != null
- set:
field: data_stream.type
value: logs
- set:
field: data_stream.dataset
value: kibana-audit-log
override: false
- set:
field: data_stream.namespace
value: platform-observability
override: false
- set:
field: event.dataset
copy_from: data_stream.dataset
12 changes: 12 additions & 0 deletions packages/platform-observability/data_stream/audit/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
- name: data_stream.type
type: constant_keyword
description: Data stream type.
- name: data_stream.dataset
type: constant_keyword
description: Data stream dataset.
- name: data_stream.namespace
type: constant_keyword
description: Data stream namespace.
- name: "@timestamp"
type: date
description: Event timestamp.
3 changes: 3 additions & 0 deletions packages/platform-observability/data_stream/audit/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# only used for tests
- name: message
external: ecs
17 changes: 17 additions & 0 deletions packages/platform-observability/data_stream/audit/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
type: logs
title: Platform Observability Kibana Audit Logs
release: experimental
streams:
- input: logfile
vars:
- name: paths
type: text
title: Paths
multi: true
required: true
show_user: true
default:
- /tmp/service_logs/audit*.log
template_path: log.yml.hbs
title: Kibana Audit Logs
description: Collect Kibana Audit Logs
4 changes: 4 additions & 0 deletions packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Authorization header is not presented.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Could not handle authentication attempt","log":{"level":"DEBUG","logger":"plugins.security.authentication"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
{"_tag":"Right","right":"update_aliases_succeeded","ecs":{"version":"8.0.0"},"@timestamp":"2022-07-04T09:17:38.611+00:00","message":"[.kibana] MARK_VERSION_INDEX_READY RESPONSE","log":{"level":"DEBUG","logger":"savedobjects-service"},"process":{"pid":7},"trace":{"id":"a167d1124764379d4121b357e20baee2"},"transaction":{"id":"14717ae6e3b30d5a"}}
116 changes: 116 additions & 0 deletions ...tform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{
"expected": [
{
"@timestamp": "2022-06-29T11:24:17.898+00:00",
"data_stream": {
"dataset": "kibana-logs",
"namespace": "platform-observability",
"type": "logs"
},
"ecs": {
"version": "8.0.0"
},
"event": {
"dataset": "kibana-logs"
},
"log": {
"level": "DEBUG",
"logger": "plugins.security.http"
},
"message": "Trying to authenticate user request to /login.",
"process": {
"pid": 7
},
"trace": {
"id": "e6e1c25936546ec690b11a3b78b2a8db"
},
"transaction": {
"id": "3be6994d7f6d5465"
}
},
{
"@timestamp": "2022-06-29T11:24:17.898+00:00",
"data_stream": {
"dataset": "kibana-logs",
"namespace": "platform-observability",
"type": "logs"
},
"ecs": {
"version": "8.0.0"
},
"event": {
"dataset": "kibana-logs"
},
"log": {
"level": "DEBUG",
"logger": "plugins.security.http"
},
"message": "Authorization header is not presented.",
"process": {
"pid": 7
},
"trace": {
"id": "e6e1c25936546ec690b11a3b78b2a8db"
},
"transaction": {
"id": "3be6994d7f6d5465"
}
},
{
"@timestamp": "2022-06-29T11:24:17.898+00:00",
"data_stream": {
"dataset": "kibana-logs",
"namespace": "platform-observability",
"type": "logs"
},
"ecs": {
"version": "8.0.0"
},
"event": {
"dataset": "kibana-logs"
},
"log": {
"level": "DEBUG",
"logger": "plugins.security.authentication"
},
"message": "Could not handle authentication attempt",
"process": {
"pid": 7
},
"trace": {
"id": "e6e1c25936546ec690b11a3b78b2a8db"
},
"transaction": {
"id": "3be6994d7f6d5465"
}
},
{
"@timestamp": "2022-07-04T09:17:38.611+00:00",
"data_stream": {
"dataset": "kibana-logs",
"namespace": "platform-observability",
"type": "logs"
},
"ecs": {
"version": "8.0.0"
},
"event": {
"dataset": "kibana-logs"
},
"log": {
"level": "DEBUG",
"logger": "savedobjects-service"
},
"message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE",
"process": {
"pid": 7
},
"trace": {
"id": "a167d1124764379d4121b357e20baee2"
},
"transaction": {
"id": "14717ae6e3b30d5a"
}
}
]
}
8 changes: 8 additions & 0 deletions packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
paths:
{{#each paths}}
- {{this}}
{{/each}}
{{#if processors}}
processors:
{{processors}}
{{/if}}
71 changes: 71 additions & 0 deletions packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
---
description: Pipeline for parsing ECS logs
processors:
- remove:
field: data_stream.dataset
ignore_missing: true
- remove:
field: event.dataset
ignore_missing: true
- set:
description: Uses event.dataset as a default for data_stream.dataset if the latter is not set.
field: data_stream.dataset
copy_from: event.dataset
if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1
override: false
- script:
source: |
ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_')
if: ctx.data_stream?.dataset != null
- script:
source: |
ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_')
if: ctx.data_stream?.namespace != null
- set:
field: data_stream.type
value: logs
- set:
field: data_stream.dataset
value: kibana-logs
override: false
- set:
field: data_stream.namespace
value: platform-observability
override: false
- set:
field: event.dataset
copy_from: data_stream.dataset
- rename:
field: message
target_field: _ecs_json_message
if: |-
def message = ctx.message;
return message != null
&& message.startsWith('{')
&& message.endsWith('}')
&& message.contains('"@timestamp"')
ignore_missing: true
- json:
field: _ecs_json_message
add_to_root: true
add_to_root_conflict_strategy: merge
allow_duplicate_keys: true
if: ctx.containsKey('_ecs_json_message')
on_failure:
- rename:
field: _ecs_json_message
target_field: message
ignore_missing: true
- set:
field: error.message
value: Error while parsing JSON
override: false
- remove:
field: _ecs_json_message
ignore_missing: true
- remove:
field: _tag
ignore_missing: true
- remove:
field: right
ignore_missing: true
12 changes: 12 additions & 0 deletions packages/platform-observability/data_stream/log/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
- name: data_stream.type
type: constant_keyword
description: Data stream type.
- name: data_stream.dataset
type: constant_keyword
description: Data stream dataset.
- name: data_stream.namespace
type: constant_keyword
description: Data stream namespace.
- name: "@timestamp"
type: date
description: Event timestamp.
15 changes: 15 additions & 0 deletions packages/platform-observability/data_stream/log/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# only used for tests
- name: ecs.version
external: ecs
- name: log.level
external: ecs
- name: log.logger
external: ecs
- name: message
external: ecs
- name: process.pid
external: ecs
- name: trace.id
external: ecs
- name: transaction.id
external: ecs
Loading