Allow only adding fields to the output; revert possibility of overwri…

…tting existing log entry field values
elastic · Oct 4, 2022 · 3028b52 · 3028b52
1 parent 141a888
commit 3028b52
Show file tree

Hide file tree

Showing 4 changed files with 175 additions and 166 deletions.
diff --git a/filebeat/module/kibana/log/config/log.yml b/filebeat/module/kibana/log/config/log.yml
@@ -6,19 +6,14 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-  # ECS: Same as json.keys_under_root: true. But here, overwrite_keys: false works, allowing module input fields to override the ingested log field values
-  - decode_json_fields:
-      fields: [message]
-      target: ""
-      overwrite_keys: false
-  # non-ECS: same as json.keys_under_root: false, allows compatibility with non-ecs logs. It also allows to store the orignal message in a separate field
-  - decode_json_fields:
-      fields: [message]
-      target: "json"
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.12.0
-      when:
-        not:
-          has_fields: ['ecs.version']
+  # non-ECS: same as json.keys_under_root: false, allows compatibility with non-ecs logs.
+- decode_json_fields:
+    fields: [message]
+    target: 'json'
+- add_fields:
+    target: ""
+    fields:
+      ecs.version: 1.12.0
+    when:
+      not:
+        has_fields: ['ecs.version']
diff --git a/filebeat/module/kibana/log/ingest/pipeline-7.yml b/filebeat/module/kibana/log/ingest/pipeline-7.yml
@@ -1,111 +1,105 @@
 description: Pipeline for parsing Kibana logs
 processors:
-  - set:
-      field: event.ingested
-      value: '{{_ingest.timestamp}}'
-  - set:
-      copy_from: '@timestamp'
-      field: event.created
-  # basically removes every property that exists in the json object from the root
-  - script:
-      lang: painless
-      source: >-
-        ctx.json.keySet()
-          .stream()
-          .forEach(key -> ctx.remove(key))
-      ignore_failure: true
-  - rename:
-      field: json
-      target_field: kibana.log.meta
-  - date:
-      field: kibana.log.meta.@timestamp
-      formats:
-        - ISO8601
-      target_field: '@timestamp'
-  - remove:
-      field: kibana.log.meta.@timestamp
-  - rename:
-      field: kibana.log.meta.message
-      target_field: message
-  - rename:
-      field: kibana.log.meta.state
-      target_field: kibana.log.state
-      ignore_missing: true
-  - rename:
-      field: kibana.log.meta.pid
-      target_field: process.pid
-  - rename:
-      field: kibana.log.meta.tags
-      target_field: kibana.log.tags
-  - rename:
-      field: kibana.log.meta.res.statusCode
-      target_field: http.response.status_code
-      ignore_missing: true
-  - script:
-      lang: painless
-      source: ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * 1000000L)
-      if: ctx?.kibana?.log?.meta?.res?.responseTime != null
-  - remove:
-      field: kibana.log.meta.res.responseTime
-      ignore_missing: true
-  - rename:
-      field: kibana.log.meta.res.contentLength
-      target_field: http.response.body.bytes
-      ignore_missing: true
-  - rename:
-      field: kibana.log.meta.req.method
-      target_field: http.request.method
-      ignore_missing: true
-  - rename:
-      field: kibana.log.meta.req.headers.referer
-      target_field: http.request.referrer
-      ignore_missing: true
-  - rename:
-      field: kibana.log.meta.req.headers.user-agent
-      target_field: user_agent.original
-      ignore_missing: true
-  - rename:
-      field: kibana.log.meta.req.remoteAddress
-      target_field: source.address
-      ignore_missing: true
-  - set:
-      field: source.ip
-      value: '{{source.address}}'
-      ignore_empty_value: true
-  - rename:
-      field: kibana.log.meta.req.url
-      target_field: url.original
-      ignore_missing: true
-  - remove:
-      field: kibana.log.meta.req.referer
-      ignore_missing: true
-  - remove:
-      field: kibana.log.meta.statusCode
-      ignore_missing: true
-  - remove:
-      field: kibana.log.meta.method
-      ignore_missing: true
-  - append:
-      field: service.name
-      value: kibana
-  - set:
-      field: event.kind
-      value: event
-  - script:
-      lang: painless
-      source: >-
-        if (ctx?.kibana?.log?.state != null) {
-          if (ctx.kibana.log.state == "red") {
-            ctx.event.type = "error";
-          } else {
-            ctx.event.type = "info";
-          }
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- set:
+    copy_from: '@timestamp'
+    field: event.created
+- rename:
+    field: json
+    target_field: kibana.log.meta
+- date:
+    field: kibana.log.meta.@timestamp
+    formats:
+      - ISO8601
+    target_field: '@timestamp'
+- remove:
+    field: kibana.log.meta.@timestamp
+- remove:
+    field: message
+- rename:
+    field: kibana.log.meta.message
+    target_field: message
+- rename:
+    field: kibana.log.meta.state
+    target_field: kibana.log.state
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.pid
+    target_field: process.pid
+- rename:
+    field: kibana.log.meta.tags
+    target_field: kibana.log.tags
+- rename:
+    field: kibana.log.meta.res.statusCode
+    target_field: http.response.status_code
+    ignore_missing: true
+- script:
+    lang: painless
+    source: ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * 1000000L)
+    if: ctx?.kibana?.log?.meta?.res?.responseTime != null
+- remove:
+    field: kibana.log.meta.res.responseTime
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.res.contentLength
+    target_field: http.response.body.bytes
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.method
+    target_field: http.request.method
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.referer
+    target_field: http.request.referrer
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.user-agent
+    target_field: user_agent.original
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.remoteAddress
+    target_field: source.address
+    ignore_missing: true
+- set:
+    field: source.ip
+    value: '{{source.address}}'
+    ignore_empty_value: true
+- rename:
+    field: kibana.log.meta.req.url
+    target_field: url.original
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.req.referer
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.statusCode
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.method
+    ignore_missing: true
+- append:
+    field: service.name
+    value: kibana
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      if (ctx?.kibana?.log?.state != null) {
+        if (ctx.kibana.log.state == "red") {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
         }
-  - set:
-      field: event.outcome
-      value: success
-      if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
-  - set:
-      field: event.outcome
-      value: failure
-      if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
+      }
+- set:
+    field: event.outcome
+    value: success
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
+- set:
+    field: event.outcome
+    value: failure
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'
diff --git a/filebeat/module/kibana/log/ingest/pipeline-ecs.yml b/filebeat/module/kibana/log/ingest/pipeline-ecs.yml
@@ -1,35 +1,55 @@
 description: Pipeline for parsing Kibana ECS logs
 processors:
-  - set:
-      field: event.ingested
-      value: '{{_ingest.timestamp}}'
-  - set:
-      copy_from: '@timestamp'
-      field: event.created
-  - set:
-      copy_from: 'json.@timestamp'
-      field: '@timestamp'
-      override: true
-  - set:
-      copy_from: json.message
-      field: message
-      override: true
-  - rename:
-      field: http.request.headers
-      target_field: kibana.log.meta.req.headers
-      ignore_missing: true
-  - rename:
-      field: http.response.headers
-      target_field: kibana.log.meta.res.headers
-      ignore_missing: true
-  - set:
-      field: event.outcome
-      value: success
-      if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
-  - set:
-      field: event.outcome
-      value: failure
-      if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'
-  - remove:
-      field: json
-      ignore_missing: true
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- set:
+    copy_from: '@timestamp'
+    field: event.created
+- rename:
+    field: message
+    target_field: _ecs_json_message
+    if: |-
+      def message = ctx.message;
+      return message != null
+          && message.startsWith('{')
+          && message.endsWith('}')
+          && message.contains('"@timestamp"')
+    ignore_missing: true
+- json:
+    field: _ecs_json_message
+    add_to_root: true
+    add_to_root_conflict_strategy: merge
+    allow_duplicate_keys: true
+    if: ctx.containsKey('_ecs_json_message')
+    on_failure:
+    - rename:
+        field: _ecs_json_message
+        target_field: message
+        ignore_missing: true
+    - set:
+        field: error.message
+        value: Error while parsing JSON
+        override: false
+- rename:
+    field: http.request.headers
+    target_field: kibana.log.meta.req.headers
+    ignore_missing: true
+- rename:
+    field: http.response.headers
+    target_field: kibana.log.meta.res.headers
+    ignore_missing: true
+- set:
+    field: event.outcome
+    value: success
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
+- set:
+    field: event.outcome
+    value: failure
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'
+- remove:
+    field: json
+    ignore_missing: true
+- remove:
+    field: _ecs_json_message
+    ignore_missing: true
diff --git a/filebeat/module/kibana/log/ingest/pipeline.yml b/filebeat/module/kibana/log/ingest/pipeline.yml
@@ -1,12 +1,12 @@
 description: Pipeline for parsing Kibana logs
 on_failure:
-  - set:
-      field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
 processors:
-  - pipeline:
-      if: 'ctx?.json?.ecs?.version == null'
-      name: '{< IngestPipeline "pipeline-7" >}'
-  - pipeline:
-      if: 'ctx?.json?.ecs?.version != null'
-      name: '{< IngestPipeline "pipeline-ecs" >}'
+- pipeline:
+    if: 'ctx?.json?.ecs?.version == null'
+    name: '{< IngestPipeline "pipeline-7" >}'
+- pipeline:
+    if: 'ctx?.json?.ecs?.version != null'
+    name: '{< IngestPipeline "pipeline-ecs" >}'