[Platform Observability] Create initial PO package for ingesting kibana ECS formatted logs

.github/CODEOWNERS

@@ -168,3 +168,4 @@
 /packages/zscaler @elastic/security-external-integrations
 /packages/zscaler_zia @elastic/security-external-integrations
 /packages/zscaler_zpa @elastic/security-external-integrations
+/packages/platform_observability @elastic/infra-monitoring-ui

packages/platform_observability/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

packages/platform_observability/_dev/build/docs/README.md

@@ -0,0 +1,23 @@
+# Platform Observability
+
+## Compatibility
+
+This package works with Kibana 8.0.0 and later.
+
+## Kibana logs
+
+The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/en/kibana/current/introduction.html) instance.
+
+### Logs
+
+#### Audit
+
+Configure `Path` pointing to the location where audit logs will be created, based on the [Kibana Audit logging settings](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html#audit-logging-settings) in `kibana.yml`
+
+{{fields "kibana_audit"}}
+
+#### Log
+
+Configure `Path` pointing to the location where the logs will be created, based on the [Kibana logging settings](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html#logging-appenders) in `kibana.yml`
+
+{{fields "kibana_log"}}

packages/platform_observability/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1

packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log

@@ -0,0 +1 @@
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}}

packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json

@@ -0,0 +1,58 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-06-29T12:05:03.742+00:00",
+            "data_stream": {
+                "dataset": "kibana-audit-log",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "http_request",
+                "category": [
+                    "web"
+                ],
+                "dataset": "kibana-audit-log",
+                "outcome": "unknown"
+            },
+            "http": {
+                "request": {
+                    "method": "get"
+                }
+            },
+            "kibana": {
+                "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=",
+                "space_id": "default"
+            },
+            "log": {
+                "level": "INFO",
+                "logger": "plugins.security.audit.ecs"
+            },
+            "message": "User is requesting [/internal/security/session] endpoint",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9"
+            },
+            "transaction": {
+                "id": "f8863d86567119e6"
+            },
+            "url": {
+                "domain": "localhost",
+                "path": "/internal/security/session",
+                "port": 5601,
+                "scheme": "http"
+            },
+            "user": {
+                "name": "elastic",
+                "roles": [
+                    "superuser"
+                ]
+            }
+        }
+    ]
+}

packages/platform_observability/data_stream/kibana_audit/agent/stream/log.yml.hbs

@@ -0,0 +1,8 @@
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+{{#if processors}}
+processors:
+{{processors}}  
+{{/if}}

packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,69 @@
+---
+description: Pipeline for parsing Kibana Audit ECS formatted logs
+processors:
+  - remove:
+      field: data_stream.dataset
+      ignore_missing: true
+  - remove:
+      field: event.dataset
+      ignore_missing: true
+  - set:
+      description: Uses event.dataset as a default for data_stream.dataset if the latter is not set.
+      field: data_stream.dataset
+      copy_from: event.dataset
+      if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1
+      override: false
+  - script:
+      source: |
+        ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_')
+      if: ctx.data_stream?.dataset != null
+  - script:
+      source: |
+        ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_')
+      if: ctx.data_stream?.namespace != null
+  - set:
+      field: data_stream.type
+      value: logs
+  - set:
+      field: data_stream.dataset
+      value: kibana-audit-log
+      override: false
+  - set:
+      field: data_stream.namespace
+      value: platform-observability
+      override: false
+  - set:
+      field: event.dataset
+      copy_from: data_stream.dataset
+  - rename:
+      field: message
+      target_field: _ecs_json_message
+      if: |-
+        def message = ctx.message;
+        return message != null
+            && message.startsWith('{')
+            && message.endsWith('}')
+            && message.contains('"@timestamp"')
+      ignore_missing: true
+  - json:
+      field: _ecs_json_message
+      add_to_root: true
+      add_to_root_conflict_strategy: merge
+      allow_duplicate_keys: true
+      if: ctx.containsKey('_ecs_json_message')
+      on_failure:
+        - rename:
+            field: _ecs_json_message
+            target_field: message
+            ignore_missing: true
+        - set:
+            field: error.message
+            value: Error while parsing JSON
+            override: false
+  - remove:
+      field: _ecs_json_message
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

packages/platform_observability/data_stream/kibana_audit/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.

packages/platform_observability/data_stream/kibana_audit/fields/beats.yml

@@ -0,0 +1,6 @@
+- name: kibana.session_id
+  description: The ID of the user session associated with this event. Each login attempt results in a unique session id
+  type: keyword
+- name: kibana.space_id
+  description: The id of the space associated with this event.
+  type: keyword

packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml

@@ -0,0 +1,31 @@
+# only used for tests
+- name: ecs.version
+  external: ecs
+- name: http.request.method
+  external: ecs
+- name: log.level
+  external: ecs
+- name: log.logger
+  external: ecs
+- name: message
+  external: ecs
+- name: process.pid
+  external: ecs
+- name: trace.id
+  external: ecs
+- name: transaction.id
+  external: ecs
+- name: url.domain
+  external: ecs
+- name: url.path
+  external: ecs
+- name: url.port
+  external: ecs
+- name: url.query
+  external: ecs
+- name: url.scheme
+  external: ecs
+- name: user.name
+  external: ecs
+- name: user.roles
+  external: ecs

packages/platform_observability/data_stream/kibana_audit/manifest.yml

@@ -0,0 +1,17 @@
+type: logs
+title: Platform Observability Kibana audit logs
+release: experimental
+streams:
+  - input: logfile
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - /var/log/kibana/*_audit.json
+    template_path: log.yml.hbs
+    title: Kibana audit logs
+    description: Collect Kibana audit logs

packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log

@@ -0,0 +1,3 @@
+{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
+{"http":{"request":{"id":"unknownId","method":"GET"},"response":{"body":{"bytes":118},"status_code":200}},"url":{"path":"/_nodes","query":"filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-07-14T10:35:25.366+00:00","message":"200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip","log":{"level":"DEBUG","logger":"elasticsearch.query.data"},"process":{"pid":7},"trace":{"id":"0cd8dd5a3483159a43c07e9205432775"},"transaction":{"id":"6301eca88fba8d99"}}
+{"_tag":"Right","right":"update_aliases_succeeded","ecs":{"version":"8.0.0"},"@timestamp":"2022-07-04T09:17:38.611+00:00","message":"[.kibana] MARK_VERSION_INDEX_READY RESPONSE","log":{"level":"DEBUG","logger":"savedobjects-service"},"process":{"pid":7},"trace":{"id":"a167d1124764379d4121b357e20baee2"},"transaction":{"id":"14717ae6e3b30d5a"}}

packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json

@@ -0,0 +1,104 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-06-29T11:24:17.898+00:00",
+            "data_stream": {
+                "dataset": "kibana-logs",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "dataset": "kibana-logs"
+            },
+            "log": {
+                "level": "DEBUG",
+                "logger": "plugins.security.http"
+            },
+            "message": "Trying to authenticate user request to /login.",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "e6e1c25936546ec690b11a3b78b2a8db"
+            },
+            "transaction": {
+                "id": "3be6994d7f6d5465"
+            }
+        },
+        {
+            "@timestamp": "2022-07-14T10:35:25.366+00:00",
+            "data_stream": {
+                "dataset": "kibana-logs",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "dataset": "kibana-logs"
+            },
+            "http": {
+                "request": {
+                    "id": "unknownId",
+                    "method": "GET"
+                },
+                "response": {
+                    "body": {
+                        "bytes": 118
+                    },
+                    "status_code": 200
+                }
+            },
+            "log": {
+                "level": "DEBUG",
+                "logger": "elasticsearch.query.data"
+            },
+            "message": "200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "0cd8dd5a3483159a43c07e9205432775"
+            },
+            "transaction": {
+                "id": "6301eca88fba8d99"
+            },
+            "url": {
+                "path": "/_nodes",
+                "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip"
+            }
+        },
+        {
+            "@timestamp": "2022-07-04T09:17:38.611+00:00",
+            "data_stream": {
+                "dataset": "kibana-logs",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "dataset": "kibana-logs"
+            },
+            "log": {
+                "level": "DEBUG",
+                "logger": "savedobjects-service"
+            },
+            "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "a167d1124764379d4121b357e20baee2"
+            },
+            "transaction": {
+                "id": "14717ae6e3b30d5a"
+            }
+        }
+    ]
+}

packages/platform_observability/data_stream/kibana_log/agent/stream/log.yml.hbs

@@ -0,0 +1,8 @@
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+{{#if processors}}
+processors:
+{{processors}}  
+{{/if}}