From 25f60de3a45032e5933d45158ab867752f8754df Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 1 Apr 2022 17:48:55 +0200 Subject: [PATCH 1/4] Add ecs_log integration --- packages/ecs_log/_dev/example.ecs.json | 3 + packages/ecs_log/changelog.yml | 6 + .../data_stream/ecs/fields/base-fields.yml | 12 ++ .../ecs_log/data_stream/ecs/fields/ecs.yml | 185 ++++++++++++++++++ .../ecs_log/data_stream/ecs/fields/fields.yml | 109 +++++++++++ packages/ecs_log/data_stream/ecs/manifest.yml | 8 + .../ecs_router/agent/stream/stream.yml.hbs | 11 ++ .../elasticsearch/ingest_pipeline/default.yml | 37 ++++ .../logs-ecs-json-pipeline.yml | 31 +++ .../ecs_router/fields/base-fields.yml | 12 ++ .../data_stream/ecs_router/manifest.yml | 21 ++ packages/ecs_log/docs/README.md | 3 + packages/ecs_log/img/icon.svg | 4 + packages/ecs_log/manifest.yml | 24 +++ 14 files changed, 466 insertions(+) create mode 100644 packages/ecs_log/_dev/example.ecs.json create mode 100644 packages/ecs_log/changelog.yml create mode 100644 packages/ecs_log/data_stream/ecs/fields/base-fields.yml create mode 100644 packages/ecs_log/data_stream/ecs/fields/ecs.yml create mode 100644 packages/ecs_log/data_stream/ecs/fields/fields.yml create mode 100644 packages/ecs_log/data_stream/ecs/manifest.yml create mode 100644 packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs create mode 100644 packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml create mode 100644 packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml create mode 100644 packages/ecs_log/data_stream/ecs_router/manifest.yml create mode 100644 packages/ecs_log/docs/README.md create mode 100644 packages/ecs_log/img/icon.svg create mode 100644 packages/ecs_log/manifest.yml diff --git a/packages/ecs_log/_dev/example.ecs.json b/packages/ecs_log/_dev/example.ecs.json new file mode 100644 index 0000000000..34aa4f31b9 --- /dev/null +++ b/packages/ecs_log/_dev/example.ecs.json @@ -0,0 +1,3 @@ +{"@timestamp":"{\"@timestamp\":\"2022-04-01T12:09:12.375Z\", \"log.level\": \"INFO\", \"message\":\"Tomcat started on port(s): 8080 (http) with context path ''\", \"service.name\":\"spring-petclinic\",\"process.thread.name\":\"restartedMain\",\"log.logger\":\"org.springframework.boot.web.embedded.tomcat.TomcatWebServer\"}", "log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"} +{"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"Started PetClinicApplication in 7.095 seconds (JVM running for 9.082)", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.samples.petclinic.PetClinicApplication"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"init find form", "service.name":"spring-petclinic","process.thread.name":"http-nio-8080-exec-8","log.logger":"org.springframework.samples.petclinic.owner.OwnerController","transaction.id":"28b7fb8d5aba51f1","trace.id":"2869b25b5469590610fea49ac04af7da"} diff --git a/packages/ecs_log/changelog.yml b/packages/ecs_log/changelog.yml new file mode 100644 index 0000000000..4faa1f186b --- /dev/null +++ b/packages/ecs_log/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial release of the package + type: enhancement + link: https://github.com/elastic/integrations/pull diff --git a/packages/ecs_log/data_stream/ecs/fields/base-fields.yml b/packages/ecs_log/data_stream/ecs/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/ecs_log/data_stream/ecs/fields/ecs.yml b/packages/ecs_log/data_stream/ecs/fields/ecs.yml new file mode 100644 index 0000000000..d6f3cd6509 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs/fields/ecs.yml @@ -0,0 +1,185 @@ +- external: ecs + name: agent.ephemeral_id +- external: ecs + name: agent.name +- external: ecs + name: agent.version +- external: ecs + name: client.domain +- external: ecs + name: client.ip +- external: ecs + name: client.port +- external: ecs + name: client.geo.city_name +- external: ecs + name: client.geo.continent_name +- external: ecs + name: client.geo.country_iso_code +- external: ecs + name: client.geo.country_name +- external: ecs + name: client.geo.location +- external: ecs + name: client.geo.region_iso_code +- external: ecs + name: client.geo.region_name +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.instance.name +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.project.id +- external: ecs + name: cloud.project.name +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: cloud.service.name +- external: ecs + name: container.id +- external: ecs + name: destination.address +- external: ecs + name: destination.ip +- external: ecs + name: destination.port +- external: ecs + name: ecs.version +- external: ecs + name: event.outcome +- external: ecs + name: event.severity +- external: ecs + name: event.action +- external: ecs + name: host.architecture +- external: ecs + name: host.hostname +- external: ecs + name: host.ip +- external: ecs + name: host.name +- external: ecs + name: host.os.platform +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: labels + dynamic: true +- external: ecs + name: tags +- external: ecs + name: log.level +- external: ecs + name: message +- external: ecs + name: observer.hostname +- external: ecs + name: observer.name +- external: ecs + name: observer.type +- external: ecs + name: observer.version +- external: ecs + name: process.args +- external: ecs + name: process.pid +- external: ecs + name: process.ppid +- external: ecs + name: process.title +- external: ecs + name: service.environment +- external: ecs + name: service.name +- external: ecs + name: service.node.name +- external: ecs + name: service.version +- external: ecs + name: source.domain +- external: ecs + name: source.ip +- external: ecs + name: source.port +- external: ecs + name: source.nat.ip +- external: ecs + name: source.nat.port +- external: ecs + name: span.id +- external: ecs + name: trace.id +- external: ecs + name: transaction.id +- external: ecs + name: url.domain +- external: ecs + name: url.fragment +- external: ecs + name: url.full +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.port +- external: ecs + name: url.query +- external: ecs + name: url.scheme +- external: ecs + name: user.domain +- external: ecs + name: user.email +- external: ecs + name: user.id +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.family +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.kernel +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.platform +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version +- external: ecs + name: error.code +- external: ecs + name: error.id +- external: ecs + name: error.message +- external: ecs + name: error.stack_trace +- external: ecs + name: error.type diff --git a/packages/ecs_log/data_stream/ecs/fields/fields.yml b/packages/ecs_log/data_stream/ecs/fields/fields.yml new file mode 100644 index 0000000000..adbc32bacd --- /dev/null +++ b/packages/ecs_log/data_stream/ecs/fields/fields.yml @@ -0,0 +1,109 @@ +- name: http + type: group + fields: + - name: request + type: group + fields: + - name: headers + type: object + description: | + The canonical headers of the monitored HTTP request. + - name: response + type: group + fields: + - name: finished + type: boolean + description: | + Used by the Node agent to indicate when in the response life cycle an error has occurred. + - name: headers + type: object + description: | + The canonical headers of the monitored HTTP response. +- name: kubernetes + title: Kubernetes + type: group + description: | + Kubernetes metadata reported by agents + fields: + - name: namespace + type: keyword + description: | + Kubernetes namespace + - name: node + type: group + fields: + - name: name + type: keyword + description: | + Kubernetes node name + - name: pod + type: group + fields: + - name: name + type: keyword + description: | + Kubernetes pod name + - name: uid + type: keyword + description: | + Kubernetes Pod UID +- name: observer + type: group + fields: + - name: ephemeral_id + type: keyword + description: | + Ephemeral identifier of the APM Server. + - name: id + type: keyword + description: | + Unique identifier of the APM Server. +- name: processor.event + type: constant_keyword + description: Processor event. +- name: processor.name + type: constant_keyword + description: Processor name. +- name: service + type: group + description: | + Service fields. + fields: + - name: framework + type: group + fields: + - name: name + type: keyword + description: | + Name of the framework used. + - name: version + type: keyword + description: | + Version of the framework used. + - name: language + type: group + fields: + - name: name + type: keyword + description: | + Name of the programming language used. + - name: version + type: keyword + description: | + Version of the programming language used. + - name: runtime + type: group + fields: + - name: name + type: keyword + description: | + Name of the runtime used. + - name: version + type: keyword + description: | + Version of the runtime used. +- name: numeric_labels + type: object + dynamic: true + description: | + Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float. diff --git a/packages/ecs_log/data_stream/ecs/manifest.yml b/packages/ecs_log/data_stream/ecs/manifest.yml new file mode 100644 index 0000000000..659892b38d --- /dev/null +++ b/packages/ecs_log/data_stream/ecs/manifest.yml @@ -0,0 +1,8 @@ +title: Log Dataset +type: logs +dataset: ecs +dataset_is_prefix: true +elasticsearch: + index_template: + mappings: + dynamic: true diff --git a/packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs b/packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..6055afd8c5 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ +{{custom}} diff --git a/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..2e688cb90a --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,37 @@ +--- +processors: +- pipeline: + name: '{{ IngestPipeline "logs-ecs-json-pipeline" }}' + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') +- script: + source: | + ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') + if: ctx.data_stream?.dataset != null +- script: + source: | + ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_') + if: ctx.data_stream?.namespace != null +- set: + field: data_stream.type + value: logs + override: false +- set: + field: data_stream.dataset + value: generic + override: false +- set: + field: data_stream.namespace + value: default + override: false +- set: + field: event.dataset + copy_from: data_stream.dataset + override: true +- set: + field: _index + value: logs-{{{data_stream.dataset}}}-{{{data_stream.namespace}}} diff --git a/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml new file mode 100644 index 0000000000..55d5b64e9a --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml @@ -0,0 +1,31 @@ +--- +processors: +- rename: + field: message + target_field: _ecs_json_message + ignore_missing: true +- json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false +- remove: + field: _ecs_json_message + ignore_missing: true +- dot_expander: + field: "*" + override: true +- join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml b/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/ecs_log/data_stream/ecs_router/manifest.yml b/packages/ecs_log/data_stream/ecs_router/manifest.yml new file mode 100644 index 0000000000..d9f31ea371 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/manifest.yml @@ -0,0 +1,21 @@ +title: ECS Log Router Dataset +type: logs +dataset: ecs_router +streams: + - input: logfile + description: Collect your custom log files. + title: Collect log files + vars: + - name: paths + required: true + title: Log file path + description: Path to log files to be collected + type: text + multi: true + - name: custom + title: Custom configurations + description: > + Here YAML configuration options can be used to be added to your configuration. Be careful using this as it might break your configuration file. + + type: yaml + default: "" diff --git a/packages/ecs_log/docs/README.md b/packages/ecs_log/docs/README.md new file mode 100644 index 0000000000..89c5fab817 --- /dev/null +++ b/packages/ecs_log/docs/README.md @@ -0,0 +1,3 @@ +# ECS Log Package + +The ECS log package is used to ingest ECS log files. diff --git a/packages/ecs_log/img/icon.svg b/packages/ecs_log/img/icon.svg new file mode 100644 index 0000000000..173fdec507 --- /dev/null +++ b/packages/ecs_log/img/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/packages/ecs_log/manifest.yml b/packages/ecs_log/manifest.yml new file mode 100644 index 0000000000..3e42720317 --- /dev/null +++ b/packages/ecs_log/manifest.yml @@ -0,0 +1,24 @@ +format_version: 1.0.0 +name: ecs_log +title: ECS Logs +description: >- + Collect ECS logs with Elastic Agent. +type: integration +version: 0.0.1 +release: experimental +license: basic +categories: + - custom +policy_templates: + - name: logs + title: ECS logs + description: Collect your ECS log files. + inputs: + - type: logfile + title: ECS log file + description: Collect your ECS log files. +icons: + - src: "/img/icon.svg" + type: "image/svg+xml" +owner: + github: elastic/integrations From 712da31f87e3d7f5c19a61c66e6a97c0a86bbaa5 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 1 Apr 2022 17:49:32 +0200 Subject: [PATCH 2/4] Remove logs-ecs.* mappings, relying solely on logs-*-* --- .../data_stream/ecs/fields/base-fields.yml | 12 -- .../ecs_log/data_stream/ecs/fields/ecs.yml | 185 ------------------ .../ecs_log/data_stream/ecs/fields/fields.yml | 109 ----------- packages/ecs_log/data_stream/ecs/manifest.yml | 8 - 4 files changed, 314 deletions(-) delete mode 100644 packages/ecs_log/data_stream/ecs/fields/base-fields.yml delete mode 100644 packages/ecs_log/data_stream/ecs/fields/ecs.yml delete mode 100644 packages/ecs_log/data_stream/ecs/fields/fields.yml delete mode 100644 packages/ecs_log/data_stream/ecs/manifest.yml diff --git a/packages/ecs_log/data_stream/ecs/fields/base-fields.yml b/packages/ecs_log/data_stream/ecs/fields/base-fields.yml deleted file mode 100644 index 7c798f4534..0000000000 --- a/packages/ecs_log/data_stream/ecs/fields/base-fields.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: data_stream.type - type: constant_keyword - description: Data stream type. -- name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. -- name: data_stream.namespace - type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. diff --git a/packages/ecs_log/data_stream/ecs/fields/ecs.yml b/packages/ecs_log/data_stream/ecs/fields/ecs.yml deleted file mode 100644 index d6f3cd6509..0000000000 --- a/packages/ecs_log/data_stream/ecs/fields/ecs.yml +++ /dev/null @@ -1,185 +0,0 @@ -- external: ecs - name: agent.ephemeral_id -- external: ecs - name: agent.name -- external: ecs - name: agent.version -- external: ecs - name: client.domain -- external: ecs - name: client.ip -- external: ecs - name: client.port -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.project.name -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.service.name -- external: ecs - name: container.id -- external: ecs - name: destination.address -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: event.outcome -- external: ecs - name: event.severity -- external: ecs - name: event.action -- external: ecs - name: host.architecture -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.name -- external: ecs - name: host.os.platform -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: labels - dynamic: true -- external: ecs - name: tags -- external: ecs - name: log.level -- external: ecs - name: message -- external: ecs - name: observer.hostname -- external: ecs - name: observer.name -- external: ecs - name: observer.type -- external: ecs - name: observer.version -- external: ecs - name: process.args -- external: ecs - name: process.pid -- external: ecs - name: process.ppid -- external: ecs - name: process.title -- external: ecs - name: service.environment -- external: ecs - name: service.name -- external: ecs - name: service.node.name -- external: ecs - name: service.version -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: span.id -- external: ecs - name: trace.id -- external: ecs - name: transaction.id -- external: ecs - name: url.domain -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.family -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.kernel -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.platform -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: error.code -- external: ecs - name: error.id -- external: ecs - name: error.message -- external: ecs - name: error.stack_trace -- external: ecs - name: error.type diff --git a/packages/ecs_log/data_stream/ecs/fields/fields.yml b/packages/ecs_log/data_stream/ecs/fields/fields.yml deleted file mode 100644 index adbc32bacd..0000000000 --- a/packages/ecs_log/data_stream/ecs/fields/fields.yml +++ /dev/null @@ -1,109 +0,0 @@ -- name: http - type: group - fields: - - name: request - type: group - fields: - - name: headers - type: object - description: | - The canonical headers of the monitored HTTP request. - - name: response - type: group - fields: - - name: finished - type: boolean - description: | - Used by the Node agent to indicate when in the response life cycle an error has occurred. - - name: headers - type: object - description: | - The canonical headers of the monitored HTTP response. -- name: kubernetes - title: Kubernetes - type: group - description: | - Kubernetes metadata reported by agents - fields: - - name: namespace - type: keyword - description: | - Kubernetes namespace - - name: node - type: group - fields: - - name: name - type: keyword - description: | - Kubernetes node name - - name: pod - type: group - fields: - - name: name - type: keyword - description: | - Kubernetes pod name - - name: uid - type: keyword - description: | - Kubernetes Pod UID -- name: observer - type: group - fields: - - name: ephemeral_id - type: keyword - description: | - Ephemeral identifier of the APM Server. - - name: id - type: keyword - description: | - Unique identifier of the APM Server. -- name: processor.event - type: constant_keyword - description: Processor event. -- name: processor.name - type: constant_keyword - description: Processor name. -- name: service - type: group - description: | - Service fields. - fields: - - name: framework - type: group - fields: - - name: name - type: keyword - description: | - Name of the framework used. - - name: version - type: keyword - description: | - Version of the framework used. - - name: language - type: group - fields: - - name: name - type: keyword - description: | - Name of the programming language used. - - name: version - type: keyword - description: | - Version of the programming language used. - - name: runtime - type: group - fields: - - name: name - type: keyword - description: | - Name of the runtime used. - - name: version - type: keyword - description: | - Version of the runtime used. -- name: numeric_labels - type: object - dynamic: true - description: | - Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as scaled_float. diff --git a/packages/ecs_log/data_stream/ecs/manifest.yml b/packages/ecs_log/data_stream/ecs/manifest.yml deleted file mode 100644 index 659892b38d..0000000000 --- a/packages/ecs_log/data_stream/ecs/manifest.yml +++ /dev/null @@ -1,8 +0,0 @@ -title: Log Dataset -type: logs -dataset: ecs -dataset_is_prefix: true -elasticsearch: - index_template: - mappings: - dynamic: true From dc99b9477116408bdfc4cb94ff3ccbc85939994e Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 1 Apr 2022 18:15:26 +0200 Subject: [PATCH 3/4] Fix example --- packages/ecs_log/_dev/example.ecs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ecs_log/_dev/example.ecs.json b/packages/ecs_log/_dev/example.ecs.json index 34aa4f31b9..79966125b2 100644 --- a/packages/ecs_log/_dev/example.ecs.json +++ b/packages/ecs_log/_dev/example.ecs.json @@ -1,3 +1,3 @@ -{"@timestamp":"{\"@timestamp\":\"2022-04-01T12:09:12.375Z\", \"log.level\": \"INFO\", \"message\":\"Tomcat started on port(s): 8080 (http) with context path ''\", \"service.name\":\"spring-petclinic\",\"process.thread.name\":\"restartedMain\",\"log.logger\":\"org.springframework.boot.web.embedded.tomcat.TomcatWebServer\"}", "log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"} +{"@timestamp":"2022-04-01T12:09:12.375Z", "log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"} {"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"Started PetClinicApplication in 7.095 seconds (JVM running for 9.082)", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.samples.petclinic.PetClinicApplication"} {"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"init find form", "service.name":"spring-petclinic","process.thread.name":"http-nio-8080-exec-8","log.logger":"org.springframework.samples.petclinic.owner.OwnerController","transaction.id":"28b7fb8d5aba51f1","trace.id":"2869b25b5469590610fea49ac04af7da"} From 8074f739e0e309d1c3090b1003d5c49ced27602b Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 4 Apr 2022 11:44:40 +0200 Subject: [PATCH 4/4] Add pipeline tests --- packages/ecs_log/_dev/example.ecs.json | 3 - .../_dev/test/pipeline/test-ecs-json.log | 10 ++ .../pipeline/test-ecs-json.log-config.yml | 11 ++ .../pipeline/test-ecs-json.log-expected.json | 118 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 17 ++- .../ecs_router/fields/base-fields.yml | 11 ++ 6 files changed, 165 insertions(+), 5 deletions(-) delete mode 100644 packages/ecs_log/_dev/example.ecs.json create mode 100644 packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log create mode 100644 packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml create mode 100644 packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json diff --git a/packages/ecs_log/_dev/example.ecs.json b/packages/ecs_log/_dev/example.ecs.json deleted file mode 100644 index 79966125b2..0000000000 --- a/packages/ecs_log/_dev/example.ecs.json +++ /dev/null @@ -1,3 +0,0 @@ -{"@timestamp":"2022-04-01T12:09:12.375Z", "log.level": "INFO", "message":"Tomcat started on port(s): 8080 (http) with context path ''", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.boot.web.embedded.tomcat.TomcatWebServer"} -{"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"Started PetClinicApplication in 7.095 seconds (JVM running for 9.082)", "service.name":"spring-petclinic","process.thread.name":"restartedMain","log.logger":"org.springframework.samples.petclinic.PetClinicApplication"} -{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"init find form", "service.name":"spring-petclinic","process.thread.name":"http-nio-8080-exec-8","log.logger":"org.springframework.samples.petclinic.owner.OwnerController","transaction.id":"28b7fb8d5aba51f1","trace.id":"2869b25b5469590610fea49ac04af7da"} diff --git a/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log new file mode 100644 index 0000000000..112a60586a --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log @@ -0,0 +1,10 @@ +{"@timestamp":"2022-04-01T12:09:12.375Z", "log.level": "INFO", "message":"With event.dataset", "event.dataset": "foo"} +{"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"With data_stream.dataset", "data_stream.dataset": "bar"} +{"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"With invalid chars in dataset", "data_stream.dataset": "my-service"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"Without dataset"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"With stack trace", "error.stack_trace": "Exception in thread \"main\" java.lang.NullPointerException\n at com.example.myproject.Book.getTitle(Book.java:16)\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"With stack trace as array", "error.stack_trace": [ + "Exception in thread \"main\" java.lang.NullPointerException\n", + " at com.example.myproject.Book.getTitle(Book.java:16)\n", + " at com.example.myproject.Author.getBookTitles(Author.java:25)\n", + " at com.example.myproject.Bootstrap.main(Bootstrap.java:14)"]} diff --git a/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml new file mode 100644 index 0000000000..7464e902d0 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml @@ -0,0 +1,11 @@ +multiline: + first_line_pattern: '^{' +fields: + ecs: + version: "1.5.0" + event: + dataset: ecs_router + data_stream: + type: logs + dataset: ecs_router + namespace: default diff --git a/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json new file mode 100644 index 0000000000..de66933eb4 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json @@ -0,0 +1,118 @@ +{ + "expected": [ + { + "@timestamp": "2022-04-01T12:09:12.375Z", + "data_stream": { + "dataset": "foo", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "foo" + }, + "log": { + "level": "INFO" + }, + "message": "With event.dataset" + }, + { + "@timestamp": "2022-04-01T12:09:12.379Z", + "data_stream": { + "dataset": "bar", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "bar" + }, + "log": { + "level": "INFO" + }, + "message": "With data_stream.dataset" + }, + { + "@timestamp": "2022-04-01T12:09:12.379Z", + "data_stream": { + "dataset": "my_service", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "my_service" + }, + "log": { + "level": "INFO" + }, + "message": "With invalid chars in dataset" + }, + { + "@timestamp": "2022-04-01T14:08:40.199Z", + "data_stream": { + "dataset": "generic", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "generic" + }, + "log": { + "level": "DEBUG" + }, + "message": "Without dataset" + }, + { + "@timestamp": "2022-04-01T14:08:40.199Z", + "data_stream": { + "dataset": "generic", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "error": { + "stack_trace": "Exception in thread \"main\" java.lang.NullPointerException\n at com.example.myproject.Book.getTitle(Book.java:16)\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)" + }, + "event": { + "dataset": "generic" + }, + "log": { + "level": "DEBUG" + }, + "message": "With stack trace" + }, + { + "@timestamp": "2022-04-01T14:08:40.199Z", + "data_stream": { + "dataset": "generic", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "error": { + "stack_trace": "Exception in thread \"main\" java.lang.NullPointerException\n\n at com.example.myproject.Book.getTitle(Book.java:16)\n\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)" + }, + "event": { + "dataset": "generic" + }, + "log": { + "level": "DEBUG" + }, + "message": "With stack trace as array" + } + ] +} diff --git a/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml index 2e688cb90a..fd56ce6b98 100644 --- a/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml @@ -1,5 +1,14 @@ --- processors: +- remove: + description: | + This data stream is meant for routing only, we want to avoid that data is written to it. + We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default. + field: data_stream.dataset + ignore_missing: true +- remove: + field: event.dataset + ignore_missing: true - pipeline: name: '{{ IngestPipeline "logs-ecs-json-pipeline" }}' if: |- @@ -8,6 +17,12 @@ processors: && message.startsWith('{') && message.endsWith('}') && message.contains('"@timestamp"') +- set: + description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. + field: data_stream.dataset + copy_from: event.dataset + if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1 + override: false - script: source: | ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') @@ -19,7 +34,6 @@ processors: - set: field: data_stream.type value: logs - override: false - set: field: data_stream.dataset value: generic @@ -31,7 +45,6 @@ processors: - set: field: event.dataset copy_from: data_stream.dataset - override: true - set: field: _index value: logs-{{{data_stream.dataset}}}-{{{data_stream.namespace}}} diff --git a/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml b/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml index 7c798f4534..9de1ddcbf8 100644 --- a/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml +++ b/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml @@ -1,3 +1,5 @@ +# no data is meant to be written to the data stream +# however, in order for package validation and tests to pass, we need to add some fields - name: data_stream.type type: constant_keyword description: Data stream type. @@ -10,3 +12,12 @@ - name: '@timestamp' type: date description: Event timestamp. +# only used for tests +- name: ecs.version + external: ecs +- name: error.stack_trace + external: ecs +- name: log.level + external: ecs +- name: message + external: ecs