Filelog severity_parser operator does not set severity_text correctly #35049

perry-mitchell · 2024-09-06T10:45:42Z

Component(s)

receiver/filelog

What happened?

Description

I'm trying to use the severity_parser to parse the log stream, either stdout or stderr to set severity. It sets the severity_number correctly but the severity_text value becomes stdout or stderr, not INFO or ERROR as I'd expect.

Steps to Reproduce

Use the following severity parser (HCL format sorry):

    {
      type       = "severity_parser"
      parse_from = "attributes[\"log.iostream\"]"
      preset     = "none"
      mapping = {
        info  = "stdout"
        error = "stderr"
      }
    }

Expected Result

Mentioned above - correct severity_text

Actual Result

Logs like this:

{
    "body": "Error: Failed starting: Example error\n    at Timeout._onTimeout (/app/applications/builder-service/dist/server.cjs:101035:17)\n    at listOnTimeout (node:internal/timers:573:17)\n    at process.processTimers (node:internal/timers:514:7)",
    "severity_number": 17,
    "severity_text": "stderr",
    "attributes": {
        "appId": "c2f1a48c-1d6a-4399-a98d-f37cf254e8ab",
        "log.file.path": "/var/log/pods/app-c2f1a48c-1d6a-4399-a98d-f37cf254e8ab_builder-c8849596-vh27x_b2a2af4e-fbb0-4abc-b796-47ddd1b6a90d/builder/0.log",
        "log.iostream": "stderr",
        "logSource": "otel-filelog",
        "logtag": "F",
        "otelConfigRev": "1",
        "time": "2024-09-06T10:23:18.751279813Z"
    },
    "resource": {
        "k8s.container.name": "builder",
        "k8s.container.restart_count": "0",
        "k8s.namespace.name": "app-c2f1a48c-1d6a-4399-a98d-f37cf254e8ab",
        "k8s.pod.name": "builder-c8849596-vh27x",
        "k8s.pod.uid": "b2a2af4e-fbb0-4abc-b796-47ddd1b6a90d",
        "service.name": "builder"
    }
}

Collector version

0.101.1

Environment information

Environment

OS: Docker - Helm chart on Kubernetes

OpenTelemetry Collector configuration

"exporters":
  "awss3/logs":
    "s3uploader":
      "region": "eu-west-1"
      "s3_bucket": "<snip>"
      "s3_partition": "hour"
      "s3_prefix": "logs/"
  "awss3/metrics":
    "s3uploader":
      "region": "eu-west-1"
      "s3_bucket": "<snip>"
      "s3_partition": "hour"
      "s3_prefix": "metrics/"
  "awss3/traces":
    "s3uploader":
      "region": "eu-west-1"
      "s3_bucket": "<snip>"
      "s3_partition": "hour"
      "s3_prefix": "traces/"
  "datadog/exporter":
    "api":
      "key": "<snip>"
      "site": "<snip>"
  "nop": {}
  "otlphttp/mycompany":
    "endpoint": "https://<snip>"
"extensions":
  "health_check":
    "endpoint": "0.0.0.0:13133"
"processors":
  "attributes/app":
    "actions":
      - "action": "insert"
        "key": "appId"
        "value": "c2f1a48c-1d6a-4399-a98d-f37cf254e8ab"
      - "action": "insert"
        "key": "otelConfigRev"
        "value": "1"
  "batch/archive":
    "send_batch_size": 10000
    "timeout": "60s"
  "batch/stream": {}
  "filter/primary-app-mycompany":
    "error_mode": "silent"
    "logs":
      "log_record":
        - 'resource.attributes["service.name"] != "connector" and resource.attributes["service.name"]
          != "primary-app" and resource.attributes["service.name"] != "build-system"'
    "traces":
      "span":
        - 'resource.attributes["service.name"] != "connector" and resource.attributes["service.name"]
          != "primary-app" and resource.attributes["service.name"] != "build-system"'
      "spanevent":
        - 'resource.attributes["service.name"] != "connector" and resource.attributes["service.name"]
          != "primary-app" and resource.attributes["service.name"] != "build-system"'
"receivers":
  "filelog/primary-app":
    "exclude":
      - "/var/log/pods/*/otel-collector/*.log"
      - "/var/log/pods/*/opentelemetry-collector-agent/*.log"
      - "/var/log/pods/*/opentelemetry-collector/*.log"
    "include":
      - "/var/log/pods/app-system-c2f1a48c-1d6a-4399-a98d-f37cf254e8ab_*/connector/*.log"
      - "/var/log/pods/app-c2f1a48c-1d6a-4399-a98d-f37cf254e8ab_*/primary-app/*.log"
      - "/var/log/pods/app-c2f1a48c-1d6a-4399-a98d-f37cf254e8ab_*/build-system/*.log"
    "include_file_name": false
    "include_file_path": true
    "operators":
      - "id": "get-format"
        "routes":
          - "expr": "body matches \"^\\\\{\""
            "output": "parser-docker"
          - "expr": 'body matches "^[^ Z]+ "'
            "output": "parser-crio"
          - "expr": 'body matches "^[^ Z]+Z"'
            "output": "parser-containerd"
        "type": "router"
      - "id": "parser-crio"
        "output": "extract_metadata_from_filepath"
        "regex": "^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)? (?P<log>.*)$"
        "timestamp":
          "layout": "2006-01-02T15:04:05.999999999Z07:00"
          "layout_type": "gotime"
          "parse_from": "attributes.time"
        "type": "regex_parser"
      - "id": "parser-containerd"
        "output": "extract_metadata_from_filepath"
        "regex":
          "^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)?
          (?P<log>.*)$"
        "timestamp":
          "layout": "%Y-%m-%dT%H:%M:%S.%LZ"
          "parse_from": "attributes.time"
        "type": "regex_parser"
      - "id": "parser-docker"
        "output": "extract_metadata_from_filepath"
        "timestamp":
          "layout": "%Y-%m-%dT%H:%M:%S.%LZ"
          "parse_from": "attributes.time"
        "type": "json_parser"
      - "cache":
          "size": 128
        "id": "extract_metadata_from_filepath"
        "parse_from": 'attributes["log.file.path"]'
        "regex": "^.*\\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\\-]{36})\\/(?P<container_name>[^\\._]+)\\/(?P<restart_count>\\d+)\\.log$"
        "type": "regex_parser"
      - "field": "attributes.logSource"
        "type": "add"
        "value": "otel-filelog"
      - "from": "attributes.log"
        "to": "body"
        "type": "move"
      - "from": "attributes.stream"
        "to": 'attributes["log.iostream"]'
        "type": "move"
      - "from": "attributes.container_name"
        "to": 'resource["k8s.container.name"]'
        "type": "move"
      - "from": "attributes.namespace"
        "to": 'resource["k8s.namespace.name"]'
        "type": "move"
      - "from": "attributes.pod_name"
        "to": 'resource["k8s.pod.name"]'
        "type": "move"
      - "from": "attributes.restart_count"
        "to": 'resource["k8s.container.restart_count"]'
        "type": "move"
      - "from": "attributes.uid"
        "to": 'resource["k8s.pod.uid"]'
        "type": "move"
      - "from": 'resource["k8s.container.name"]'
        "to": 'resource["service.name"]'
        "type": "copy"
      - "combine_field": "body"
        "is_first_entry": "body matches \"^[^\\\\s]\""
        "source_identifier": 'attributes["log.file.path"]'
        "type": "recombine"
      - "mapping":
          "error": "stderr"
          "info": "stdout"
        "parse_from": 'attributes["log.iostream"]'
        "preset": "none"
        "type": "severity_parser"
    "start_at": "beginning"
  "filelog/global":
    "exclude":
      - "/var/log/pods/*/otel-collector/*.log"
      - "/var/log/pods/*/opentelemetry-collector-agent/*.log"
      - "/var/log/pods/*/opentelemetry-collector/*.log"
    "include":
      - "/var/log/pods/*/*/*.log"
    "include_file_name": false
    "include_file_path": true
    "operators":
      - "id": "get-format"
        "routes":
          - "expr": "body matches \"^\\\\{\""
            "output": "parser-docker"
          - "expr": 'body matches "^[^ Z]+ "'
            "output": "parser-crio"
          - "expr": 'body matches "^[^ Z]+Z"'
            "output": "parser-containerd"
        "type": "router"
      - "id": "parser-crio"
        "output": "extract_metadata_from_filepath"
        "regex": "^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)? (?P<log>.*)$"
        "timestamp":
          "layout": "2006-01-02T15:04:05.999999999Z07:00"
          "layout_type": "gotime"
          "parse_from": "attributes.time"
        "type": "regex_parser"
      - "id": "parser-containerd"
        "output": "extract_metadata_from_filepath"
        "regex":
          "^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)?
          (?P<log>.*)$"
        "timestamp":
          "layout": "%Y-%m-%dT%H:%M:%S.%LZ"
          "parse_from": "attributes.time"
        "type": "regex_parser"
      - "id": "parser-docker"
        "output": "extract_metadata_from_filepath"
        "timestamp":
          "layout": "%Y-%m-%dT%H:%M:%S.%LZ"
          "parse_from": "attributes.time"
        "type": "json_parser"
      - "cache":
          "size": 128
        "id": "extract_metadata_from_filepath"
        "parse_from": 'attributes["log.file.path"]'
        "regex": "^.*\\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\\-]{36})\\/(?P<container_name>[^\\._]+)\\/(?P<restart_count>\\d+)\\.log$"
        "type": "regex_parser"
      - "field": "attributes.logSource"
        "type": "add"
        "value": "otel-filelog"
      - "from": "attributes.log"
        "to": "body"
        "type": "move"
      - "from": "attributes.stream"
        "to": 'attributes["log.iostream"]'
        "type": "move"
      - "from": "attributes.container_name"
        "to": 'resource["k8s.container.name"]'
        "type": "move"
      - "from": "attributes.namespace"
        "to": 'resource["k8s.namespace.name"]'
        "type": "move"
      - "from": "attributes.pod_name"
        "to": 'resource["k8s.pod.name"]'
        "type": "move"
      - "from": "attributes.restart_count"
        "to": 'resource["k8s.container.restart_count"]'
        "type": "move"
      - "from": "attributes.uid"
        "to": 'resource["k8s.pod.uid"]'
        "type": "move"
      - "from": 'resource["k8s.container.name"]'
        "to": 'resource["service.name"]'
        "type": "copy"
      - "combine_field": "body"
        "is_first_entry": "body matches \"^[^\\\\s]\""
        "source_identifier": 'attributes["log.file.path"]'
        "type": "recombine"
      - "mapping":
          "error": "stderr"
          "info": "stdout"
        "parse_from": 'attributes["log.iostream"]'
        "preset": "none"
        "type": "severity_parser"
    "start_at": "beginning"
  "hostmetrics":
    "collection_interval": "15s"
    "root_path": "/hostfs"
    "scrapers":
      "cpu": {}
      "disk": {}
      "filesystem": {}
      "load": {}
      "memory": {}
      "network": {}
  "kubeletstats":
    "auth_type": "serviceAccount"
    "collection_interval": "15s"
    "endpoint": "${env:K8S_NODE_NAME}:10250"
    "insecure_skip_verify": true
    "metric_groups":
      - "node"
      - "pod"
      - "container"
  "otlp":
    "protocols":
      "http":
        "endpoint": "0.0.0.0:4318"
"service":
  "extensions":
    - "health_check"
  "pipelines":
    "logs/archive":
      "exporters":
        - "awss3/logs"
      "processors":
        - "attributes/app"
        - "batch/archive"
      "receivers":
        - "otlp"
        - "filelog/global"
    "logs/stream":
      "exporters":
        - "nop"
        - "datadog/exporter"
      "processors":
        - "attributes/app"
        - "batch/stream"
      "receivers":
        - "otlp"
        - "filelog/global"
    "logs/stream-mycompany":
      "exporters":
        - "otlphttp/mycompany"
      "processors":
        - "attributes/app"
        - "filter/primary-app-mycompany"
        - "batch/stream"
      "receivers":
        - "otlp"
        - "filelog/primary-app"
    "metrics/archive":
      "exporters":
        - "awss3/metrics"
      "processors":
        - "attributes/app"
        - "batch/archive"
      "receivers":
        - "otlp"
    "metrics/stream":
      "exporters":
        - "nop"
        - "datadog/exporter"
      "processors":
        - "attributes/app"
        - "batch/stream"
      "receivers":
        - "otlp"
        - "hostmetrics"
        - "kubeletstats"
    "metrics/stream-mycompany":
      "exporters":
        - "otlphttp/mycompany"
      "processors":
        - "attributes/app"
        - "batch/stream"
      "receivers":
        - "otlp"
        - "hostmetrics"
        - "kubeletstats"
    "traces/archive":
      "exporters":
        - "awss3/traces"
      "processors":
        - "attributes/app"
        - "batch/archive"
      "receivers":
        - "otlp"
    "traces/stream":
      "exporters":
        - "nop"
        - "datadog/exporter"
      "processors":
        - "attributes/app"
        - "batch/stream"
      "receivers":
        - "otlp"
    "traces/stream-mycompany":
      "exporters":
        - "otlphttp/mycompany"
      "processors":
        - "attributes/app"
        - "filter/primary-app-mycompany"
        - "batch/stream"
      "receivers":
        - "otlp"

Log output

2024-09-06T10:40:59.840Z    error    reader/reader.go:140    process: %w    {"kind": "receiver", "name": "filelog/global", "data_type": "logs", "component": "fileconsumer", "path": "/var/log/pods/ten-
21f752b4-d33c-42e2-a6a7-9cb69ae0dab9_dep-f16a1f57-4614-47ce-b038-3197165bb291-7cbbfbf96b-km5m8_3f5c3c4d-bc6e-40d2-9e99-6cb3239dc5b9/dep-f16a1f57-4614-47ce-b038-3197165bb291/0.log", "error": "failed to
 send entry after error: failed to send entry after error: failed to send entry after error: log entry does not have the expected parse_from field: {\"parse_from\":\"attributes['log.iostream']\"}"}   
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/reader.(*Reader).ReadToEnd                                                                                   
    github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/fileconsumer/internal/reader/reader.go:140                                                                            
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer.(*Manager).consume.func1                                                                                              
    github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/fileconsumer/file.go:160                                                                                              
2024-09-06T10:40:59.853Z    error    reader/reader.go:140    process: %w    {"kind": "receiver", "name": "filelog/global", "data_type": "logs", "component": "fileconsumer", "path": "/var/log/pods/ten-
21f752b4-d33c-42e2-a6a7-9cb69ae0dab9_dep-f16a1f57-4614-47ce-b038-3197165bb291-7cbbfbf96b-km5m8_3f5c3c4d-bc6e-40d2-9e99-6cb3239dc5b9/dep-f16a1f57-4614-47ce-b038-3197165bb291/0.log", "error": "failed to
 send entry after error: failed to send entry after error: move: field does not exist: attributes.stream"}                                                                                              
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/reader.(*Reader).ReadToEnd                                                                                   
    github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/fileconsumer/internal/reader/reader.go:140                                                                            
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer.(*Manager).consume.func1                                                                                              
    github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/fileconsumer/file.go:160                                                                                              
2024-09-06T10:40:59.853Z    error    reader/reader.go:140    process: %w    {"kind": "receiver", "name": "filelog/global", "data_type": "logs", "component": "fileconsumer", "path": "/var/log/pods/ten-
21f752b4-d33c-42e2-a6a7-9cb69ae0dab9_dep-f16a1f57-4614-47ce-b038-3197165bb291-7cbbfbf96b-km5m8_3f5c3c4d-bc6e-40d2-9e99-6cb3239dc5b9/dep-f16a1f57-4614-47ce-b038-3197165bb291/0.log", "error": "log entry
 does not have the expected parse_from field: {\"parse_from\":\"attributes['log.iostream']\"}"}
2024-09-06T10:41:00.425Z    info    TracesExporter    {"kind": "exporter", "data_type": "traces", "name": "debug", "resource spans": 1, "spans": 2}                             
2024-09-06T10:41:00.826Z    info    MetricsExporter    {"kind": "exporter", "data_type": "metrics", "name": "debug", "resource metrics": 60, "metrics": 622, "data points": 746}
2024-09-06T10:41:01.037Z    info    LogsExporter    {"kind": "exporter", "data_type": "logs", "name": "debug", "resource logs": 1, "log records": 4}                            
2024-09-06T10:41:01.247Z    info    MetricsExporter    {"kind": "exporter", "data_type": "metrics", "name": "debug", "resource metrics": 1, "metrics": 7, "data points": 17}    
2024-09-06T10:41:03.057Z    info    TracesExporter    {"kind": "exporter", "data_type": "traces", "name": "debug", "resource spans": 1, "spans": 3}                             
2024-09-06T10:41:03.256Z    info    LogsExporter    {"kind": "exporter", "data_type": "logs", "name": "debug", "resource logs": 1, "log records": 2}                            
2024-09-06T10:41:03.872Z    info    LogsExporter    {"kind": "exporter", "data_type": "logs", "name": "debug", "resource logs": 1, "log records": 2}

Additional context

No response

The text was updated successfully, but these errors were encountered:

github-actions · 2024-09-06T10:45:59Z

Pinging code owners:

receiver/filelog: @djaglowski

See Adding Labels via Comments if you do not have permissions to add labels yourself.

djaglowski · 2024-09-06T13:05:21Z

#26671 may do just what you need.

perry-mitchell · 2024-09-09T05:15:27Z

Thanks @djaglowski! Indeed that overwrite_text option was exactly what I needed here:

- "id": "parser_containerd"
  "output": "get_sub_format"
  "regex": "^(?P<time>[^ ^Z] Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)?
    (?P<log>.*)$"
  "severity":
    "mapping":
      "ERROR": "stderr"
      "INFO": "stdout"
    "overwrite_text": true
    "parse_from": "attributes.stream"
  "timestamp":
    "layout": "%Y-%m-%dT%H:%M:%S.%LZ"
    "parse_from": "attributes.time"
  "type": "regex_parser"

Seems it's missing from the docs. Regardless, I'll close this issue as my problem is solved (the severity text is now being set correctly). Cheers :)

djaglowski · 2024-09-09T13:44:33Z

I've opened #35074 to address the missing documentation. Thanks for calling this out @perry-mitchell.

…ng (#35074) See #35049 (comment)

…ng (open-telemetry#35074) See open-telemetry#35049 (comment)

perry-mitchell added bug Something isn't working needs triage New item requiring triage labels Sep 6, 2024

github-actions bot added the receiver/filelog label Sep 6, 2024

perry-mitchell mentioned this issue Sep 6, 2024

Not able to copy/move from sub object to attributes (merge) #35020

Open

Frapschen removed the needs triage New item requiring triage label Sep 6, 2024

perry-mitchell closed this as completed Sep 9, 2024

djaglowski mentioned this issue Sep 9, 2024

[chore] Add documenation for severity parser's 'overwrite_with' setting #35074

Merged

github-actions bot mentioned this issue Sep 10, 2024

Weekly Report: 2024-09-03 - 2024-09-10 #35086

Closed

djaglowski added a commit that referenced this issue Oct 10, 2024

[chore] Add documenation for severity parser's 'overwrite_with' setti…

86699bd

…ng (#35074) See #35049 (comment)

sbylica-splunk pushed a commit to sbylica-splunk/opentelemetry-collector-contrib that referenced this issue Dec 17, 2024

[chore] Add documenation for severity parser's 'overwrite_with' setti…

52b9c87

…ng (open-telemetry#35074) See open-telemetry#35049 (comment)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Filelog severity_parser operator does not set severity_text correctly #35049

Filelog severity_parser operator does not set severity_text correctly #35049

perry-mitchell commented Sep 6, 2024

github-actions bot commented Sep 6, 2024

djaglowski commented Sep 6, 2024

perry-mitchell commented Sep 9, 2024

djaglowski commented Sep 9, 2024