feat(otellogs): add multipart merge configuration for docker and cri (#…

…2162) * feat(otellogs): add multipart merge configuration for docker and cri Signed-off-by: Dominik Rosiek <[email protected]> * doc(values): update comments for otellogs recombine plugin Signed-off-by: Dominik Rosiek <[email protected]> * docs(changelog): update Signed-off-by: Dominik Rosiek <[email protected]> * doc(otellogs): fix comments in value.yaml Signed-off-by: Dominik Rosiek <[email protected]>
SumoLogic · Mar 1, 2022 · 0bc1d37 · 0bc1d37
1 parent d22bbe5
commit 0bc1d37
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - feat(fluent-bit): drop all capabilities for container [#2151][#2151]
 - feat: allow to collect logs from /var/log/pods and add instruction how to do it [#2153][#2153] [#2156][#2156]
 - feat(otellogs): support tolerations, nodeSelector and affinity for daemonset [#2158][#2158]
+- feat(otellogs): add multipart merge configuration for docker and cri [#2162][#2162]
 
 ### Fixed
 
@@ -37,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2153]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2153
 [#2156]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2156
 [#2158]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2158
+[#2162]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2162
 
 ## [v2.5.2]
 

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -4466,7 +4466,7 @@ otellogs:
           - id: parser-crio
             type: regex_parser
             regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
-            output: extract-metadata-from-filepath
+            output: merge-cri-lines
             timestamp:
               parse_from: time
               layout_type: gotime
@@ -4475,7 +4475,7 @@ otellogs:
           - id: parser-containerd
             type: regex_parser
             regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
-            output: extract-metadata-from-filepath
+            output: merge-cri-lines
             timestamp:
               parse_from: time
               layout: '%Y-%m-%dT%H:%M:%S.%LZ'
@@ -4488,28 +4488,36 @@ otellogs:
           ## Output Timestamp: 2021-11-25 09:59:13.23887954 +0000 UTC
           - id: parser-docker
             type: json_parser
-            output: extract-metadata-from-filepath
+            output: merge-docker-lines
             timestamp:
               parse_from: time
               layout: '%Y-%m-%dT%H:%M:%S.%LZ'
 
-          ## This operator is currently disabled as it adds additional newlines
-          ## https://github.com/open-telemetry/opentelemetry-log-collection/issues/314
-          ##
-          ## merge-split-lines stitches back together log lines split by Docker logging driver.
+          ## merge-docker-lines stitches back together log lines split by Docker logging driver.
           ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "stream": "stdout" }
           ## Input Body (JSON): { "log": "ne that was split by the logging driver\n", "stream": "stdout" }
           ## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }
-          # - id: merge-split-lines
-          #   type: recombine
-          #   combine_field: log
-          #   is_last_entry: $$body.log matches "\n$"
+          - id: merge-docker-lines
+            type: recombine
+            output: extract-metadata-from-filepath
+            combine_field: log
+            is_last_entry: $$body.log matches "\n$"
 
-          ## This operator is currently disabled due to the following issues:
-          ## - Additional newlines being added between the merged logs:
-          ## https://github.com/open-telemetry/opentelemetry-log-collection/issues/314
-          ## - No flushing: https://github.com/open-telemetry/opentelemetry-log-collection/issues/306
-          #
+          ## merge-cri-lines stitches back together log lines split by CRI logging drivers.
+          ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" }
+          ## Input Body (JSON): { "log": "ne that was split by the logging driver", "logtag": "F" }
+          ## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }
+          - id: merge-cri-lines
+            type: recombine
+            output: extract-metadata-from-filepath
+            combine_field: log
+            combine_with: ""
+            is_last_entry: $$body.logtag == "F"
+            overwrite_with: newest
+
+          ## This operator is currently disabled because it combines logs before any entry has been matched:
+          ## https://github.com/open-telemetry/opentelemetry-log-collection/pull/416
+          ##
           ## merge-multiline-logs merges incoming log records into multiline logs.
           ## Input Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" }
           ## Input Body (JSON): { "log": "  second line\n", "stream": "stdout" }

diff --git a/tests/helm/logs_otc/static/basic.output.yaml b/tests/helm/logs_otc/static/basic.output.yaml
@@ -41,26 +41,38 @@ data:
             output: parser-containerd
           type: router
         - id: parser-crio
-          output: extract-metadata-from-filepath
+          output: merge-cri-lines
           regex: ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$
           timestamp:
             layout: "2006-01-02T15:04:05.000000000-07:00"
             layout_type: gotime
             parse_from: time
           type: regex_parser
         - id: parser-containerd
-          output: extract-metadata-from-filepath
+          output: merge-cri-lines
           regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$
           timestamp:
             layout: '%Y-%m-%dT%H:%M:%S.%LZ'
             parse_from: time
           type: regex_parser
         - id: parser-docker
-          output: extract-metadata-from-filepath
+          output: merge-docker-lines
           timestamp:
             layout: '%Y-%m-%dT%H:%M:%S.%LZ'
             parse_from: time
           type: json_parser
+        - combine_field: log
+          id: merge-docker-lines
+          is_last_entry: $$body.log matches "\n$"
+          output: extract-metadata-from-filepath
+          type: recombine
+        - combine_field: log
+          combine_with: ""
+          id: merge-cri-lines
+          is_last_entry: $$body.logtag == "F"
+          output: extract-metadata-from-filepath
+          overwrite_with: newest
+          type: recombine
         - id: extract-metadata-from-filepath
           parse_from: $$attributes["file.path"]
           regex: ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$