Cherry-pick to 7.0: Add syslog support for ISO8601 format timestamp (#…

…10736) (#10776) * Add syslog support for ISO8601 format timestamp (#10736) * Add syslog support for ISO8601 format timestamp * Add changelog * Change timestamp pattern (cherry picked from commit cb241f3) * only cherrypick the correct changelog
elastic · Feb 18, 2019 · b1fd1f4 · b1fd1f4
1 parent ba372bb
commit b1fd1f4
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -64,6 +64,8 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di
 
 *Filebeat*
 
+- Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736]
+
 *Heartbeat*
 
 *Journalbeat*

diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json
@@ -6,7 +6,8 @@
                 "field": "message",
                 "patterns": [
                     "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.syslog.message}",
-                    "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}"
+                    "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}",
+                    "%{TIMESTAMP_ISO8601:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.syslog.message}"
                 ],
                 "pattern_definitions" : {
                     "GREEDYMULTILINE" : "(.|\n)*"
@@ -32,7 +33,8 @@
                 "target_field": "@timestamp",
                 "formats": [
                     "MMM  d HH:mm:ss",
-                    "MMM dd HH:mm:ss"
+                    "MMM dd HH:mm:ss",
+                    "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZZ"
                 ],
                 {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >}
                 "ignore_failure": true

diff --git a/filebeat/module/system/syslog/test/suse-syslog.log b/filebeat/module/system/syslog/test/suse-syslog.log
@@ -0,0 +1,2 @@
+2018-08-14T14:30:02.203151+02:00 linux-sqrz systemd[4179]: Stopped target Basic System.
+2018-08-14T14:30:02.203251+02:00 linux-sqrz systemd[4179]: Stopped target Paths.
diff --git a/filebeat/module/system/syslog/test/suse-syslog.log-expected.json b/filebeat/module/system/syslog/test/suse-syslog.log-expected.json
@@ -0,0 +1,28 @@
+[
+    {
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "system.syslog",
+        "event.module": "system",
+        "fileset.name": "syslog",
+        "host.hostname": "linux-sqrz",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "Stopped target Basic System.",
+        "process.name": "systemd",
+        "process.pid": 4179,
+        "service.type": "system"
+    },
+    {
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "system.syslog",
+        "event.module": "system",
+        "fileset.name": "syslog",
+        "host.hostname": "linux-sqrz",
+        "input.type": "log",
+        "log.offset": 88,
+        "message": "Stopped target Paths.",
+        "process.name": "systemd",
+        "process.pid": 4179,
+        "service.type": "system"
+    }
+]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		2018-08-14T14:30:02.203151+02:00 linux-sqrz systemd[4179]: Stopped target Basic System.
		2018-08-14T14:30:02.203251+02:00 linux-sqrz systemd[4179]: Stopped target Paths.