From 9e85a220d6cfb43b9394d2cc03ea124ee4b4dce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 Aug 2018 19:12:47 +0200 Subject: [PATCH 01/13] add function to add tag to a message --- filebeat/reader/message.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/filebeat/reader/message.go b/filebeat/reader/message.go index a92bc9092ccf..5051e676c60e 100644 --- a/filebeat/reader/message.go +++ b/filebeat/reader/message.go @@ -74,3 +74,15 @@ func (m *Message) AddFlagsWithKey(key string, flags ...string) error { return common.AddTagsWithKey(m.Fields, key, flags) } + +func (msg *Message) AddTagsWithKey(key string, tags []string) error { + if len(tags) == 0 { + return nil + } + + if msg.Fields == nil { + msg.Fields = common.MapStr{} + } + + return common.AddTagsWithKey(msg.Fields, key, tags) +} From d882b2452b8abf0955b772be0a20b7c24cda8a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 Aug 2018 19:13:44 +0200 Subject: [PATCH 02/13] add changelog entry --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6b2827debab3..1496d523f522 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -105,6 +105,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Keep raw user agent information after parsing as user_agent_raw in Filebeat modules. {pull}7823[7832] - Make docker input check if container strings are empty {pull}7960[7960] - Add tag "truncated" to "log.flags" if incoming line is longer than configured limit. {pull}7991[7991] +- Add tag "truncated" to "log.status" if incoming line is longer than configured limit. {pull}7991[7991] *Heartbeat* From 4516764b93a202f2f112c61f64c3cd594d6ac20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 Aug 2018 20:31:56 +0200 Subject: [PATCH 03/13] fix variable --- filebeat/reader/message.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/filebeat/reader/message.go b/filebeat/reader/message.go index 5051e676c60e..0ce2d2b37498 100644 --- a/filebeat/reader/message.go +++ b/filebeat/reader/message.go @@ -80,9 +80,9 @@ func (msg *Message) AddTagsWithKey(key string, tags []string) error { return nil } - if msg.Fields == nil { - msg.Fields = common.MapStr{} + if m.Fields == nil { + m.Fields = common.MapStr{} } - return common.AddTagsWithKey(msg.Fields, key, tags) + return common.AddTagsWithKey(m.Fields, key, tags) } From 9df6ac655bfb5f11c64b3960d394591d76c50896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 28 Aug 2018 12:01:34 +0200 Subject: [PATCH 04/13] follow up in filebeat modules test --- .../server/test/test.log-expected.json | 15 +++++++++++++++ .../slowlog/test/test.log-expected.json | 3 +++ .../icinga/main/test/test.log-expected.json | 3 +++ ...esql-9.6-debian-with-slowlog.log-expected.json | 15 +++++++++++++++ .../test/darwin-syslog-sample.log-expected.json | 3 +++ 5 files changed, 39 insertions(+) diff --git a/filebeat/module/elasticsearch/server/test/test.log-expected.json b/filebeat/module/elasticsearch/server/test/test.log-expected.json index d58429547ca5..8ddf0e51bd81 100644 --- a/filebeat/module/elasticsearch/server/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/test.log-expected.json @@ -168,6 +168,9 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.status": [ + "multiline" + ], "message": "master left (reason = failed to ping, tried [3] times, each with maximum [30s] timeout), current nodes: nodes:\n {srvmulpvlsk252_md}{uc5xdiQgRhaBIY-sszgjvQ}{X9pC0t1UQQix_NNOM0J6JQ}{srvmulpvlsk252.loganalytics.santanderuk.corp}{180.39.9.93:9300}{ml.max_open_jobs=10, ml.enabled=true}, local\n {srvmulpvlsk258_md}{HgW6EDn5QCmWVmICy4saHw}{o8zku7OJR4CTp0IjY8Ag4Q}{srvmulpvlsk258.loganalytics.santanderuk.corp}{180.39.9.99:9300}{ml.max_open_jobs=10, ml.enabled=true}\n {srvmulpvlsk250_md}{igrwSoPGSJ6u_5b8k26tgQ}{PuRqciBFRbiQvL2_lS7LrQ}{srvmulpvlsk250.loganalytics.santanderuk.corp}{180.39.9.91:9300}{ml.max_open_jobs=10, ml.enabled=true}, master\n {srvmulpvlsk254_id}{wZYeAh2URc2NwBIHZolLWQ}{3nduupo-TzSPaXjQaNu4Sg}{srvmulpvlsk254.loganalytics.santanderuk.corp}{180.39.9.95:9300}{ml.max_open_jobs=10, ml.enabled=true}", "offset": 2008, "prospector.type": "log", @@ -180,6 +183,9 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.status": [ + "multiline" + ], "message": "path: /_xpack/monitoring/_bulk, params: {system_id=logstash, system_api_version=2, interval=1s}\norg.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/2/no master];\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:151) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:57) ~[?:?]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:40) ~[?:?]\n at org.elasticsearch.action.support.TransportAction.doExecute(TransportAction.java:146) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:133) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$authorizeRequest$4(SecurityActionFilter.java:208) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.maybeRun(AuthorizationUtils.java:127) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.setRunAsRoles(AuthorizationUtils.java:121) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.authorize(AuthorizationUtils.java:109) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:186) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:212) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$4(AuthenticationService.java:246) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:257) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:159) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:122) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:185) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:145) ~[?:?]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:168) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:408) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction.lambda$doPrepareRequest$0(RestMonitoringBulkAction.java:77) ~[?:?]\n at org.elasticsearch.rest.BaseRestHandler.handleReques", "offset": 2907, "prospector.type": "log", @@ -192,6 +198,9 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.status": [ + "multiline" + ], "message": "path: /_xpack/license, params: {}\norg.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$4.onTimeout(TransportMasterNodeAction.java:209) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:311) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:139) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:111) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:194) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:107) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.handleException(TransportMasterNodeAction.java:183) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$Adapter.lambda$onConnectionClosed$6(TransportService.java:893) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) [elasticsearch-5.6.3.jar:5.6.3]\n at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]\n at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\nCaused by: org.elasticsearch.transport.NodeDisconnectedException: [srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected", "offset": 7412, "prospector.type": "log", @@ -207,6 +216,9 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.status": [ + "multiline" + ], "message": "duration [3.8s], collections [1]/[4.3s], total [3.8s]/[8.8h], memory [16.5gb]->[15.7gb]/[30.8gb], all_po\nols {[young] [1.2gb]->[24mb]/[1.4gb]}{[survivor] [191.3mb]->[191.3mb]/[191.3mb]}{[old] [15.1gb]->[15.5gb]/[29.1gb]}", "offset": 9873, "prospector.type": "log", @@ -247,6 +259,9 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.status": [ + "multiline" + ], "message": "monitoring execution failed\norg.elasticsearch.xpack.monitoring.exporter.ExportException: Exception when closing export bulk\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1$1.(ExportBulk.java:106) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1.onFailure(ExportBulk.java:104) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:217) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:211) ~[?:?]\n at org.elasticsearch.xpack.common.IteratingActionListener.onResponse(IteratingActionListener.java:108) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) [elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk$1.onSuccess(HttpExportBulk.java:115) [x-pack-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$FailureTrackingResponseListener.onSuccess(RestClient.java:597) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:352) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:343) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119) [httpcore-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:326) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) [httpcore-nio-4.4.5.jar:4.4.5]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\n", "offset": 10648, "prospector.type": "log", diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index b1c1828b6cad..fc7cc067e9fc 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -128,6 +128,9 @@ "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", + "log.status": [ + "multiline" + ], "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }]", "offset": 4753, "prospector.type": "log", diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index 57174332ba33..11322e4fbaa2 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -18,6 +18,9 @@ "icinga.main.message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", "icinga.main.severity": "warning", "input.type": "log", + "log.status": [ + "multiline" + ], "offset": 133, "prospector.type": "log" }, diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 9ee8cba16172..5006ccccda69 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -76,6 +76,9 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.status": [ + "multiline" + ], "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "offset": 445, "postgresql.log.database": "postgres", @@ -93,6 +96,9 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.status": [ + "multiline" + ], "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "offset": 873, "postgresql.log.database": "postgres", @@ -110,6 +116,9 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.status": [ + "multiline" + ], "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "offset": 1300, "postgresql.log.database": "postgres", @@ -159,6 +168,9 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.status": [ + "multiline" + ], "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "offset": 1907, "postgresql.log.database": "postgres", @@ -210,6 +222,9 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.status": [ + "multiline" + ], "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "offset": 2847, "postgresql.log.database": "clients", diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index 8bf8a368e5cb..18d11d11e516 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -4,6 +4,9 @@ "fileset.module": "system", "fileset.name": "syslog", "input.type": "log", + "log.status": [ + "multiline" + ], "offset": 0, "prospector.type": "log", "system.syslog.hostname": "a-mac-with-esc-key", From 6e822b3896384264fe026d3a090889028f2985a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 17 Aug 2018 13:33:09 +0200 Subject: [PATCH 05/13] add `log.multiline` if message consists of multiple lines Closes #957 --- filebeat/reader/multiline/multiline.go | 4 ++++ filebeat/tests/system/test_multiline.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/filebeat/reader/multiline/multiline.go b/filebeat/reader/multiline/multiline.go index 2e974c19b235..b7a1656f99da 100644 --- a/filebeat/reader/multiline/multiline.go +++ b/filebeat/reader/multiline/multiline.go @@ -273,6 +273,10 @@ func (mlr *Reader) finalize() reader.Message { mlr.message.AddFlagsWithKey("log.flags", "truncated") } + if mlr.numLines > 1 { + mlr.message.AddTagsWithKey("log.status", []string{"multiline"}) + } + // Copy message from existing content msg := mlr.message diff --git a/filebeat/tests/system/test_multiline.py b/filebeat/tests/system/test_multiline.py index bc219d07079d..f31408a5b9f3 100644 --- a/filebeat/tests/system/test_multiline.py +++ b/filebeat/tests/system/test_multiline.py @@ -150,6 +150,10 @@ def test_max_lines(self): assert False == self.log_contains( "InternalClusterService.java:388", "output/filebeat") + # Check if multiline counters show up + assert True == self.log_contains( + "\"log\":{\"multiline\":3}", "output/filebeat") + # Check that output file has the same number of lines as the log file assert 20 == len(output) From d0998bbbd35b73139fd928b2d1d4230b6c20a4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 17 Aug 2018 13:42:10 +0200 Subject: [PATCH 06/13] add changelog entry --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1496d523f522..f46b0dc97b6a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Make docker input check if container strings are empty {pull}7960[7960] - Add tag "truncated" to "log.flags" if incoming line is longer than configured limit. {pull}7991[7991] - Add tag "truncated" to "log.status" if incoming line is longer than configured limit. {pull}7991[7991] +- Add `log.multiline` if event consists of multiple lines. {pull}7997[7997] *Heartbeat* From deb139e8e0cf8dcaad9fe2b814fb35e82614977a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 21 Aug 2018 14:56:31 +0200 Subject: [PATCH 07/13] remove outdated test --- filebeat/tests/system/test_json.py | 34 ------------------------------ 1 file changed, 34 deletions(-) diff --git a/filebeat/tests/system/test_json.py b/filebeat/tests/system/test_json.py index d98b6025aa6e..1d6b0bee8e19 100644 --- a/filebeat/tests/system/test_json.py +++ b/filebeat/tests/system/test_json.py @@ -64,40 +64,6 @@ def test_docker_logs_filtering(self): assert all(o["stream"] == "stdout" for o in output) assert all("windows" not in o["log"] for o in output) - def test_docker_logs_multiline(self): - """ - Should be able to do multiline on docker logs. - """ - self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*", - json=dict(message_key="log", keys_under_root=True), - multiline=True, - pattern="^\[log\]", - match="after", - negate="true" - ) - - os.mkdir(self.working_dir + "/log/") - self.copy_files(["logs/docker_multiline.log"], - target_dir="log") - - proc = self.start_beat() - self.wait_until( - lambda: self.output_has(lines=3), - max_timeout=10) - - proc.check_kill_and_wait() - - output = self.read_output() - assert len(output) == 3 - - assert all("time" in o for o in output) - assert all("log" in o for o in output) - assert all("message" not in o for o in output) - assert all(o["stream"] == "stdout" for o in output) - assert output[1]["log"] == \ - "[log] This one is\n on multiple\n lines" - def test_simple_json_overwrite(self): """ Should be able to overwrite keys when requested. From f97e03f5e8926b4fa2613b7767f6e8114b232763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 Aug 2018 20:47:32 +0200 Subject: [PATCH 08/13] rm unnecessary test --- filebeat/tests/system/test_multiline.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/filebeat/tests/system/test_multiline.py b/filebeat/tests/system/test_multiline.py index f31408a5b9f3..bc219d07079d 100644 --- a/filebeat/tests/system/test_multiline.py +++ b/filebeat/tests/system/test_multiline.py @@ -150,10 +150,6 @@ def test_max_lines(self): assert False == self.log_contains( "InternalClusterService.java:388", "output/filebeat") - # Check if multiline counters show up - assert True == self.log_contains( - "\"log\":{\"multiline\":3}", "output/filebeat") - # Check that output file has the same number of lines as the log file assert 20 == len(output) From 629e78da8c83f9ddf581fc87adf32cde09635e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 Aug 2018 20:48:31 +0200 Subject: [PATCH 09/13] follow up changelog title --- CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f46b0dc97b6a..bf4af1f71ef1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -106,7 +106,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Make docker input check if container strings are empty {pull}7960[7960] - Add tag "truncated" to "log.flags" if incoming line is longer than configured limit. {pull}7991[7991] - Add tag "truncated" to "log.status" if incoming line is longer than configured limit. {pull}7991[7991] -- Add `log.multiline` if event consists of multiple lines. {pull}7997[7997] +- Add tag "multiline" to "log.status" if event consists of multiple lines. {pull}7997[7997] *Heartbeat* From 6391c2095ff9db7aecee0cd1fed80767e585584e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 30 Aug 2018 10:36:28 +0200 Subject: [PATCH 10/13] follow up renaming --- filebeat/reader/multiline/multiline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/reader/multiline/multiline.go b/filebeat/reader/multiline/multiline.go index b7a1656f99da..ae31c399ecb9 100644 --- a/filebeat/reader/multiline/multiline.go +++ b/filebeat/reader/multiline/multiline.go @@ -274,7 +274,7 @@ func (mlr *Reader) finalize() reader.Message { } if mlr.numLines > 1 { - mlr.message.AddTagsWithKey("log.status", []string{"multiline"}) + mlr.message.AddFlagsWithKey("log.flags", "multiline") } // Copy message from existing content From ef90468eadccc281a824eac4ad90e4235fd16c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 30 Aug 2018 10:36:43 +0200 Subject: [PATCH 11/13] add test to check both flags --- filebeat/reader/multiline/multiline_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/filebeat/reader/multiline/multiline_test.go b/filebeat/reader/multiline/multiline_test.go index 1c6ea96c6f38..96dfd6922ccc 100644 --- a/filebeat/reader/multiline/multiline_test.go +++ b/filebeat/reader/multiline/multiline_test.go @@ -232,6 +232,7 @@ func testMultilineTruncated(t *testing.T, cfg Config, events int, truncated bool for _, message := range messages { found := false + multiline := false statusFlags, err := message.Fields.GetValue("log.flags") if err != nil { if !truncated { @@ -247,6 +248,9 @@ func testMultilineTruncated(t *testing.T, cfg Config, events int, truncated bool if f == "truncated" { found = true } + if f == "multiline" { + multiline = true + } } default: t.Fatalf("incorrect type for log.flags") @@ -257,6 +261,7 @@ func testMultilineTruncated(t *testing.T, cfg Config, events int, truncated bool } else { assert.False(t, found) } + assert.True(t, multiline) } } From 841e72b37240f6cca4f65d3d39b05fff5cd8d5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 30 Aug 2018 12:32:40 +0200 Subject: [PATCH 12/13] follow up renaming in tests and CHANGELOG --- CHANGELOG.asciidoc | 3 +-- .../server/test/test.log-expected.json | 20 +++++++++---------- .../slowlog/test/test.log-expected.json | 4 ++-- .../icinga/main/test/test.log-expected.json | 2 +- ...-9.6-debian-with-slowlog.log-expected.json | 10 +++++----- .../darwin-syslog-sample.log-expected.json | 2 +- 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index bf4af1f71ef1..990d09ff645f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -105,8 +105,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Keep raw user agent information after parsing as user_agent_raw in Filebeat modules. {pull}7823[7832] - Make docker input check if container strings are empty {pull}7960[7960] - Add tag "truncated" to "log.flags" if incoming line is longer than configured limit. {pull}7991[7991] -- Add tag "truncated" to "log.status" if incoming line is longer than configured limit. {pull}7991[7991] -- Add tag "multiline" to "log.status" if event consists of multiple lines. {pull}7997[7997] +- Add tag "multiline" to "log.flags" if event consists of multiple lines. {pull}7997[7997] *Heartbeat* diff --git a/filebeat/module/elasticsearch/server/test/test.log-expected.json b/filebeat/module/elasticsearch/server/test/test.log-expected.json index 8ddf0e51bd81..f53a28cf9fb4 100644 --- a/filebeat/module/elasticsearch/server/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/test.log-expected.json @@ -167,10 +167,10 @@ "fileset.module": "elasticsearch", "fileset.name": "server", "input.type": "log", - "log.level": "WARN", - "log.status": [ + "log.flags": [ "multiline" ], + "log.level": "WARN", "message": "master left (reason = failed to ping, tried [3] times, each with maximum [30s] timeout), current nodes: nodes:\n {srvmulpvlsk252_md}{uc5xdiQgRhaBIY-sszgjvQ}{X9pC0t1UQQix_NNOM0J6JQ}{srvmulpvlsk252.loganalytics.santanderuk.corp}{180.39.9.93:9300}{ml.max_open_jobs=10, ml.enabled=true}, local\n {srvmulpvlsk258_md}{HgW6EDn5QCmWVmICy4saHw}{o8zku7OJR4CTp0IjY8Ag4Q}{srvmulpvlsk258.loganalytics.santanderuk.corp}{180.39.9.99:9300}{ml.max_open_jobs=10, ml.enabled=true}\n {srvmulpvlsk250_md}{igrwSoPGSJ6u_5b8k26tgQ}{PuRqciBFRbiQvL2_lS7LrQ}{srvmulpvlsk250.loganalytics.santanderuk.corp}{180.39.9.91:9300}{ml.max_open_jobs=10, ml.enabled=true}, master\n {srvmulpvlsk254_id}{wZYeAh2URc2NwBIHZolLWQ}{3nduupo-TzSPaXjQaNu4Sg}{srvmulpvlsk254.loganalytics.santanderuk.corp}{180.39.9.95:9300}{ml.max_open_jobs=10, ml.enabled=true}", "offset": 2008, "prospector.type": "log", @@ -182,10 +182,10 @@ "fileset.module": "elasticsearch", "fileset.name": "server", "input.type": "log", - "log.level": "WARN", - "log.status": [ + "log.flags": [ "multiline" ], + "log.level": "WARN", "message": "path: /_xpack/monitoring/_bulk, params: {system_id=logstash, system_api_version=2, interval=1s}\norg.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/2/no master];\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:151) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:57) ~[?:?]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:40) ~[?:?]\n at org.elasticsearch.action.support.TransportAction.doExecute(TransportAction.java:146) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:133) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$authorizeRequest$4(SecurityActionFilter.java:208) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.maybeRun(AuthorizationUtils.java:127) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.setRunAsRoles(AuthorizationUtils.java:121) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.authorize(AuthorizationUtils.java:109) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:186) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:212) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$4(AuthenticationService.java:246) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:257) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:159) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:122) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:185) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:145) ~[?:?]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:168) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:408) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction.lambda$doPrepareRequest$0(RestMonitoringBulkAction.java:77) ~[?:?]\n at org.elasticsearch.rest.BaseRestHandler.handleReques", "offset": 2907, "prospector.type": "log", @@ -197,10 +197,10 @@ "fileset.module": "elasticsearch", "fileset.name": "server", "input.type": "log", - "log.level": "WARN", - "log.status": [ + "log.flags": [ "multiline" ], + "log.level": "WARN", "message": "path: /_xpack/license, params: {}\norg.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$4.onTimeout(TransportMasterNodeAction.java:209) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:311) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:139) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:111) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:194) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:107) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.handleException(TransportMasterNodeAction.java:183) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$Adapter.lambda$onConnectionClosed$6(TransportService.java:893) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) [elasticsearch-5.6.3.jar:5.6.3]\n at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]\n at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\nCaused by: org.elasticsearch.transport.NodeDisconnectedException: [srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected", "offset": 7412, "prospector.type": "log", @@ -215,10 +215,10 @@ "fileset.module": "elasticsearch", "fileset.name": "server", "input.type": "log", - "log.level": "WARN", - "log.status": [ + "log.flags": [ "multiline" ], + "log.level": "WARN", "message": "duration [3.8s], collections [1]/[4.3s], total [3.8s]/[8.8h], memory [16.5gb]->[15.7gb]/[30.8gb], all_po\nols {[young] [1.2gb]->[24mb]/[1.4gb]}{[survivor] [191.3mb]->[191.3mb]/[191.3mb]}{[old] [15.1gb]->[15.5gb]/[29.1gb]}", "offset": 9873, "prospector.type": "log", @@ -258,10 +258,10 @@ "fileset.module": "elasticsearch", "fileset.name": "server", "input.type": "log", - "log.level": "WARN", - "log.status": [ + "log.flags": [ "multiline" ], + "log.level": "WARN", "message": "monitoring execution failed\norg.elasticsearch.xpack.monitoring.exporter.ExportException: Exception when closing export bulk\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1$1.(ExportBulk.java:106) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1.onFailure(ExportBulk.java:104) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:217) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:211) ~[?:?]\n at org.elasticsearch.xpack.common.IteratingActionListener.onResponse(IteratingActionListener.java:108) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) [elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk$1.onSuccess(HttpExportBulk.java:115) [x-pack-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$FailureTrackingResponseListener.onSuccess(RestClient.java:597) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:352) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:343) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119) [httpcore-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:326) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) [httpcore-nio-4.4.5.jar:4.4.5]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\n", "offset": 10648, "prospector.type": "log", diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index fc7cc067e9fc..c2a6ba286342 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -127,10 +127,10 @@ "fileset.module": "elasticsearch", "fileset.name": "slowlog", "input.type": "log", - "log.level": "INFO", - "log.status": [ + "log.flags": [ "multiline" ], + "log.level": "INFO", "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }]", "offset": 4753, "prospector.type": "log", diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index 11322e4fbaa2..59d4822ce5d8 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -18,7 +18,7 @@ "icinga.main.message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", "icinga.main.severity": "warning", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "offset": 133, diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 5006ccccda69..b17481ca76b3 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -76,7 +76,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", @@ -96,7 +96,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", @@ -116,7 +116,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", @@ -168,7 +168,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", @@ -222,7 +222,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index 18d11d11e516..4d667d28a17d 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -4,7 +4,7 @@ "fileset.module": "system", "fileset.name": "syslog", "input.type": "log", - "log.status": [ + "log.flags": [ "multiline" ], "offset": 0, From 1df545512581f4892bd797934d9ac1a19c4591cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 30 Aug 2018 16:23:15 +0200 Subject: [PATCH 13/13] fix after rebase --- filebeat/reader/message.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/filebeat/reader/message.go b/filebeat/reader/message.go index 0ce2d2b37498..a92bc9092ccf 100644 --- a/filebeat/reader/message.go +++ b/filebeat/reader/message.go @@ -74,15 +74,3 @@ func (m *Message) AddFlagsWithKey(key string, flags ...string) error { return common.AddTagsWithKey(m.Fields, key, flags) } - -func (msg *Message) AddTagsWithKey(key string, tags []string) error { - if len(tags) == 0 { - return nil - } - - if m.Fields == nil { - m.Fields = common.MapStr{} - } - - return common.AddTagsWithKey(m.Fields, key, tags) -}