[Packetbeat] Output ECS-compatible TLS fields (elastic#15497)

Packetbeat now outputs TLS fields from ECS 1.3+: - The additional information not covered by ECS is nested under tls.detailed. - Fields already in ECS are removed from detailed to avoid bloat. - A new configuration flag tls.include_detailed_fields allows to toggle the inclusion of extra fields. It's enabled by default. Caveats: - Originally it would output the top-level certificate in tls.server_certificate and the rest under tls.server_certificate_chain. ECS mandates that tls.server.certificate and tls.server.certificate_chain are mutually exclusive. To avoid confusion, a chain is always generated, even if it consists of a single certificate. - Same for tls.client certificates. - The behavior of the configuration options tls.send_certificates and tls.include_raw_certificates has slightly changed. Non-populated TLS ECS fields: - tls.curve: Not implemented. Requires parsing the server key exchange. - tls.server.ja3s: JA3s is not implemented yet.
adriansr · Jan 14, 2020 · fd5fd3d · fd5fd3d
1 parent dba8f74
commit fd5fd3d
Show file tree

Hide file tree

Showing 27 changed files with 1,857 additions and 855 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -87,6 +87,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594]
 - NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503]
 - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
+- TLS: Fields have been changed to adapt to ECS. {pull}15497[15497]
+- TLS: The behavior of send_certificates and include_raw_certificates options has changed. {pull}15497[15497]
 
 *Winlogbeat*
 

diff --git a/packetbeat/_meta/kibana/7/dashboard/Packetbeat-overview.json b/packetbeat/_meta/kibana/7/dashboard/Packetbeat-overview.json
@@ -1246,7 +1246,7 @@
               "id": "3",
               "params": {
                 "customLabel": "Handshake completed",
-                "field": "tls.handshake_completed",
+                "field": "tls.established",
                 "json": "",
                 "order": "desc",
                 "orderBy": "1",
@@ -1665,13 +1665,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.handshake_completed"
+                  "field": "tls.established"
                 },
                 "meta": {
                   "alias": "TLS sessions",
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.handshake_completed",
+                  "key": "tls.established",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"

diff --git a/packetbeat/_meta/kibana/7/dashboard/Packetbeat-tls.json b/packetbeat/_meta/kibana/7/dashboard/Packetbeat-tls.json
@@ -83,7 +83,7 @@
               "id": "3",
               "params": {
                 "customLabel": "Handshake completed",
-                "field": "tls.handshake_completed",
+                "field": "tls.established",
                 "json": "",
                 "order": "desc",
                 "orderBy": "1",
@@ -227,7 +227,7 @@
               "enabled": true,
               "id": "2",
               "params": {
-                "field": "tls.alert_types",
+                "field": "tls.detailed.alert_types",
                 "include": ".*",
                 "json": "{\"missing\": \"None\"}",
                 "order": "desc",
@@ -265,13 +265,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.handshake_completed"
+                  "field": "tls.established"
                 },
                 "meta": {
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.handshake_completed",
+                  "key": "tls.established",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -383,7 +383,7 @@
               "id": "2",
               "params": {
                 "customLabel": "Server Name Indication",
-                "field": "tls.client_hello.extensions.server_name_indication",
+                "field": "tls.client.server_name",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -438,7 +438,7 @@
               "id": "2",
               "params": {
                 "customLabel": "TLS version",
-                "field": "tls.version",
+                "field": "tls.detailed.version",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -493,7 +493,7 @@
               "id": "2",
               "params": {
                 "customLabel": "Public Key Size",
-                "field": "tls.server_certificate.public_key_size",
+                "field": "tls.detailed.server_certificate.public_key_size",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -548,7 +548,7 @@
               "id": "2",
               "params": {
                 "exclude": "",
-                "field": "tls.resumption_method",
+                "field": "tls.detailed.resumption_method",
                 "json": "{\n\"missing\": \"none\"\n}",
                 "missingBucket": false,
                 "missingBucketLabel": "Missing",
@@ -625,7 +625,7 @@
               "id": "4",
               "params": {
                 "customLabel": "Subject Common Name",
-                "field": "tls.server_certificate.subject.common_name",
+                "field": "tls.detailed.server_certificate.subject.common_name",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 10
@@ -638,7 +638,7 @@
               "id": "3",
               "params": {
                 "customLabel": "Organization",
-                "field": "tls.server_certificate.subject.organization",
+                "field": "tls.detailed.server_certificate.subject.organization",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -706,7 +706,7 @@
               "id": "2",
               "params": {
                 "customLabel": "Subject Common Name",
-                "field": "tls.client_certificate.subject.common_name",
+                "field": "tls.detailed.client_certificate.subject.common_name",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -719,7 +719,7 @@
               "id": "3",
               "params": {
                 "customLabel": "Signature Algorithm",
-                "field": "tls.client_certificate.signature_algorithm",
+                "field": "tls.detailed.client_certificate.signature_algorithm",
                 "json": "{ \"missing\": \"N/A\" }",
                 "order": "desc",
                 "orderBy": "1",
@@ -788,7 +788,7 @@
               "id": "2",
               "params": {
                 "customLabel": "Cipher",
-                "field": "tls.server_hello.selected_cipher",
+                "field": "tls.cipher",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -856,7 +856,7 @@
               "id": "2",
               "params": {
                 "customLabel": "JA3 Fingerprint",
-                "field": "tls.fingerprints.ja3.hash",
+                "field": "tls.client.ja3",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 5
@@ -1046,13 +1046,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.handshake_completed"
+                  "field": "tls.established"
                 },
                 "meta": {
                   "alias": "TLS sessions",
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.handshake_completed",
+                  "key": "tls.established",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -1095,13 +1095,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.client_hello.extensions.server_name_indication"
+                  "field": "tls.client.server_name"
                 },
                 "meta": {
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.client_hello.extensions.server_name_indication",
+                  "key": "tls.client.server_name",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -1144,13 +1144,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.version"
+                  "field": "tls.detailed.version"
                 },
                 "meta": {
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.version",
+                  "key": "tls.detailed.version",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -1193,13 +1193,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.server_certificate.public_key_size"
+                  "field": "tls.detailed.server_certificate.public_key_size"
                 },
                 "meta": {
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.server_certificate.public_key_size",
+                  "key": "tls.detailed.server_certificate.public_key_size",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -1242,13 +1242,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.fingerprints.ja3.hash"
+                  "field": "tls.client.ja3"
                 },
                 "meta": {
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.fingerprints.ja3.hash",
+                  "key": "tls.client.ja3",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -1291,13 +1291,13 @@
                   "store": "appState"
                 },
                 "exists": {
-                  "field": "tls.handshake_completed"
+                  "field": "tls.established"
                 },
                 "meta": {
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "tls.handshake_completed",
+                  "key": "tls.established",
                   "negate": false,
                   "type": "exists",
                   "value": "exists"
@@ -1602,4 +1602,4 @@
     }
   ],
   "version": "7.0.0-SNAPSHOT"
-}
+}