Set <event>.duration.us field in ingest pipeline (#7261)

* model: <event>.duration.us -> event.duration We'll set the <event>.duration.us fields in ingest. * apmpackage: translate event.duration in ingest Translate event.duration to <event>.duration.us for transaction and span events, and then drop event.duration. * processor/otel: don't copy event.* to span events * Update approvals * Update changelogs
elastic · Feb 13, 2022 · f599dbb · f599dbb
1 parent f227841
commit f599dbb
Show file tree

Hide file tree

Showing 51 changed files with 144 additions and 318 deletions.
diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml
@@ -3,9 +3,9 @@
 # change type can be one of: enhancement, bugfix, breaking-change
 - version: "8.2.0"
   changes:
-    - description: Placeholder
+    - description: updated traces and rum_traces ingest pipelines to translate `event.duration` to `<event>.duration.us`
       type: enhancement
-      link: https://github.com/elastic/apm-server/pull/123
+      link: https://github.com/elastic/apm-server/pull/7261
 - version: "8.1.0"
   changes:
     - description: Added field mapping for `faas.coldstart` and `faas.trigger.type`

diff --git a/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/rum_traces/elasticsearch/ingest_pipeline/default.yml
@@ -7,6 +7,8 @@ processors:
       name: user_agent
   - pipeline:
       name: client_geoip
+  - pipeline:
+      name: event_duration
   - remove:
       # Remove some metadata from spans that is available in the parent transaction, to cut down on storage costs.
       if: ctx.processor?.event == 'span'

diff --git a/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/default.yml
@@ -7,6 +7,8 @@ processors:
       name: user_agent
   - pipeline:
       name: client_geoip
+  - pipeline:
+      name: event_duration
   - remove:
       # Remove some metadata from spans that is available in the parent transaction, to cut down on storage costs.
       if: ctx.processor?.event == 'span'

diff --git a/apmpackage/cmd/genpackage/pipelines.go b/apmpackage/cmd/genpackage/pipelines.go
@@ -19,6 +19,7 @@ package main
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 )
@@ -38,6 +39,7 @@ func getCommonPipeline(name string, version *common.Version) []map[string]interf
 		"observer_version": getObserverVersionPipeline(version),
 		"user_agent":       userAgentPipeline,
 		"client_geoip":     clientGeoIPPipeline,
+		"event_duration":   eventDurationPipeline,
 	}
 	return commonPipelines[name]
 }
@@ -109,3 +111,22 @@ var clientGeoIPPipeline = []map[string]interface{}{{
 		}},
 	},
 }}
+
+// TODO(axw) remove this pipeline when we are ready to migrate the UI to
+// `event.duration`. See https://github.com/elastic/apm-server/issues/5999.
+var eventDurationPipeline = []map[string]interface{}{{
+	"script": map[string]interface{}{
+		"if": "ctx.processor?.event != null && ctx.get(ctx.processor.event) != null",
+		"source": strings.TrimSpace(`
+def durationNanos = ctx.event?.duration ?: 0;
+def eventType = ctx.processor.event;
+ctx.get(ctx.processor.event).duration = ["us": (int)(durationNanos/1000)];
+`),
+	},
+}, {
+	"remove": map[string]interface{}{
+		"field":          "event.duration",
+		"ignore_missing": true,
+		"ignore_failure": true,
+	},
+}}
diff --git a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json
@@ -311,6 +311,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 3781912,
                 "outcome": "success"
             },
             "host": {
@@ -413,9 +414,6 @@
                         "name": "postgres"
                     }
                 },
-                "duration": {
-                    "us": 3781
-                },
                 "id": "1234567890aaaade",
                 "name": "GET users-authenticated",
                 "stacktrace": [
@@ -480,6 +478,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 32592981,
                 "outcome": "success"
             },
             "host": {
@@ -628,9 +627,6 @@
                     "my_key": 1,
                     "some_other_value": "foobar"
                 },
-                "duration": {
-                    "us": 32592
-                },
                 "id": "4340a8e0df1906ecbfa9",
                 "name": "ResourceHttpRequestHandler",
                 "result": "HTTP2xx",

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json
@@ -142,6 +142,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 3564298,
                 "outcome": "unknown"
             },
             "host": {
@@ -166,9 +167,6 @@
                 "name": "1234_service-12a3"
             },
             "span": {
-                "duration": {
-                    "us": 3564
-                },
                 "id": "0123456a89012345",
                 "name": "GET /api/types",
                 "type": "request"
@@ -193,6 +191,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 3564298,
                 "outcome": "unknown"
             },
             "host": {
@@ -217,9 +216,6 @@
                 "name": "1234_service-12a3"
             },
             "span": {
-                "duration": {
-                    "us": 3564
-                },
                 "id": "0123456a89012345",
                 "name": "GET /api/types",
                 "type": "request"
@@ -244,6 +240,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 32592981,
                 "outcome": "unknown"
             },
             "host": {
@@ -274,9 +271,6 @@
                 "id": "01234567890123456789abcdefabcdef"
             },
             "transaction": {
-                "duration": {
-                    "us": 32592
-                },
                 "id": "abcdef1478523690",
                 "sampled": true,
                 "span_count": {

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json
@@ -44,6 +44,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 141581000,
                 "outcome": "success"
             },
             "host": {
@@ -115,9 +116,6 @@
             },
             "span": {
                 "action": "query.custom",
-                "duration": {
-                    "us": 141581
-                },
                 "id": "abcdef01234567",
                 "name": "GET /api/types",
                 "subtype": "postgresql",
@@ -178,6 +176,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 32592981,
                 "outcome": "unknown"
             },
             "host": {
@@ -248,9 +247,6 @@
                 "version": "5.1.3"
             },
             "span": {
-                "duration": {
-                    "us": 32592
-                },
                 "id": "1234abcdef567895",
                 "name": "GET /api/types",
                 "type": "request"
@@ -310,6 +306,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 3564298,
                 "outcome": "unknown"
             },
             "host": {
@@ -384,9 +381,6 @@
                 "version": "5.1.3"
             },
             "span": {
-                "duration": {
-                    "us": 3564
-                },
                 "id": "0123456a89012345",
                 "name": "GET /api/types",
                 "subtype": "http",
@@ -447,6 +441,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 13980298,
                 "outcome": "unknown"
             },
             "host": {
@@ -518,9 +513,6 @@
             },
             "span": {
                 "action": "call",
-                "duration": {
-                    "us": 13980
-                },
                 "id": "abcde56a89012345",
                 "name": "get /api/types",
                 "subtype": "http",
@@ -585,6 +577,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 3781912,
                 "outcome": "success"
             },
             "host": {
@@ -684,9 +677,6 @@
                         "type": "db"
                     }
                 },
-                "duration": {
-                    "us": 3781
-                },
                 "id": "1234567890aaaade",
                 "name": "SELECT FROM product_types",
                 "stacktrace": [
@@ -801,6 +791,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 141581000,
                 "outcome": "unknown"
             },
             "host": {
@@ -872,9 +863,6 @@
             },
             "span": {
                 "action": "receive",
-                "duration": {
-                    "us": 141581
-                },
                 "id": "00xxx12312312312",
                 "message": {
                     "age": {
@@ -944,6 +932,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 378191200,
                 "outcome": "success"
             },
             "host": {
@@ -1022,9 +1011,6 @@
                         "us": 359298
                     }
                 },
-                "duration": {
-                    "us": 378191
-                },
                 "id": "abcdef01234567",
                 "name": "SELECT FROM p_details",
                 "subtype": "postgresql",

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json
@@ -39,6 +39,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 32592981,
                 "outcome": "unknown"
             },
             "host": {
@@ -121,9 +122,6 @@
                 "id": "0123456789abcdef0123456789abcdef"
             },
             "transaction": {
-                "duration": {
-                    "us": 32592
-                },
                 "id": "945254c567a5417e",
                 "sampled": true,
                 "span_count": {
@@ -179,6 +177,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 32592981,
                 "outcome": "success"
             },
             "host": {
@@ -329,9 +328,6 @@
                     "my_key": 1,
                     "some_other_value": "foo bar"
                 },
-                "duration": {
-                    "us": 32592
-                },
                 "id": "4340a8e0df1906ecbfa9",
                 "name": "GET /api/types",
                 "result": "success",
@@ -400,6 +396,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 13980558,
                 "outcome": "unknown"
             },
             "host": {
@@ -484,9 +481,6 @@
                 "id": "0acd456789abcdef0123456789abcdef"
             },
             "transaction": {
-                "duration": {
-                    "us": 13980
-                },
                 "experience": {
                     "cls": 1,
                     "fid": 2,
@@ -560,6 +554,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 3000000,
                 "outcome": "unknown"
             },
             "host": {
@@ -646,9 +641,6 @@
                 "id": "0123456789abcdef0123456789abcdef"
             },
             "transaction": {
-                "duration": {
-                    "us": 3000
-                },
                 "id": "00xxxxFFaaaa1234",
                 "message": {
                     "age": {
@@ -727,6 +719,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 38853000,
                 "outcome": "unknown"
             },
             "faas": {
@@ -817,9 +810,6 @@
                 "id": "eb56529a1f461c5e7e2f66ecb075e983"
             },
             "transaction": {
-                "duration": {
-                    "us": 38853
-                },
                 "id": "142e61450efb8574",
                 "name": "july-2021-delete-after-july-31",
                 "result": "success",

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationTransactionsHugeTraces.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationTransactionsHugeTraces.approved.json
@@ -42,6 +42,7 @@
                 "version": "1.12.0"
             },
             "event": {
+                "duration": 32592981,
                 "outcome": "success"
             },
             "host": {
@@ -192,9 +193,6 @@
                     "my_key": 1,
                     "some_other_value": "foo bar"
                 },
-                "duration": {
-                    "us": 32592
-                },
                 "id": "ddf109a4c4aa5f2b6e984548ca57774c",
                 "name": "GET /api/types",
                 "result": "success",