[EEM] Use top_metrics for identity fields lifting (#188315)

This PR converts the identity fields in the latest transform from a `terms` aggregation to a `top_metric` aggregation in an effort to simplify the implementation since we have to convert a terms aggregation from keys to an array of strings. With the `top_metrics` implementation, we just need to use the `dot_expander` processors along with a `set` processor to get a single (non-array) field. --------- Co-authored-by: Chris Cowan <[email protected]>
elastic · Jul 22, 2024 · dd126c4 · dd126c4
1 parent a68f812
commit dd126c4
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 31 deletions.
diff --git a/...server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap b/...server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap
diff --git a/...solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/...solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
@@ -38,16 +38,30 @@ function createMetadataPainlessScript(definition: EntityDefinition) {
 }
 
 function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) {
-  return definition.identityFields.map((identityField) => {
-    const optionalFieldPath = identityField.field.replaceAll('.', '?.');
-    const assignValue = `ctx.${identityField.field} = ctx.entity.identity.${identityField.field}.keySet().toArray()[0];`;
-    return {
-      script: {
-        if: `ctx.entity.identity.${optionalFieldPath} != null && ctx.entity.identity.${identityField.field}.size() != 0`,
-        source: cleanScript(`${initializePathScript(identityField.field)}\n${assignValue}`),
-      },
-    };
-  });
+  return definition.identityFields
+    .map((identityField) => {
+      const setProcessor = {
+        set: {
+          field: identityField.field,
+          value: `{{entity.identity.${identityField.field}.top_metric.${identityField.field}}}`,
+        },
+      };
+
+      if (!identityField.field.includes('.')) {
+        return [setProcessor];
+      }
+
+      return [
+        {
+          dot_expander: {
+            field: identityField.field,
+            path: `entity.identity.${identityField.field}.top_metric`,
+          },
+        },
+        setProcessor,
+      ];
+    })
+    .flat();
 }
 
 export function generateLatestProcessors(definition: EntityDefinition) {

diff --git a/...anager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap b/...anager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap
diff --git a/...y_solution/entity_manager/server/lib/entities/transform/generate_identity_aggregations.ts b/...y_solution/entity_manager/server/lib/entities/transform/generate_identity_aggregations.ts
@@ -12,9 +12,20 @@ export function generateIdentityAggregations(definition: EntityDefinition) {
     (aggs, identityField) => ({
       ...aggs,
       [`entity.identity.${identityField.field}`]: {
-        terms: {
-          field: identityField.field,
-          size: 1,
+        filter: {
+          exists: {
+            field: identityField.field,
+          },
+        },
+        aggs: {
+          top_metric: {
+            top_metrics: {
+              metrics: {
+                field: identityField.field,
+              },
+              sort: '_score',
+            },
+          },
         },
       },
     }),