[Maps] use style metadata to calculate symbolization bands (#51713) (#…

…52401) * [Maps] use style metadata to calculate symbolization bands * only update style meta when fields change * load join source style meta * use style meta data request to populate range * apply source filter to style meta request * fix heatmap * only use style meta range if field supports field meta * add fieldMetaOptions to style prperty descriptor and add migration script * add UI for setting fieldMetaOptions.isEnabled * clean up * review feedback * fix can_skip_fetch tests * review feedback * only show field meta popover for fields that support field meta * avoid duplicate fields re-fetching style meta * clean up problems when first creating grid source * update text for enabling field meta toggle * provide UI for setting sigma * allow users to include global time in style meta request * update SIEM saved objects * add less than and greater than symbols when styling by field stats * fix functional tests * review feedback * add support for date fields * review feedback * only show less then and greater then in legend when values will be outside of std range * unnest VectorStyle._getFieldRange * remove unused function * only show style isTimeAware switch when style fields use field meta
elastic · Dec 6, 2019 · 159a3a1 · 159a3a1
1 parent 1059283
commit 159a3a1
Show file tree

Hide file tree

Showing 39 changed files with 910 additions and 119 deletions.
diff --git a/x-pack/legacy/plugins/maps/common/constants.js b/x-pack/legacy/plugins/maps/common/constants.js
@@ -57,6 +57,8 @@ export const FIELD_ORIGIN = {
 };
 
 export const SOURCE_DATA_ID_ORIGIN = 'source';
+export const META_ID_ORIGIN_SUFFIX = 'meta';
+export const SOURCE_META_ID_ORIGIN = `${SOURCE_DATA_ID_ORIGIN}_${META_ID_ORIGIN_SUFFIX}`;
 
 export const GEOJSON_FILE = 'GEOJSON_FILE';
 
@@ -124,6 +126,11 @@ export const COUNT_PROP_LABEL =  i18n.translate('xpack.maps.aggs.defaultCountLab
 export const COUNT_PROP_NAME = 'doc_count';
 
 export const STYLE_TYPE = {
-  'STATIC': 'STATIC',
-  'DYNAMIC': 'DYNAMIC'
+  STATIC: 'STATIC',
+  DYNAMIC: 'DYNAMIC'
+};
+
+export const LAYER_STYLE_TYPE = {
+  VECTOR: 'VECTOR',
+  HEATMAP: 'HEATMAP'
 };
diff --git a/x-pack/legacy/plugins/maps/common/migrations/add_field_meta_options.js b/x-pack/legacy/plugins/maps/common/migrations/add_field_meta_options.js
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import {  LAYER_TYPE, STYLE_TYPE } from '../constants';
+
+function isVectorLayer(layerDescriptor) {
+  const layerType = _.get(layerDescriptor, 'type');
+  return layerType === LAYER_TYPE.VECTOR;
+}
+
+export function addFieldMetaOptions({ attributes }) {
+  if (!attributes.layerListJSON) {
+    return attributes;
+  }
+
+  const layerList = JSON.parse(attributes.layerListJSON);
+  layerList.forEach((layerDescriptor) => {
+    if (isVectorLayer(layerDescriptor) && _.has(layerDescriptor, 'style.properties')) {
+      Object.values(layerDescriptor.style.properties).forEach(stylePropertyDescriptor => {
+        if (stylePropertyDescriptor.type === STYLE_TYPE.DYNAMIC) {
+          stylePropertyDescriptor.options.fieldMetaOptions = {
+            isEnabled: false, // turn off field metadata to avoid changing behavior of existing saved objects
+            sigma: 3,
+          };
+        }
+      });
+    }
+  });
+
+  return {
+    ...attributes,
+    layerListJSON: JSON.stringify(layerList),
+  };
+}
diff --git a/x-pack/legacy/plugins/maps/common/migrations/add_field_meta_options.test.js b/x-pack/legacy/plugins/maps/common/migrations/add_field_meta_options.test.js
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { addFieldMetaOptions } from './add_field_meta_options';
+import {  LAYER_TYPE, STYLE_TYPE } from '../constants';
+
+describe('addFieldMetaOptions', () => {
+
+  test('Should handle missing layerListJSON attribute', () => {
+    const attributes = {
+      title: 'my map',
+    };
+    expect(addFieldMetaOptions({ attributes })).toEqual({
+      title: 'my map',
+    });
+  });
+
+  test('Should ignore non-vector layers', () => {
+    const layerListJSON = JSON.stringify([
+      {
+        type: LAYER_TYPE.HEATMAP,
+        style: {
+          type: 'HEATMAP',
+          colorRampName: 'Greens'
+        }
+      }
+    ]);
+    const attributes = {
+      title: 'my map',
+      layerListJSON
+    };
+    expect(addFieldMetaOptions({ attributes })).toEqual({
+      title: 'my map',
+      layerListJSON
+    });
+  });
+
+  test('Should ignore static style properties', () => {
+    const layerListJSON = JSON.stringify([
+      {
+        type: LAYER_TYPE.VECTOR,
+        style: {
+          type: 'VECTOR',
+          properties: {
+            lineColor: {
+              type: STYLE_TYPE.STATIC,
+              options: {
+                color: '#FFFFFF'
+              }
+            }
+          }
+        }
+      }
+    ]);
+    const attributes = {
+      title: 'my map',
+      layerListJSON
+    };
+    expect(addFieldMetaOptions({ attributes })).toEqual({
+      title: 'my map',
+      layerListJSON
+    });
+  });
+
+  test('Should add field meta options to dynamic style properties', () => {
+    const layerListJSON = JSON.stringify([
+      {
+        type: LAYER_TYPE.VECTOR,
+        style: {
+          type: 'VECTOR',
+          properties: {
+            fillColor: {
+              type: STYLE_TYPE.DYNAMIC,
+              options: {
+                field: {
+                  name: 'my_field',
+                  origin: 'source'
+                },
+                color: 'Greys'
+              }
+            }
+          }
+        }
+      }
+    ]);
+    const attributes = {
+      title: 'my map',
+      layerListJSON
+    };
+    expect(addFieldMetaOptions({ attributes })).toEqual({
+      title: 'my map',
+      layerListJSON: JSON.stringify([
+        {
+          type: LAYER_TYPE.VECTOR,
+          style: {
+            type: 'VECTOR',
+            properties: {
+              fillColor: {
+                type: STYLE_TYPE.DYNAMIC,
+                options: {
+                  field: {
+                    name: 'my_field',
+                    origin: 'source'
+                  },
+                  color: 'Greys',
+                  fieldMetaOptions: {
+                    isEnabled: false,
+                    sigma: 3,
+                  }
+                }
+              }
+            }
+          }
+        }
+      ])
+    });
+  });
+});
diff --git a/x-pack/legacy/plugins/maps/migrations.js b/x-pack/legacy/plugins/maps/migrations.js
@@ -8,6 +8,7 @@ import { extractReferences } from './common/migrations/references';
 import { emsRasterTileToEmsVectorTile } from './common/migrations/ems_raster_tile_to_ems_vector_tile';
 import { topHitsTimeToSort } from './common/migrations/top_hits_time_to_sort';
 import { moveApplyGlobalQueryToSources } from './common/migrations/move_apply_global_query';
+import { addFieldMetaOptions } from './common/migrations/add_field_meta_options';
 
 export const migrations = {
   'map': {
@@ -37,11 +38,12 @@ export const migrations = {
       };
     },
     '7.6.0': (doc) => {
-      const attributes = moveApplyGlobalQueryToSources(doc);
+      const attributesPhase1 = moveApplyGlobalQueryToSources(doc);
+      const attributesPhase2 = addFieldMetaOptions({ attributes: attributesPhase1 });
 
       return {
         ...doc,
-        attributes,
+        attributes: attributesPhase2,
       };
     }
   },

diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js
@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-
 import { AbstractField } from './field';
 import { COUNT_AGG_TYPE } from '../../../common/constants';
+import { isMetricCountable } from '../util/is_metric_countable';
 import { ESAggMetricTooltipProperty } from '../tooltips/es_aggmetric_tooltip_property';
 
 export class ESAggMetricField extends AbstractField {
@@ -36,6 +36,11 @@ export class ESAggMetricField extends AbstractField {
     return (this.getAggType() === COUNT_AGG_TYPE) ? true : !!this._esDocField;
   }
 
+  async getDataType() {
+    // aggregations only provide numerical data
+    return 'number';
+  }
+
   getESDocFieldName() {
     return this._esDocField ? this._esDocField.getName() : '';
   }
@@ -55,7 +60,6 @@ export class ESAggMetricField extends AbstractField {
     );
   }
 
-
   makeMetricAggConfig() {
     const metricAggConfig = {
       id: this.getName(),
@@ -69,4 +73,13 @@ export class ESAggMetricField extends AbstractField {
     }
     return metricAggConfig;
   }
+
+  supportsFieldMeta() {
+    // count and sum aggregations are not within field bounds so they do not support field meta.
+    return !isMetricCountable(this.getAggType());
+  }
+
+  async getFieldMetaRequest(config) {
+    return this._esDocField.getFieldMetaRequest(config);
+  }
 }
diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ESAggMetricField } from './es_agg_field';
+import { METRIC_TYPE } from '../../../common/constants';
+
+describe('supportsFieldMeta', () => {
+
+  test('Non-counting aggregations should support field meta', () => {
+    const avgMetric = new ESAggMetricField({ aggType: METRIC_TYPE.AVG });
+    expect(avgMetric.supportsFieldMeta()).toBe(true);
+    const maxMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MAX });
+    expect(maxMetric.supportsFieldMeta()).toBe(true);
+    const minMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MIN });
+    expect(minMetric.supportsFieldMeta()).toBe(true);
+  });
+
+  test('Counting aggregations should not support field meta', () => {
+    const countMetric = new ESAggMetricField({ aggType: METRIC_TYPE.COUNT });
+    expect(countMetric.supportsFieldMeta()).toBe(false);
+    const sumMetric = new ESAggMetricField({ aggType: METRIC_TYPE.SUM });
+    expect(sumMetric.supportsFieldMeta()).toBe(false);
+    const uniqueCountMetric = new ESAggMetricField({ aggType: METRIC_TYPE.UNIQUE_COUNT });
+    expect(uniqueCountMetric.supportsFieldMeta()).toBe(false);
+  });
+});
diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_doc_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_doc_field.js
@@ -27,4 +27,31 @@ export class ESDocField extends AbstractField {
     return field.type;
   }
 
+  supportsFieldMeta() {
+    return true;
+  }
+
+  async getFieldMetaRequest(/* config */) {
+    const field = await this._getField();
+
+    if (field.type !== 'number' && field.type !== 'date') {
+      return null;
+    }
+
+    const extendedStats = {};
+    if (field.scripted) {
+      extendedStats.script = {
+        source: field.script,
+        lang: field.lang
+      };
+    } else {
+      extendedStats.field = this._fieldName;
+    }
+    return {
+      [this._fieldName]: {
+        extended_stats: extendedStats
+      }
+    };
+  }
+
 }
diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/field.js b/x-pack/legacy/plugins/maps/public/layers/fields/field.js
@@ -42,4 +42,12 @@ export class AbstractField {
   getOrigin() {
     return this._origin;
   }
+
+  supportsFieldMeta() {
+    return false;
+  }
+
+  async getFieldMetaRequest(/* config */) {
+    return null;
+  }
 }
diff --git a/x-pack/legacy/plugins/maps/public/layers/joins/inner_join.js b/x-pack/legacy/plugins/maps/public/layers/joins/inner_join.js
@@ -7,6 +7,7 @@
 
 import { ESTermSource } from '../sources/es_term_source';
 import { getComputedFieldNamePrefix } from '../styles/vector/style_util';
+import { META_ID_ORIGIN_SUFFIX } from '../../../common/constants';
 
 export class InnerJoin {
 
@@ -36,10 +37,14 @@ export class InnerJoin {
   // Source request id must be static and unique because the re-fetch logic uses the id to locate the previous request.
   // Elasticsearch sources have a static and unique id so that requests can be modified in the inspector.
   // Using the right source id as the source request id because it meets the above criteria.
-  getSourceId() {
+  getSourceDataRequestId() {
     return `join_source_${this._rightSource.getId()}`;
   }
 
+  getSourceMetaDataRequestId() {
+    return `${this.getSourceDataRequestId()}_${META_ID_ORIGIN_SUFFIX}`;
+  }
+
   getLeftField() {
     return this._leftField;
   }

diff --git a/x-pack/legacy/plugins/maps/public/layers/layer.js b/x-pack/legacy/plugins/maps/public/layers/layer.js
@@ -80,7 +80,7 @@ export class AbstractLayer {
   }
 
   supportsElasticsearchFilters() {
-    return this._source.supportsElasticsearchFilters();
+    return this._source.isESSource();
   }
 
   async supportsFitToBounds() {