Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Maps] Track geo_shape agg usage #71759

Merged
merged 12 commits into from
Jul 15, 2020
Merged

[Maps] Track geo_shape agg usage #71759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
b33f4b3
[Maps] get isClustered from count request instead of source data request
nreese Jul 13, 2020
028db6a
better naming
nreese Jul 13, 2020
27d053d
Merge branch 'count_data_request' of https://github.com/nreese/kibana
thomasneirynck Jul 13, 2020
539cfa9
tslint
nreese Jul 13, 2020
10cc5ed
Merge branch 'count_data_request' of https://github.com/nreese/kibana
thomasneirynck Jul 14, 2020
b350bc2
Merge branch 'master' of github.com:elastic/kibana
thomasneirynck Jul 14, 2020
e0cdd1c
make ts-typing explicit
thomasneirynck Jul 14, 2020
7660cc6
Merge branch 'master' of github.com:elastic/kibana into maps/add_gold…
thomasneirynck Jul 14, 2020
bd6d07b
Merge branch 'master' of github.com:elastic/kibana into maps/add_gold…
thomasneirynck Jul 14, 2020
2020a38
add unit tests
thomasneirynck Jul 14, 2020
9c0f7ae
Merge branch 'master' into maps/add_goldp_telemetry
thomasneirynck Jul 15, 2020
29d7fad
feedback
thomasneirynck Jul 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions x-pack/plugins/maps/server/maps_telemetry/maps_telemetry.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ describe('buildMapsTelemetry', () => {
indexPatternsWithGeoFieldCount: 0,
indexPatternsWithGeoPointFieldCount: 0,
indexPatternsWithGeoShapeFieldCount: 0,
geoShapeAggLayersCount: 0,
attributesPerMap: {
dataSourcesCount: {
avg: 0,
Expand Down Expand Up @@ -50,48 +51,49 @@ describe('buildMapsTelemetry', () => {
indexPatternsWithGeoFieldCount: 3,
indexPatternsWithGeoPointFieldCount: 2,
indexPatternsWithGeoShapeFieldCount: 1,
geoShapeAggLayersCount: 2,
attributesPerMap: {
dataSourcesCount: {
avg: 2.6666666666666665,
avg: 2,
max: 3,
min: 2,
min: 1,
},
emsVectorLayersCount: {
canada_provinces: {
avg: 0.3333333333333333,
avg: 0.2,
max: 1,
min: 1,
},
france_departments: {
avg: 0.3333333333333333,
avg: 0.2,
max: 1,
min: 1,
},
italy_provinces: {
avg: 0.3333333333333333,
avg: 0.2,
max: 1,
min: 1,
},
},
layerTypesCount: {
TILE: {
avg: 1,
avg: 0.6,
max: 1,
min: 1,
},
VECTOR: {
avg: 1.6666666666666667,
avg: 1.2,
max: 2,
min: 1,
},
},
layersCount: {
avg: 2.6666666666666665,
avg: 2,
max: 3,
min: 2,
min: 1,
},
},
mapsTotalCount: 3,
mapsTotalCount: 5,
settings: {
showMapVisualizationTypes: false,
},
Expand Down
150 changes: 130 additions & 20 deletions x-pack/plugins/maps/server/maps_telemetry/maps_telemetry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,25 @@

import _ from 'lodash';
import {
SavedObjectsClientContract,
SavedObjectAttributes,
SavedObjectAttribute,
SavedObjectAttributes,
SavedObjectsClientContract,
} from 'kibana/server';
import { IFieldType, IIndexPattern } from 'src/plugins/data/public';
import { SOURCE_TYPES, ES_GEO_FIELD_TYPE, MAP_SAVED_OBJECT_TYPE } from '../../common/constants';
import { LayerDescriptor } from '../../common/descriptor_types';
import {
ES_GEO_FIELD_TYPE,
LAYER_TYPE,
MAP_SAVED_OBJECT_TYPE,
SCALING_TYPES,
SOURCE_TYPES,
} from '../../common/constants';
import {
AbstractSourceDescriptor,
ESGeoGridSourceDescriptor,
ESSearchSourceDescriptor,
LayerDescriptor,
SourceDescriptor,
} from '../../common/descriptor_types';
import { MapSavedObject } from '../../common/map_saved_object_type';
// @ts-ignore
import { getInternalRepository } from '../kibana_server_services';
Expand Down Expand Up @@ -82,6 +94,111 @@ function getIndexPatternsWithGeoFieldCount(indexPatterns: IIndexPattern[]) {
};
}

function getEMSLayerCount(layerLists: LayerDescriptor[][]): ILayerTypeCount[] {
return layerLists.map((layerList: LayerDescriptor[]) => {
const emsLayers = layerList.filter((layer: LayerDescriptor) => {
return (
layer.sourceDescriptor !== null &&
layer.sourceDescriptor.type === SOURCE_TYPES.EMS_FILE &&
(layer.sourceDescriptor as AbstractSourceDescriptor).id
);
});
const emsCountsById = _(emsLayers).countBy((layer: LayerDescriptor) => {
return (layer.sourceDescriptor as AbstractSourceDescriptor).id;
});

const layerTypeCount = emsCountsById.value();
return layerTypeCount as ILayerTypeCount;
}) as ILayerTypeCount[];
}

function isFieldGeoShape(
indexPatterns: IIndexPattern[],
indexPatternId: string,
geoField: string | undefined
): boolean {
if (!geoField) {
return false;
}

const matchIndexPattern = indexPatterns.find((indexPattern: IIndexPattern) => {
return indexPattern.id === indexPatternId;
});

if (!matchIndexPattern) {
return false;
}

const fieldList: IFieldType[] =
matchIndexPattern.attributes && matchIndexPattern.attributes.fields
? JSON.parse(matchIndexPattern.attributes.fields)
: [];

const matchField = fieldList.find((field: IFieldType) => {
return field.name === geoField;
});

return !!matchField && matchField.type === ES_GEO_FIELD_TYPE.GEO_SHAPE;
}

function isGeoShapeAggLayer(indexPatterns: IIndexPattern[], layer: LayerDescriptor): boolean {
if (layer.sourceDescriptor === null) {
return false;
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably good to return early if no geoField here instead of on lines 119-121. You could revise this to:

const sourceDescriptor: SourceDescriptor = layer.sourceDescriptor;
if (!sourceDescriptor || !sourceDescriptor.geoField) {
    return false;
}

Copy link
Contributor Author

@thomasneirynck thomasneirynck Jul 15, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer to avoid it, since we then lose typing-clarity and end up duck typing (which now we have TS there is no reason to. The TS-typecheck will fail with this pattern too). geoField is very specific to ES-sources, and that type-check appears higher-up.


if (
layer.type !== LAYER_TYPE.VECTOR &&
layer.type !== LAYER_TYPE.BLENDED_VECTOR &&
layer.type !== LAYER_TYPE.HEATMAP
) {
return false;
}

const sourceDescriptor: SourceDescriptor = layer.sourceDescriptor;
if (sourceDescriptor.type === SOURCE_TYPES.ES_GEO_GRID) {
return isFieldGeoShape(
indexPatterns,
(sourceDescriptor as ESGeoGridSourceDescriptor).indexPatternId,
(sourceDescriptor as ESGeoGridSourceDescriptor).geoField
);
} else if (
sourceDescriptor.type === SOURCE_TYPES.ES_SEARCH &&
(sourceDescriptor as ESSearchSourceDescriptor).scalingType === SCALING_TYPES.CLUSTERS
) {
return isFieldGeoShape(
indexPatterns,
(sourceDescriptor as ESSearchSourceDescriptor).indexPatternId,
(sourceDescriptor as ESSearchSourceDescriptor).geoField
);
} else {
return false;
}
}

function getGeoShapeAggCount(
layerLists: LayerDescriptor[][],
indexPatterns: IIndexPattern[]
): number {
const countsPerMap: number[] = layerLists.map((layerList: LayerDescriptor[]) => {
const geoShapeAggLayers = layerList.filter((layerDescriptor) => {
return isGeoShapeAggLayer(indexPatterns, layerDescriptor);
});
return geoShapeAggLayers.length;
});

return _.sum(countsPerMap);
}

export function getLayerLists(mapSavedObjects: MapSavedObject[]): LayerDescriptor[][] {
return mapSavedObjects.map((savedMapObject) => {
const layerList =
savedMapObject.attributes && savedMapObject.attributes.layerListJSON
? JSON.parse(savedMapObject.attributes.layerListJSON)
: [];
return layerList as LayerDescriptor[];
});
}

export function buildMapsTelemetry({
mapSavedObjects,
indexPatternSavedObjects,
Expand All @@ -91,33 +208,21 @@ export function buildMapsTelemetry({
indexPatternSavedObjects: IIndexPattern[];
settings: SavedObjectAttribute;
}): SavedObjectAttributes {
const layerLists = mapSavedObjects.map((savedMapObject) =>
savedMapObject.attributes && savedMapObject.attributes.layerListJSON
? JSON.parse(savedMapObject.attributes.layerListJSON)
: []
);
const layerLists: LayerDescriptor[][] = getLayerLists(mapSavedObjects);
const mapsCount = layerLists.length;

const dataSourcesCount = layerLists.map((lList) => {
const dataSourcesCount = layerLists.map((layerList: LayerDescriptor[]) => {
// todo: not every source-descriptor has an id
// @ts-ignore
const sourceIdList = lList.map((layer: LayerDescriptor) => layer.sourceDescriptor.id);
const sourceIdList = layerList.map((layer: LayerDescriptor) => layer.sourceDescriptor.id);
return _.uniq(sourceIdList).length;
});

const layersCount = layerLists.map((lList) => lList.length);
const layerTypesCount = layerLists.map((lList) => _.countBy(lList, 'type'));

// Count of EMS Vector layers used
const emsLayersCount = layerLists.map((lList) =>
_(lList)
.countBy((layer: LayerDescriptor) => {
const isEmsFile = _.get(layer, 'sourceDescriptor.type') === SOURCE_TYPES.EMS_FILE;
return isEmsFile && _.get(layer, 'sourceDescriptor.id');
})
.pickBy((val, key) => key !== 'false')
.value()
) as ILayerTypeCount[];
const emsLayersCount = getEMSLayerCount(layerLists);

const dataSourcesCountSum = _.sum(dataSourcesCount);
const layersCountSum = _.sum(layersCount);
Expand All @@ -127,11 +232,16 @@ export function buildMapsTelemetry({
indexPatternsWithGeoPointFieldCount,
indexPatternsWithGeoShapeFieldCount,
} = getIndexPatternsWithGeoFieldCount(indexPatternSavedObjects);

// Tracks whether user users Gold+ only functionality
const geoShapeAggLayersCount = getGeoShapeAggCount(layerLists, indexPatternSavedObjects);

return {
settings,
indexPatternsWithGeoFieldCount,
indexPatternsWithGeoPointFieldCount,
indexPatternsWithGeoShapeFieldCount,
geoShapeAggLayersCount,
// Total count of maps
mapsTotalCount: mapsCount,
// Time of capture
Expand Down
33 changes: 33 additions & 0 deletions x-pack/plugins/maps/server/maps_telemetry/test_resources/sample_map_saved_objects.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,38 @@
],
"updated_at": "2019-01-31T23:19:55.855Z",
"version": 1
},
{
"type": "gis-map",
"id": "643da1e6-c628-11ea-87d0-0242ac130003",
"attributes": {
"title": "Single cluster layer with geo_shape field",
"description": "",
"mapStateJSON": "{\"zoom\":2.12,\"center\":{\"lon\":-88.67592,\"lat\":34.23257},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\",\"mode\":\"quick\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"lucene\"}}",
"layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"51afb7d0-c628-11ea-87d0-0242ac130003\",\"geoField\":\"geometry\",\"metrics\":[{\"type\":\"count\"}],\"requestType\":\"point\",\"resolution\":\"COARSE\",\"indexPatternId\":\"4a7f6010-0aed-11ea-9dd2-95afd7ad44d4\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blues\",\"colorCategory\":\"palette_0\",\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"ORDINAL\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":0}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":7,\"maxSize\":32,\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"}}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"8d384d5d-6353-468f-b8f8-8eaa487358c4\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"type\":\"VECTOR\",\"joins\":[]}]",
"uiStateJSON": "{}"
},
"references": [
],
"updated_at": "2019-01-31T23:19:55.855Z",
"version": 1
},
{
"type": "gis-map",
"id": "5efd136a-c628-11ea-87d0-0242ac130003",
"attributes": {
"title": "Single heatmap layer with geo_shape field",
"description": "",
"mapStateJSON": "{\"zoom\":2.12,\"center\":{\"lon\":-88.67592,\"lat\":34.23257},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\",\"mode\":\"quick\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"lucene\"}}",
"layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"51afb7d0-c628-11ea-87d0-0242ac130003\",\"geoField\":\"geometry\",\"metrics\":[{\"type\":\"count\"}],\"requestType\":\"heatmap\",\"resolution\":\"COARSE\",\"indexPatternId\":\"4a7f6010-0aed-11ea-9dd2-95afd7ad44d4\"},\"id\":\"52eade74-1c78-4e18-8670-2061f38b613b\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"HEATMAP\",\"colorRampName\":\"theclassic\"},\"type\":\"HEATMAP\",\"joins\":[]}]",
"uiStateJSON": "{}"
},
"references": [
],
"updated_at": "2019-01-31T23:19:55.855Z",
"version": 1
}

]