diff --git a/src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js b/src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js index 1b7997831882c..fe7b0fa77970f 100644 --- a/src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js +++ b/src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js @@ -34,23 +34,6 @@ app.directive('discoverField', function ($compile) { const getWarnings = function (field) { let warnings = []; - if (!field.scripted) { - if (!field.doc_values && field.type !== 'boolean' && !(field.analyzed && field.type === 'string')) { - warnings.push('Doc values are not enabled on this field. This may lead to excess heap consumption when visualizing.'); - } - - if (field.analyzed && field.type === 'string') { - warnings.push('This is an analyzed string field.' + - ' Analyzed strings are highly unique and can use a lot of memory to visualize.' + - ' Values such as foo-bar will be broken into foo and bar.'); - } - - if (!field.indexed && !field.searchable) { - warnings.push('This field is not indexed and might not be usable in visualizations.'); - } - } - - if (field.scripted) { warnings.push('Scripted fields can take a long time to execute.'); } diff --git a/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html b/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html index 56e0d7c174146..75664dfcf3896 100644 --- a/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html +++ b/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html @@ -62,21 +62,21 @@
Available Fields
diff --git a/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js b/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js index bcda23fca9598..a22fe04717585 100644 --- a/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js +++ b/src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js @@ -45,13 +45,14 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou const filter = $scope.filter = { props: [ 'type', - 'indexed', - 'analyzed', + 'aggregatable', + 'searchable', 'missing', 'name' ], defaults: { - missing: true + missing: true, + type: 'any' }, boolOpts: [ { label: 'any', value: undefined }, @@ -69,16 +70,16 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou return field.display; }, isFieldFiltered: function (field) { - const matchFilter = (filter.vals.type == null || field.type === filter.vals.type); - const isAnalyzed = (filter.vals.analyzed == null || field.analyzed === filter.vals.analyzed); - const isIndexed = (filter.vals.indexed == null || field.indexed === filter.vals.indexed); + const matchFilter = (filter.vals.type === 'any' || field.type === filter.vals.type); + const isAggregatable = (filter.vals.aggregatable == null || field.aggregatable === filter.vals.aggregatable); + const isSearchable = (filter.vals.searchable == null || field.searchable === filter.vals.searchable); const scriptedOrMissing = (!filter.vals.missing || field.scripted || field.rowCount > 0); const matchName = (!filter.vals.name || field.name.indexOf(filter.vals.name) !== -1); return !field.display && matchFilter - && isAnalyzed - && isIndexed + && isAggregatable + && isSearchable && scriptedOrMissing && matchName ; @@ -146,7 +147,7 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou .commit(); // include undefined so the user can clear the filter - $scope.fieldTypes = _.union([undefined], _.pluck(fields, 'type')); + $scope.fieldTypes = _.union(['any'], _.pluck(fields, 'type')); }); $scope.increaseFieldCounter = function (fieldName) { diff --git a/src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js b/src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js index 5218ff2bf67de..2c0d3aecd6c9a 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js +++ b/src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js @@ -27,7 +27,6 @@ uiModules.get('apps/management') { title: 'format' }, { title: 'searchable', info: 'These fields can be used in the filter bar' }, { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' }, - { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, { title: 'excluded', info: 'Fields that are excluded from _source when it is fetched' }, { title: 'controls', sortable: false } ]; @@ -77,10 +76,6 @@ uiModules.get('apps/management') markup: field.aggregatable ? yesTemplate : noTemplate, value: field.aggregatable }, - { - markup: field.analyzed ? yesTemplate : noTemplate, - value: field.analyzed - }, { markup: excluded ? yesTemplate : noTemplate, value: excluded diff --git a/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js index 1606149ad36ac..fe69e4721fe15 100644 --- a/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js +++ b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js @@ -1,5 +1,7 @@ import _ from 'lodash'; + import handleESError from '../../../lib/handle_es_error'; +import { shouldReadFieldFromDocValues } from './should_read_field_from_doc_values'; export function registerFieldCapabilities(server) { server.route({ @@ -19,7 +21,11 @@ export function registerFieldCapabilities(server) { (res) => { const fields = _.get(res, 'indices._all.fields', {}); const fieldsFilteredValues = _.mapValues(fields, (value) => { - return _.pick(value, ['searchable', 'aggregatable']); + return { + searchable: value.searchable, + aggregatable: value.aggregatable, + readFromDocValues: shouldReadFieldFromDocValues(value.aggregatable, value.type) + }; }); const retVal = { fields: fieldsFilteredValues }; diff --git a/src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js b/src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js new file mode 100644 index 0000000000000..7e4e14536cfec --- /dev/null +++ b/src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js @@ -0,0 +1,4 @@ +// see https://github.com/elastic/kibana/issues/11141 for details +export function shouldReadFieldFromDocValues(aggregatable, esType) { + return aggregatable && esType !== 'text' && !esType.startsWith('_'); +} diff --git a/src/fixtures/logstash_fields.js b/src/fixtures/logstash_fields.js index 603e962c28168..912a0b2961db4 100644 --- a/src/fixtures/logstash_fields.js +++ b/src/fixtures/logstash_fields.js @@ -1,52 +1,48 @@ import { castEsToKbnFieldTypeName } from '../utils'; +import { shouldReadFieldFromDocValues } from '../core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values'; function stubbedLogstashFields() { return [ - // |indexed - // | |analyzed - // | | |aggregatable - // | | | |searchable - // name esType | | | | |metadata - ['bytes', 'long', true, true, true, true, { count: 10, docValues: true } ], - ['ssl', 'boolean', true, true, true, true, { count: 20 } ], - ['@timestamp', 'date', true, true, true, true, { count: 30 } ], - ['time', 'date', true, true, true, true, { count: 30 } ], - ['@tags', 'keyword', true, true, true, true ], - ['utc_time', 'date', true, true, true, true ], - ['phpmemory', 'integer', true, true, true, true ], - ['ip', 'ip', true, true, true, true ], - ['request_body', 'attachment', true, true, true, true ], - ['point', 'geo_point', true, true, true, true ], - ['area', 'geo_shape', true, true, true, true ], - ['hashed', 'murmur3', true, true, false, true ], - ['geo.coordinates', 'geo_point', true, true, true, true ], - ['extension', 'keyword', true, true, true, true ], - ['machine.os', 'text', true, true, true, true ], - ['machine.os.raw', 'keyword', true, false, true, true, { docValues: true } ], - ['geo.src', 'keyword', true, true, true, true ], - ['_id', 'keyword', false, false, true, true ], - ['_type', 'keyword', false, false, true, true ], - ['_source', 'keyword', false, false, true, true ], - ['non-filterable', 'text', false, false, true, false], - ['non-sortable', 'text', false, false, false, false], - ['custom_user_field', 'conflict', false, false, true, true ], - ['script string', 'text', false, false, true, false, { script: '\'i am a string\'' } ], - ['script number', 'long', false, false, true, false, { script: '1234' } ], - ['script date', 'date', false, false, true, false, { script: '1234', lang: 'painless' } ], - ['script murmur3', 'murmur3', false, false, true, false, { script: '1234' } ], + // |aggregatable + // | |searchable + // name esType | | |metadata + ['bytes', 'long', true, true, { count: 10 } ], + ['ssl', 'boolean', true, true, { count: 20 } ], + ['@timestamp', 'date', true, true, { count: 30 } ], + ['time', 'date', true, true, { count: 30 } ], + ['@tags', 'keyword', true, true ], + ['utc_time', 'date', true, true ], + ['phpmemory', 'integer', true, true ], + ['ip', 'ip', true, true ], + ['request_body', 'attachment', true, true ], + ['point', 'geo_point', true, true ], + ['area', 'geo_shape', true, true ], + ['hashed', 'murmur3', false, true ], + ['geo.coordinates', 'geo_point', true, true ], + ['extension', 'keyword', true, true ], + ['machine.os', 'text', true, true ], + ['machine.os.raw', 'keyword', true, true ], + ['geo.src', 'keyword', true, true ], + ['_id', '_id', true, true ], + ['_type', '_type', true, true ], + ['_source', '_source', true, true ], + ['non-filterable', 'text', true, false], + ['non-sortable', 'text', false, false], + ['custom_user_field', 'conflict', true, true ], + ['script string', 'text', true, false, { script: '\'i am a string\'' } ], + ['script number', 'long', true, false, { script: '1234' } ], + ['script date', 'date', true, false, { script: '1234', lang: 'painless' } ], + ['script murmur3', 'murmur3', true, false, { script: '1234' } ], ].map(function (row) { const [ name, esType, - indexed, - analyzed, aggregatable, searchable, metadata = {} ] = row; const { - docValues = false, count = 0, script, lang = script ? 'expression' : undefined, @@ -60,9 +56,7 @@ function stubbedLogstashFields() { return { name, type, - doc_values: docValues, - indexed, - analyzed, + readFromDocValues: shouldReadFieldFromDocValues(aggregatable, esType), aggregatable, searchable, count, diff --git a/src/ui/public/agg_types/__tests__/metrics/top_hit.js b/src/ui/public/agg_types/__tests__/metrics/top_hit.js index 59042df69cc8e..f7e876b73203c 100644 --- a/src/ui/public/agg_types/__tests__/metrics/top_hit.js +++ b/src/ui/public/agg_types/__tests__/metrics/top_hit.js @@ -71,15 +71,27 @@ describe('Top hit metric', function () { expect(aggDsl.top_hits.docvalue_fields).to.be(undefined); }); - it('should request both for the source and doc_values fields', function () { + it('requests both source and docvalues_fields for non-text aggregatable fields', function () { init({ field: 'bytes' }); expect(aggDsl.top_hits._source).to.be('bytes'); expect(aggDsl.top_hits.docvalue_fields).to.eql([ 'bytes' ]); }); - it('should only request for the source if the field does not have the doc_values property', function () { - init({ field: 'ssl' }); - expect(aggDsl.top_hits._source).to.be('ssl'); + it('requests just source for aggregatable text fields', function () { + init({ field: 'machine.os' }); + expect(aggDsl.top_hits._source).to.be('machine.os'); + expect(aggDsl.top_hits.docvalue_fields).to.be(undefined); + }); + + it('requests just source for not-aggregatable text fields', function () { + init({ field: 'non-sortable' }); + expect(aggDsl.top_hits._source).to.be('non-sortable'); + expect(aggDsl.top_hits.docvalue_fields).to.be(undefined); + }); + + it('requests just source for not-aggregatable, non-text fields', function () { + init({ field: 'hashed' }); + expect(aggDsl.top_hits._source).to.be('hashed'); expect(aggDsl.top_hits.docvalue_fields).to.be(undefined); }); diff --git a/src/ui/public/agg_types/controls/field.html b/src/ui/public/agg_types/controls/field.html index bc15e9b823d79..9b5a2797c9f6e 100644 --- a/src/ui/public/agg_types/controls/field.html +++ b/src/ui/public/agg_types/controls/field.html @@ -3,25 +3,6 @@ Field - - Analyzed Field - -
-

- Careful! The field selected contains analyzed strings. Analyzed strings are highly unique and can use a lot of memory to visualize. Values such as foo-bar will be broken into foo and bar. See Mapping Types for more information on setting this field as keyword (not analyzed). -

- -

- Tip: {{agg.params.field.name + '.keyword'}} may be a non analyzed version of this field. -

- -

- Tip: {{agg.params.field.name + '.raw'}} may be a non analyzed version of this field. -

-
- - { - return _.has(field, 'aggregatable') && _.has(field, 'searchable'); + function isFieldRefreshRequired(indexPattern) { + if (!indexPattern.fields) { + return true; + } + + return indexPattern.fields.every(field => { + // See https://github.com/elastic/kibana/pull/8421 + const hasFieldCaps = ('aggregatable' in field) && ('searchable' in field); + + // See https://github.com/elastic/kibana/pull/11969 + const hasDocValuesFlag = ('readFromDocValues' in field); + + return !hasFieldCaps || !hasDocValuesFlag; }); } @@ -108,7 +118,7 @@ export function IndexPatternProvider(Private, Notifier, config, kbnIndex, Promis return promise; } - if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) { + if (isFieldRefreshRequired(indexPattern)) { promise = indexPattern.refreshFields(); } return promise.then(() => {initFields(indexPattern);}); diff --git a/src/utils/kbn_field_types.js b/src/utils/kbn_field_types.js index af3ffbb447f4f..c83ea814ae4d1 100644 --- a/src/utils/kbn_field_types.js +++ b/src/utils/kbn_field_types.js @@ -22,7 +22,7 @@ const KBN_FIELD_TYPES = [ name: 'string', sortable: true, filterable: true, - esTypes: ['string', 'text', 'keyword'], + esTypes: ['string', 'text', 'keyword', '_type', '_id'], }), new KbnFieldType({ name: 'number', diff --git a/test/functional/apps/management/_index_pattern_create_delete.js b/test/functional/apps/management/_index_pattern_create_delete.js index fb12f5f4fadd8..5f55e744c94b5 100644 --- a/test/functional/apps/management/_index_pattern_create_delete.js +++ b/test/functional/apps/management/_index_pattern_create_delete.js @@ -52,7 +52,6 @@ export default function ({ getService, getPageObjects }) { 'format', 'searchable', 'aggregatable', - 'analyzed', 'excluded', 'controls' ];