Skip to content

Commit

Permalink
Merge branch '7.x' of https://github.com/elastic/kibana into backport…
Browse files Browse the repository at this point in the history
…/7.x/pr-53822
  • Loading branch information
Dosant committed Jan 3, 2020
2 parents cc4def4 + 6f6aacb commit d77ad44
Show file tree
Hide file tree
Showing 39 changed files with 5,783 additions and 5,183 deletions.
1 change: 1 addition & 0 deletions docs/maps/maps-aggregations.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ To enable top hits:
. Set *Entity* to the field that identifies entities in your documents.
This field will be used in the terms aggregation to group your documents into entity buckets.
. Set *Documents per entity* to configure the maximum number of documents accumulated per entity.
This setting is limited to the `index.max_inner_result_window` index setting, which defaults to 100.

[role="screenshot"]
image::maps/images/top_hits.png[]
Expand Down
2 changes: 1 addition & 1 deletion docs/maps/vector-layer.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ See map.regionmap.* in <<settings>> for details.
*Documents*:: Vector data from a Kibana index pattern.
The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape].

NOTE: Document results are limited to the first 10000 matching documents.
NOTE: Document results are limited to the `index.max_result_window` index setting, which defaults to 10000.
Use <<maps-aggregations, aggregations>> to plot large data sets.

*Grid aggregation*:: Geospatial data grouped in grids with metrics for each gridded cell.
Expand Down
11 changes: 7 additions & 4 deletions docs/user/reporting/reporting-troubleshooting.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ dependencies for Chromium.
Make sure Kibana server OS has the appropriate packages installed for the distribution.

If you are using CentOS/RHEL systems, install the following packages:

* `ipa-gothic-fonts`
* `xorg-x11-fonts-100dpi`
* `xorg-x11-fonts-75dpi`
Expand All @@ -28,6 +29,7 @@ If you are using CentOS/RHEL systems, install the following packages:
* `freetype`

If you are using Ubuntu/Debian systems, install the following packages:

* `fonts-liberation`
* `libfontconfig1`

Expand Down Expand Up @@ -105,9 +107,10 @@ has its own command-line method to generate its own debug logs, which can someti
caused by Kibana or Chromium. See more at https://github.com/GoogleChrome/puppeteer/blob/v1.19.0/README.md#debugging-tips

Using Puppeteer's debug method when launching Kibana would look like:
> Enable verbose logging - internal DevTools protocol traffic will be logged via the debug module under the puppeteer namespace.
> ```
> env DEBUG="puppeteer:*" ./bin/kibana
> ```
```
env DEBUG="puppeteer:*" ./bin/kibana
```
The internal DevTools protocol traffic will be logged via the `debug` module under the `puppeteer` namespace.


The Puppeteer logs are very verbose and could possibly contain sensitive information. Handle the generated output with care.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { withoutControlCharacters } from './string_utils';

describe('ensureSingleLineString()', () => {
test('works with plain ole strings', () => {
expect(withoutControlCharacters('')).toEqual('');
expect(withoutControlCharacters(' a b c ')).toEqual(' a b c ');
});

test('works with multiple control characters', () => {
expect(withoutControlCharacters(' \r \n ')).toEqual(' ; ; ');
expect(withoutControlCharacters('\r \n ')).toEqual('; ; ');
expect(withoutControlCharacters(' \r \n')).toEqual(' ; ;');
expect(withoutControlCharacters('\r \n')).toEqual('; ;');
});

test('works with /00-/1F, except tab', () => {
for (let c = 0; c <= 0x1f; c++) {
if (c === 0x09) {
expect(withoutControlCharacters(String.fromCharCode(c))).toEqual('\t');
} else {
expect(withoutControlCharacters(String.fromCharCode(c))).toEqual(';');
}
}
expect(withoutControlCharacters(String.fromCharCode(0x20))).toEqual(' ');
});

test('works with /7F-/9F', () => {
expect(withoutControlCharacters(String.fromCharCode(0x7e))).toEqual('~');
for (let c = 0x7f; c <= 0x9f; c++) {
expect(withoutControlCharacters(String.fromCharCode(c))).toEqual(';');
}
const nbsp = String.fromCharCode(0xa0);
expect(withoutControlCharacters(nbsp)).toEqual(nbsp);
});

test('works with UCS newlines', () => {
expect(withoutControlCharacters('\u2027')).toEqual('\u2027');
expect(withoutControlCharacters('\u2028')).toEqual(';');
expect(withoutControlCharacters('\u2029')).toEqual(';');
expect(withoutControlCharacters('\u202A')).toEqual('\u202A');
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

// see: https://en.wikipedia.org/wiki/Unicode_control_characters
// but don't include tabs (0x09), they're fine
const CONTROL_CHAR_PATTERN = /[\x00-\x08]|[\x0A-\x1F]|[\x7F-\x9F]|[\u2028-\u2029]/g;

// replaces control characters in string with ;, but leaves tabs
export function withoutControlCharacters(s: string): string {
return s.replace(CONTROL_CHAR_PATTERN, ';');
}
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,6 @@ describe('execute()', () => {
config: {},
secrets: {},
});
expect(mockedLogger.info).toHaveBeenCalledWith('message text here');
expect(mockedLogger.info).toHaveBeenCalledWith('server-log: message text here');
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,34 @@ import { schema, TypeOf } from '@kbn/config-schema';

import { Logger } from '../../../../../../src/core/server';
import { ActionType, ActionTypeExecutorOptions, ActionTypeExecutorResult } from '../types';
import { withoutControlCharacters } from './lib/string_utils';

const ACTION_NAME = 'server-log';

// params definition

export type ActionParamsType = TypeOf<typeof ParamsSchema>;

const ParamsSchema = schema.object({
message: schema.string(),
level: schema.oneOf([
schema.literal('trace'),
schema.literal('debug'),
schema.literal('info'),
schema.literal('warn'),
schema.literal('error'),
schema.literal('fatal'),
]),
level: schema.oneOf(
[
schema.literal('trace'),
schema.literal('debug'),
schema.literal('info'),
schema.literal('warn'),
schema.literal('error'),
schema.literal('fatal'),
],
{ defaultValue: 'info' }
),
});

// action type definition
export function getActionType({ logger }: { logger: Logger }): ActionType {
return {
id: '.server-log',
name: 'server-log',
name: ACTION_NAME,
validate: {
params: ParamsSchema,
},
Expand All @@ -48,8 +54,9 @@ async function executor(
const actionId = execOptions.actionId;
const params = execOptions.params as ActionParamsType;

const sanitizedMessage = withoutControlCharacters(params.message);
try {
logger[params.level](params.message);
logger[params.level](`${ACTION_NAME}: ${sanitizedMessage}`);
} catch (err) {
const message = i18n.translate('xpack.actions.builtin.serverLog.errorLoggingErrorMessage', {
defaultMessage: 'error logging message',
Expand Down
2 changes: 1 addition & 1 deletion x-pack/legacy/plugins/actions/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export class Plugin {
private licenseState: LicenseState | null = null;

constructor(initializerContext: ActionsPluginInitializerContext) {
this.logger = initializerContext.logger.get('plugins', 'alerting');
this.logger = initializerContext.logger.get('plugins', 'actions');
this.config$ = initializerContext.config.create();
this.kibana$ = initializerContext.config.kibana$;
}
Expand Down
5 changes: 4 additions & 1 deletion x-pack/legacy/plugins/maps/common/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export const APP_ICON = 'gisApp';

export const MAP_APP_PATH = `app/${APP_ID}`;
export const GIS_API_PATH = `api/${APP_ID}`;
export const INDEX_SETTINGS_API_PATH = `${GIS_API_PATH}/indexSettings`;

export const MAP_BASE_URL = `/${MAP_APP_PATH}#/${MAP_SAVED_OBJECT_TYPE}`;

Expand Down Expand Up @@ -69,7 +70,9 @@ export const MAX_ZOOM = 24;

export const DECIMAL_DEGREES_PRECISION = 5; // meters precision
export const ZOOM_PRECISION = 2;
export const ES_SIZE_LIMIT = 10000;
export const DEFAULT_MAX_RESULT_WINDOW = 10000;
export const DEFAULT_MAX_INNER_RESULT_WINDOW = 100;
export const DEFAULT_MAX_BUCKETS_LIMIT = 10000;

export const FEATURE_ID_PROPERTY_NAME = '__kbn__feature_id__';
export const FEATURE_VISIBLE_PROPERTY_NAME = '__kbn_isvisibleduetojoin__';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@

import { AbstractVectorSource } from '../vector_source';
import React from 'react';
import { ES_GEO_FIELD_TYPE, GEOJSON_FILE, ES_SIZE_LIMIT } from '../../../../common/constants';
import {
ES_GEO_FIELD_TYPE,
GEOJSON_FILE,
DEFAULT_MAX_RESULT_WINDOW,
} from '../../../../common/constants';
import { ClientFileCreateSourceEditor } from './create_client_file_source_editor';
import { ESSearchSource } from '../es_search_source';
import uuid from 'uuid/v4';
Expand Down Expand Up @@ -82,7 +86,7 @@ export class GeojsonFileSource extends AbstractVectorSource {
addAndViewSource(null);
} else {
// Only turn on bounds filter for large doc counts
const filterByMapBounds = indexDataResp.docCount > ES_SIZE_LIMIT;
const filterByMapBounds = indexDataResp.docCount > DEFAULT_MAX_RESULT_WINDOW;
const source = new ESSearchSource(
{
id: uuid(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@ import { NoIndexPatternCallout } from '../../../components/no_index_pattern_call
import { FormattedMessage } from '@kbn/i18n/react';
import { i18n } from '@kbn/i18n';
import { kfetch } from 'ui/kfetch';
import { ES_GEO_FIELD_TYPE, GIS_API_PATH, ES_SIZE_LIMIT } from '../../../../common/constants';
import {
ES_GEO_FIELD_TYPE,
GIS_API_PATH,
DEFAULT_MAX_RESULT_WINDOW,
} from '../../../../common/constants';
import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';

import { npStart } from 'ui/new_platform';
Expand Down Expand Up @@ -96,7 +100,7 @@ export class CreateSourceEditor extends Component {
let indexHasSmallDocCount = false;
try {
const indexDocCount = await this.loadIndexDocCount(indexPattern.title);
indexHasSmallDocCount = indexDocCount <= ES_SIZE_LIMIT;
indexHasSmallDocCount = indexDocCount <= DEFAULT_MAX_RESULT_WINDOW;
} catch (error) {
// retrieving index count is a nice to have and is not essential
// do not interrupt user flow if unable to retrieve count
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@ import { UpdateSourceEditor } from './update_source_editor';
import {
ES_SEARCH,
ES_GEO_FIELD_TYPE,
ES_SIZE_LIMIT,
DEFAULT_MAX_BUCKETS_LIMIT,
SORT_ORDER,
} from '../../../../common/constants';
import { i18n } from '@kbn/i18n';
import { getDataSourceLabel } from '../../../../common/i18n_getters';
import { getSourceFields } from '../../../index_pattern_util';
import { loadIndexSettings } from './load_index_settings';

import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';
import { ESDocField } from '../../fields/es_doc_field';
Expand Down Expand Up @@ -267,8 +268,8 @@ export class ESSearchSource extends AbstractESSource {
entitySplit: {
terms: {
field: topHitsSplitField,
size: ES_SIZE_LIMIT,
shard_size: ES_SIZE_LIMIT,
size: DEFAULT_MAX_BUCKETS_LIMIT,
shard_size: DEFAULT_MAX_BUCKETS_LIMIT,
},
aggs: {
entityHits: {
Expand All @@ -290,7 +291,7 @@ export class ESSearchSource extends AbstractESSource {
const entityBuckets = _.get(resp, 'aggregations.entitySplit.buckets', []);
const totalEntities = _.get(resp, 'aggregations.totalEntities.value', 0);
// can not compare entityBuckets.length to totalEntities because totalEntities is an approximate
const areEntitiesTrimmed = entityBuckets.length >= ES_SIZE_LIMIT;
const areEntitiesTrimmed = entityBuckets.length >= DEFAULT_MAX_BUCKETS_LIMIT;
let areTopHitsTrimmed = false;
entityBuckets.forEach(entityBucket => {
const total = _.get(entityBucket, 'entityHits.hits.total', 0);
Expand All @@ -315,7 +316,7 @@ export class ESSearchSource extends AbstractESSource {

// searchFilters.fieldNames contains geo field and any fields needed for styling features
// Performs Elasticsearch search request being careful to pull back only required fields to minimize response size
async _getSearchHits(layerName, searchFilters, registerCancelCallback) {
async _getSearchHits(layerName, searchFilters, maxResultWindow, registerCancelCallback) {
const initialSearchContext = {
docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames),
};
Expand All @@ -331,7 +332,7 @@ export class ESSearchSource extends AbstractESSource {
);
searchSource = await this._makeSearchSource(
searchFilters,
ES_SIZE_LIMIT,
maxResultWindow,
initialSearchContext
);
searchSource.setField('source', false); // do not need anything from _source
Expand All @@ -340,7 +341,7 @@ export class ESSearchSource extends AbstractESSource {
// geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source
searchSource = await this._makeSearchSource(
searchFilters,
ES_SIZE_LIMIT,
maxResultWindow,
initialSearchContext
);
// Setting "fields" instead of "source: { includes: []}"
Expand Down Expand Up @@ -382,11 +383,19 @@ export class ESSearchSource extends AbstractESSource {
}

async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) {
const indexPattern = await this.getIndexPattern();

const indexSettings = await loadIndexSettings(indexPattern.title);

const { hits, meta } = this._isTopHits()
? await this._getTopHits(layerName, searchFilters, registerCancelCallback)
: await this._getSearchHits(layerName, searchFilters, registerCancelCallback);
: await this._getSearchHits(
layerName,
searchFilters,
indexSettings.maxResultWindow,
registerCancelCallback
);

const indexPattern = await this.getIndexPattern();
const unusedMetaFields = indexPattern.metaFields.filter(metaField => {
return !['_id', '_index'].includes(metaField);
});
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import {
DEFAULT_MAX_RESULT_WINDOW,
DEFAULT_MAX_INNER_RESULT_WINDOW,
INDEX_SETTINGS_API_PATH,
} from '../../../../common/constants';
import { kfetch } from 'ui/kfetch';
import { toastNotifications } from 'ui/notify';
import { i18n } from '@kbn/i18n';

let toastDisplayed = false;
const indexSettings = new Map();

export async function loadIndexSettings(indexPatternTitle) {
if (indexSettings.has(indexPatternTitle)) {
return indexSettings.get(indexPatternTitle);
}

const fetchPromise = fetchIndexSettings(indexPatternTitle);
indexSettings.set(indexPatternTitle, fetchPromise);
return fetchPromise;
}

async function fetchIndexSettings(indexPatternTitle) {
try {
const indexSettings = await kfetch({
pathname: `../${INDEX_SETTINGS_API_PATH}`,
query: {
indexPatternTitle,
},
});
return indexSettings;
} catch (err) {
const warningMsg = i18n.translate('xpack.maps.indexSettings.fetchErrorMsg', {
defaultMessage: `Unable to fetch index settings for index pattern '{indexPatternTitle}'.
Ensure you have '{viewIndexMetaRole}' role.`,
values: {
indexPatternTitle,
viewIndexMetaRole: 'view_index_metadata',
},
});
if (!toastDisplayed) {
// Only show toast for first failure to avoid flooding user with warnings
toastDisplayed = true;
toastNotifications.addWarning(warningMsg);
}
console.warn(warningMsg);
return {
maxResultWindow: DEFAULT_MAX_RESULT_WINDOW,
maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW,
};
}
}
Loading

0 comments on commit d77ad44

Please sign in to comment.