From 8458e476142f2f940822e62740cf6248881a5023 Mon Sep 17 00:00:00 2001 From: Gidi Meir Morris Date: Thu, 16 Jan 2020 09:55:51 +0000 Subject: [PATCH 1/5] removes usage of the _id field in Task manager (#54765) As of Elasticsearch 8.0.0 it will no longer be possible to use the _id field on documents. This PR removes the usage that Task Manager makes of this field and switches to pinned queries to achieve a similar effect. --- .../plugins/task_manager/server/legacy.ts | 4 +- x-pack/plugins/task_manager/server/config.ts | 7 +- .../mark_available_tasks_as_claimed.test.ts | 8 +- .../mark_available_tasks_as_claimed.ts | 74 ++-- .../server/queries/query_clauses.test.ts | 51 +++ .../server/queries/query_clauses.ts | 222 ++++++++-- .../task_manager/server/task_manager.ts | 6 +- .../task_manager/server/task_store.test.ts | 388 +++++++----------- .../plugins/task_manager/server/task_store.ts | 131 +++--- .../plugins/task_manager/index.js | 2 + .../plugins/task_manager/init_routes.js | 37 +- .../task_manager/task_manager_integration.js | 116 +++++- 12 files changed, 612 insertions(+), 434 deletions(-) create mode 100644 x-pack/plugins/task_manager/server/queries/query_clauses.test.ts diff --git a/x-pack/legacy/plugins/task_manager/server/legacy.ts b/x-pack/legacy/plugins/task_manager/server/legacy.ts index 772309d67c334..f5e81bfd90169 100644 --- a/x-pack/legacy/plugins/task_manager/server/legacy.ts +++ b/x-pack/legacy/plugins/task_manager/server/legacy.ts @@ -17,7 +17,7 @@ import { TaskInstanceWithId, TaskDefinition, } from '../../../../plugins/task_manager/server/task.js'; -import { FetchOpts } from '../../../../plugins/task_manager/server/task_store.js'; +import { SearchOpts } from '../../../../plugins/task_manager/server/task_store.js'; // Once all plugins are migrated to NP and we can remove Legacy TaskManager in version 8.0.0, // this can be removed @@ -46,7 +46,7 @@ export function createLegacyApi(legacyTaskManager: Promise): Legacy registerTaskDefinitions: (taskDefinitions: TaskDictionary) => { legacyTaskManager.then((tm: TaskManager) => tm.registerTaskDefinitions(taskDefinitions)); }, - fetch: (opts: FetchOpts) => legacyTaskManager.then((tm: TaskManager) => tm.fetch(opts)), + fetch: (opts: SearchOpts) => legacyTaskManager.then((tm: TaskManager) => tm.fetch(opts)), remove: (id: string) => legacyTaskManager.then((tm: TaskManager) => tm.remove(id)), schedule: (taskInstance: TaskInstanceWithDeprecatedFields, options?: any) => legacyTaskManager.then((tm: TaskManager) => tm.schedule(taskInstance, options)), diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index 06e6ad3e62282..e2752f80ad6c4 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -6,6 +6,9 @@ import { schema, TypeOf } from '@kbn/config-schema'; +export const DEFAULT_MAX_WORKERS = 10; +export const DEFAULT_POLL_INTERVAL = 3000; + export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), /* The maximum number of times a task will be attempted before being abandoned as failed */ @@ -15,7 +18,7 @@ export const configSchema = schema.object({ }), /* How often, in milliseconds, the task manager will look for more work. */ poll_interval: schema.number({ - defaultValue: 3000, + defaultValue: DEFAULT_POLL_INTERVAL, min: 100, }), /* How many requests can Task Manager buffer before it rejects new requests. */ @@ -35,7 +38,7 @@ export const configSchema = schema.object({ }), /* The maximum number of tasks that this Kibana instance will run simultaneously. */ max_workers: schema.number({ - defaultValue: 10, + defaultValue: DEFAULT_MAX_WORKERS, // disable the task manager rather than trying to specify it with 0 workers min: 1, }), diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts index 93a8187b673be..ac98fbbda5aa2 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts @@ -9,9 +9,9 @@ import { asUpdateByQuery, shouldBeOneOf, mustBeAllOf, - ExistsBoolClause, - TermBoolClause, - RangeBoolClause, + ExistsFilter, + TermFilter, + RangeFilter, } from './query_clauses'; import { @@ -51,7 +51,7 @@ describe('mark_available_tasks_as_claimed', () => { // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), // Either task has an schedule or the attempts < the maximum configured - shouldBeOneOf( + shouldBeOneOf( TaskWithSchedule, ...Object.entries(definitions).map(([type, { maxAttempts }]) => taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts index 6691b31a546bc..b0d9dc61c9667 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts @@ -3,24 +3,24 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { defaultsDeep } from 'lodash'; import { - BoolClause, - IDsClause, SortClause, ScriptClause, - ExistsBoolClause, - TermBoolClause, - RangeBoolClause, + ExistsFilter, + TermFilter, + RangeFilter, + mustBeAllOf, + MustCondition, + MustNotCondition, } from './query_clauses'; -export const TaskWithSchedule: ExistsBoolClause = { +export const TaskWithSchedule: ExistsFilter = { exists: { field: 'task.schedule' }, }; export function taskWithLessThanMaxAttempts( type: string, maxAttempts: number -): BoolClause { +): MustCondition { return { bool: { must: [ @@ -37,34 +37,37 @@ export function taskWithLessThanMaxAttempts( }; } -export const IdleTaskWithExpiredRunAt: BoolClause = { +export function tasksClaimedByOwner(taskManagerId: string) { + return mustBeAllOf( + { + term: { + 'task.ownerId': taskManagerId, + }, + }, + { term: { 'task.status': 'claiming' } } + ); +} + +export const IdleTaskWithExpiredRunAt: MustCondition = { bool: { must: [{ term: { 'task.status': 'idle' } }, { range: { 'task.runAt': { lte: 'now' } } }], }, }; -export const taskWithIDsAndRunnableStatus = ( - claimTasksById: string[] -): BoolClause => ({ +export const InactiveTasks: MustNotCondition = { bool: { - must: [ + must_not: [ { bool: { - should: [{ term: { 'task.status': 'idle' } }, { term: { 'task.status': 'failed' } }], - }, - }, - { - ids: { - values: claimTasksById, + should: [{ term: { 'task.status': 'running' } }, { term: { 'task.status': 'claiming' } }], }, }, + { range: { 'task.retryAt': { gt: 'now' } } }, ], }, -}); +}; -export const RunningOrClaimingTaskWithExpiredRetryAt: BoolClause< - TermBoolClause | RangeBoolClause -> = { +export const RunningOrClaimingTaskWithExpiredRetryAt: MustCondition = { bool: { must: [ { @@ -95,31 +98,6 @@ if (doc['task.runAt'].size()!=0) { }, }; -const SORT_VALUE_TO_BE_FIRST = 0; -export const sortByIdsThenByScheduling = (claimTasksById: string[]): SortClause => { - const { - _script: { - script: { source }, - }, - } = SortByRunAtAndRetryAt; - return defaultsDeep( - { - _script: { - script: { - source: ` -if(params.ids.contains(doc['_id'].value)){ - return ${SORT_VALUE_TO_BE_FIRST}; -} -${source} -`, - params: { ids: claimTasksById }, - }, - }, - }, - SortByRunAtAndRetryAt - ); -}; - export const updateFields = (fieldUpdates: { [field: string]: string | number | Date; }): ScriptClause => ({ diff --git a/x-pack/plugins/task_manager/server/queries/query_clauses.test.ts b/x-pack/plugins/task_manager/server/queries/query_clauses.test.ts new file mode 100644 index 0000000000000..beb8f864bd754 --- /dev/null +++ b/x-pack/plugins/task_manager/server/queries/query_clauses.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import _ from 'lodash'; +import { + MustCondition, + shouldBeOneOf, + mustBeAllOf, + ExistsFilter, + TermFilter, + RangeFilter, + matchesClauses, +} from './query_clauses'; + +describe('matchesClauses', () => { + test('merges multiple types of Bool Clauses into one', () => { + const TaskWithSchedule: ExistsFilter = { + exists: { field: 'task.schedule' }, + }; + + const IdleTaskWithExpiredRunAt: MustCondition = { + bool: { + must: [{ term: { 'task.status': 'idle' } }, { range: { 'task.runAt': { lte: 'now' } } }], + }, + }; + + const RunningTask: MustCondition = { + bool: { + must: [{ term: { 'task.status': 'running' } }], + }, + }; + + expect( + matchesClauses( + mustBeAllOf(TaskWithSchedule), + shouldBeOneOf( + RunningTask, + IdleTaskWithExpiredRunAt + ) + ) + ).toMatchObject({ + bool: { + must: [TaskWithSchedule], + should: [RunningTask, IdleTaskWithExpiredRunAt], + }, + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/queries/query_clauses.ts b/x-pack/plugins/task_manager/server/queries/query_clauses.ts index 1f76ce99e600a..de7e3b085ba2d 100644 --- a/x-pack/plugins/task_manager/server/queries/query_clauses.ts +++ b/x-pack/plugins/task_manager/server/queries/query_clauses.ts @@ -4,30 +4,150 @@ * you may not use this file except in compliance with the Elastic License. */ -export interface TermBoolClause { +/** + * Terminology + * =========== + * The terms for the differenct clauses in an Elasticsearch query can be confusing, here are some + * clarifications that might help you understand the Typescript types we use here. + * + * Given the following Query: + * { + * "query": { (1) + * "bool": { (2) + * "must": + * [ + * (3) { "term" : { "tag" : "wow" } }, + * { "term" : { "tag" : "elasticsearch" } }, + * { + * "must" : { "term" : { "user" : "kimchy" } } + * } + * ] + * } + * } + * } + * + * These are referred to as: + * (1). BoolClause / BoolClauseWithAnyCondition + * (2). BoolCondition / AnyBoolCondition + * (3). BoolClauseFilter + * + */ + +export interface TermFilter { term: { [field: string]: string | string[] }; } -export interface RangeBoolClause { - range: { [field: string]: { lte: string | number } | { lt: string | number } }; +export interface RangeFilter { + range: { + [field: string]: { lte: string | number } | { lt: string | number } | { gt: string | number }; + }; } -export interface ExistsBoolClause { +export interface ExistsFilter { exists: { field: string }; } -export interface IDsClause { - ids: { - values: string[]; - }; -} -export interface ShouldClause { - should: Array | IDsClause | T>; +type BoolClauseFilter = TermFilter | RangeFilter | ExistsFilter; +type BoolClauseFiltering = + | BoolClauseWithAnyCondition + | PinnedQuery + | T; + +enum Conditions { + Should = 'should', + Must = 'must', + MustNot = 'must_not', + Filter = 'filter', } -export interface MustClause { - must: Array | IDsClause | T>; + +/** + * Describe a specific BoolClause Condition with a BoolClauseFilter on it, such as: + * ``` + * { + * must : [ + * T, ... + * ] + * } + * ``` + */ +type BoolCondition = { + [c in C]: Array>; +}; + +/** + * Describe a Bool clause with a specific Condition, such as: + * ``` + * { + * // described by BoolClause + * bool: { + * // described by BoolCondition + * must: [ + * T, ... + * ] + * } + * } + * ``` + */ +interface BoolClause { + bool: BoolCondition; } -export interface BoolClause { - bool: MustClause | ShouldClause; + +/** + * Describe a Bool clause with mixed Conditions, such as: + * ``` + * { + * // described by BoolClause<...> + * bool: { + * // described by BoolCondition + * must : { + * ... + * }, + * // described by BoolCondition + * should : { + * ... + * } + * } + * } + * ``` + */ +type AnyBoolCondition = { + [Condition in Conditions]?: Array>; +}; + +/** + * Describe a Bool Condition with any Condition on it, so it can handle both: + * ``` + * { + * bool: { + * must : { + * ... + * } + * } + * } + * ``` + * + * and: + * + * ``` + * { + * bool: { + * must_not : { + * ... + * } + * } + * } + * ``` + */ +export interface BoolClauseWithAnyCondition { + bool: AnyBoolCondition; } + +/** + * Describe the various Bool Clause Conditions we support, as specified in the Conditions enum + */ +export type ShouldCondition = BoolClause; +export type MustCondition = BoolClause; +export type MustNotCondition = BoolClause; +export type FilterCondition = BoolClause; + export interface SortClause { _script: { type: string; @@ -39,6 +159,8 @@ export interface SortClause { }; }; } +export type SortOptions = string | SortClause | Array; + export interface ScriptClause { source: string; lang: string; @@ -46,18 +168,34 @@ export interface ScriptClause { [field: string]: string | number | Date; }; } -export interface UpdateByQuery { - query: BoolClause; - sort: SortClause; + +export interface UpdateByQuery { + query: PinnedQuery | BoolClauseWithAnyCondition; + sort: SortOptions; seq_no_primary_term: true; script: ScriptClause; } -export function shouldBeOneOf( - ...should: Array | IDsClause | T> -): { - bool: ShouldClause; -} { +export interface PinnedQuery { + pinned: PinnedClause; +} + +export interface PinnedClause { + ids: string[]; + organic: BoolClauseWithAnyCondition; +} + +export function matchesClauses( + ...clauses: Array> +): BoolClauseWithAnyCondition { + return { + bool: Object.assign({}, ...clauses.map(clause => clause.bool)), + }; +} + +export function shouldBeOneOf( + ...should: Array> +): ShouldCondition { return { bool: { should, @@ -65,11 +203,9 @@ export function shouldBeOneOf( }; } -export function mustBeAllOf( - ...must: Array | IDsClause | T> -): { - bool: MustClause; -} { +export function mustBeAllOf( + ...must: Array> +): MustCondition { return { bool: { must, @@ -77,14 +213,36 @@ export function mustBeAllOf( }; } -export function asUpdateByQuery({ +export function filterDownBy( + ...filter: Array> +): FilterCondition { + return { + bool: { + filter, + }, + }; +} + +export function asPinnedQuery( + ids: PinnedClause['ids'], + organic: PinnedClause['organic'] +): PinnedQuery { + return { + pinned: { + ids, + organic, + }, + }; +} + +export function asUpdateByQuery({ query, update, sort, }: { - query: BoolClause; - update: ScriptClause; - sort: SortClause; + query: UpdateByQuery['query']; + update: UpdateByQuery['script']; + sort: UpdateByQuery['sort']; }): UpdateByQuery { return { query, diff --git a/x-pack/plugins/task_manager/server/task_manager.ts b/x-pack/plugins/task_manager/server/task_manager.ts index c0baed3708a0a..93e98f33a30b0 100644 --- a/x-pack/plugins/task_manager/server/task_manager.ts +++ b/x-pack/plugins/task_manager/server/task_manager.ts @@ -48,11 +48,11 @@ import { createTaskPoller, PollingError, PollingErrorType } from './task_poller' import { TaskPool } from './task_pool'; import { TaskManagerRunner, TaskRunner } from './task_runner'; import { - FetchOpts, FetchResult, TaskStore, OwnershipClaimingOpts, ClaimOwnershipResult, + SearchOpts, } from './task_store'; import { identifyEsError } from './lib/identify_es_error'; import { ensureDeprecatedFieldsAreCorrected } from './lib/correct_deprecated_fields'; @@ -323,12 +323,12 @@ export class TaskManager { } /** - * Fetches a paginatable list of scheduled tasks. + * Fetches a list of scheduled tasks. * * @param opts - The query options used to filter tasks * @returns {Promise} */ - public async fetch(opts: FetchOpts): Promise { + public async fetch(opts: SearchOpts): Promise { await this.waitUntilStarted(); return this.store.fetch(opts); } diff --git a/x-pack/plugins/task_manager/server/task_store.test.ts b/x-pack/plugins/task_manager/server/task_store.test.ts index f47cc41c2d045..e6cce7a664d91 100644 --- a/x-pack/plugins/task_manager/server/task_store.test.ts +++ b/x-pack/plugins/task_manager/server/task_store.test.ts @@ -16,7 +16,7 @@ import { TaskStatus, TaskLifecycleResult, } from './task'; -import { FetchOpts, StoreOpts, OwnershipClaimingOpts, TaskStore } from './task_store'; +import { StoreOpts, OwnershipClaimingOpts, TaskStore, SearchOpts } from './task_store'; import { savedObjectsRepositoryMock } from '../../../../src/core/server/mocks'; import { SavedObjectsSerializer, @@ -175,7 +175,7 @@ describe('TaskStore', () => { }); describe('fetch', () => { - async function testFetch(opts?: FetchOpts, hits: any[] = []) { + async function testFetch(opts?: SearchOpts, hits: any[] = []) { const callCluster = sinon.spy(async (name: string, params?: any) => ({ hits: { hits } })); const store = new TaskStore({ index: 'tasky', @@ -203,7 +203,7 @@ describe('TaskStore', () => { expect(args).toMatchObject({ index: 'tasky', body: { - sort: [{ 'task.runAt': 'asc' }, { _id: 'desc' }], + sort: [{ 'task.runAt': 'asc' }], query: { term: { type: 'task' } }, }, }); @@ -226,122 +226,6 @@ describe('TaskStore', () => { }, }); }); - - test('sorts by id if custom sort does not have an id sort in it', async () => { - const { args } = await testFetch({ - sort: [{ 'task.taskType': 'desc' }], - }); - - expect(args).toMatchObject({ - body: { - sort: [{ 'task.taskType': 'desc' }, { _id: 'desc' }], - }, - }); - }); - - test('allows custom sort by id', async () => { - const { args } = await testFetch({ - sort: [{ _id: 'asc' }], - }); - - expect(args).toMatchObject({ - body: { - sort: [{ _id: 'asc' }], - }, - }); - }); - - test('allows specifying pagination', async () => { - const now = new Date(); - const searchAfter = [now, '143243sdafa32']; - const { args } = await testFetch({ - searchAfter, - }); - - expect(args).toMatchObject({ - body: { - search_after: searchAfter, - }, - }); - }); - - test('returns paginated tasks', async () => { - const runAt = new Date(); - const { result } = await testFetch(undefined, [ - { - _id: 'aaa', - _source: { - type: 'task', - task: { - runAt, - taskType: 'foo', - schedule: undefined, - attempts: 0, - status: 'idle', - params: '{ "hello": "world" }', - state: '{ "baby": "Henhen" }', - user: 'jimbo', - scope: ['reporting'], - }, - }, - sort: ['a', 1], - }, - { - _id: 'bbb', - _source: { - type: 'task', - task: { - runAt, - taskType: 'bar', - schedule: { interval: '5m' }, - attempts: 2, - status: 'running', - params: '{ "shazm": 1 }', - state: '{ "henry": "The 8th" }', - user: 'dabo', - scope: ['reporting', 'ceo'], - }, - }, - sort: ['b', 2], - }, - ]); - - expect(result).toEqual({ - docs: [ - { - attempts: 0, - id: 'aaa', - schedule: undefined, - params: { hello: 'world' }, - runAt, - scheduledAt: mockedDate, - scope: ['reporting'], - state: { baby: 'Henhen' }, - status: 'idle', - taskType: 'foo', - user: 'jimbo', - retryAt: undefined, - startedAt: undefined, - }, - { - attempts: 2, - id: 'bbb', - schedule: { interval: '5m' }, - params: { shazm: 1 }, - runAt, - scheduledAt: mockedDate, - scope: ['reporting', 'ceo'], - state: { henry: 'The 8th' }, - status: 'running', - taskType: 'bar', - user: 'dabo', - retryAt: undefined, - startedAt: undefined, - }, - ], - searchAfter: ['b', 2], - }); - }); }); describe('claimAvailableTasks', () => { @@ -450,65 +334,88 @@ describe('TaskStore', () => { must: [ { bool: { - should: [ + must: [ { bool: { - must: [ - { term: { 'task.status': 'idle' } }, - { range: { 'task.runAt': { lte: 'now' } } }, + should: [ + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { range: { 'task.runAt': { lte: 'now' } } }, + ], + }, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + }, + }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], + }, + }, ], }, }, { bool: { - must: [ + should: [ + { exists: { field: 'task.schedule' } }, + { + bool: { + must: [ + { term: { 'task.taskType': 'foo' } }, + { + range: { + 'task.attempts': { + lt: maxAttempts, + }, + }, + }, + ], + }, + }, { bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, + must: [ + { term: { 'task.taskType': 'bar' } }, + { + range: { + 'task.attempts': { + lt: customMaxAttempts, + }, + }, + }, ], }, }, - { range: { 'task.retryAt': { lte: 'now' } } }, ], }, }, ], }, }, + ], + filter: [ { bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, + must_not: [ { bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, ], }, }, + { range: { 'task.retryAt': { gt: 'now' } } }, ], }, }, @@ -562,96 +469,99 @@ describe('TaskStore', () => { { term: { type: 'task' } }, { bool: { - should: [ + must: [ { - bool: { - must: [ - { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'task.status': 'idle' } }, - { range: { 'task.runAt': { lte: 'now' } } }, - ], - }, - }, - { - bool: { - must: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - }, + pinned: { + ids: [ + 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], + organic: { + bool: { + must: [ + { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { range: { 'task.runAt': { lte: 'now' } } }, + ], }, - { range: { 'task.retryAt': { lte: 'now' } } }, - ], - }, - }, - ], - }, - }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + }, }, - }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], }, - ], - }, + }, + ], }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, + }, + { + bool: { + should: [ + { exists: { field: 'task.schedule' } }, + { + bool: { + must: [ + { term: { 'task.taskType': 'foo' } }, + { + range: { + 'task.attempts': { + lt: maxAttempts, + }, + }, }, - }, + ], }, - ], - }, + }, + { + bool: { + must: [ + { term: { 'task.taskType': 'bar' } }, + { + range: { + 'task.attempts': { + lt: customMaxAttempts, + }, + }, + }, + ], + }, + }, + ], }, - ], - }, + }, + ], }, - ], + }, }, }, + ], + filter: [ { bool: { - must: [ + must_not: [ { bool: { should: [ - { term: { 'task.status': 'idle' } }, - { term: { 'task.status': 'failed' } }, - ], - }, - }, - { - ids: { - values: [ - 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', - 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, ], }, }, + { range: { 'task.retryAt': { gt: 'now' } } }, ], }, }, @@ -662,34 +572,26 @@ describe('TaskStore', () => { }, }); - expect(sort).toMatchObject({ - _script: { - type: 'number', - order: 'asc', - script: { - lang: 'painless', - source: ` -if(params.ids.contains(doc['_id'].value)){ - return 0; -} - + expect(sort).toMatchObject([ + '_score', + { + _script: { + type: 'number', + order: 'asc', + script: { + lang: 'painless', + source: ` if (doc['task.retryAt'].size()!=0) { return doc['task.retryAt'].value.toInstant().toEpochMilli(); } if (doc['task.runAt'].size()!=0) { return doc['task.runAt'].value.toInstant().toEpochMilli(); } - -`, - params: { - ids: [ - 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', - 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', - ], + `, }, }, }, - }); + ]); }); test('it claims tasks by setting their ownerId, status and retryAt', async () => { diff --git a/x-pack/plugins/task_manager/server/task_store.ts b/x-pack/plugins/task_manager/server/task_store.ts index f4695b152237a..4f2e97704941f 100644 --- a/x-pack/plugins/task_manager/server/task_store.ts +++ b/x-pack/plugins/task_manager/server/task_store.ts @@ -36,22 +36,23 @@ import { asUpdateByQuery, shouldBeOneOf, mustBeAllOf, - ExistsBoolClause, - TermBoolClause, - RangeBoolClause, - BoolClause, - IDsClause, + filterDownBy, + ExistsFilter, + TermFilter, + RangeFilter, + asPinnedQuery, + matchesClauses, } from './queries/query_clauses'; import { updateFields, IdleTaskWithExpiredRunAt, + InactiveTasks, RunningOrClaimingTaskWithExpiredRetryAt, TaskWithSchedule, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, - taskWithIDsAndRunnableStatus, - sortByIdsThenByScheduling, + tasksClaimedByOwner, } from './queries/mark_available_tasks_as_claimed'; export interface StoreOpts { @@ -65,18 +66,13 @@ export interface StoreOpts { } export interface SearchOpts { - searchAfter?: any[]; - sort?: object | object[]; + sort?: string | object | object[]; query?: object; size?: number; seq_no_primary_term?: boolean; search_after?: any[]; } -export interface FetchOpts extends SearchOpts { - sort?: object[]; -} - export interface UpdateByQuerySearchOpts extends SearchOpts { script?: object; } @@ -92,7 +88,6 @@ export interface OwnershipClaimingOpts { } export interface FetchResult { - searchAfter: any[]; docs: ConcreteTaskInstance[]; } @@ -152,8 +147,8 @@ export class TaskStore { return this.events$; } - private emitEvent = (event: TaskClaim) => { - this.events$.next(event); + private emitEvents = (events: TaskClaim[]) => { + events.forEach(event => this.events$.next(event)); }; /** @@ -180,16 +175,16 @@ export class TaskStore { } /** - * Fetches a paginatable list of scheduled tasks. + * Fetches a list of scheduled tasks with default sorting. * * @param opts - The query options used to filter tasks */ - public async fetch(opts: FetchOpts = {}): Promise { - const sort = paginatableSort(opts.sort); + public async fetch({ sort = [{ 'task.runAt': 'asc' }], ...opts }: SearchOpts = {}): Promise< + FetchResult + > { return this.search({ + ...opts, sort, - search_after: opts.searchAfter, - query: opts.query, }); } @@ -211,28 +206,30 @@ export class TaskStore { this.serializer.generateRawId(undefined, 'task', id) ); - const claimedTasks = await this.markAvailableTasksAsClaimed( + const numberOfTasksClaimed = await this.markAvailableTasksAsClaimed( claimOwnershipUntil, claimTasksByIdWithRawIds, size ); const docs = - claimedTasks > 0 ? await this.sweepForClaimedTasks(claimTasksByIdWithRawIds, size) : []; + numberOfTasksClaimed > 0 + ? await this.sweepForClaimedTasks(claimTasksByIdWithRawIds, size) + : []; // emit success/fail events for claimed tasks by id if (claimTasksById && claimTasksById.length) { - docs.map(doc => asTaskClaimEvent(doc.id, asOk(doc))).forEach(this.emitEvent); + this.emitEvents(docs.map(doc => asTaskClaimEvent(doc.id, asOk(doc)))); - difference( - claimTasksById, - docs.map(doc => doc.id) - ) - .map(id => asTaskClaimEvent(id, asErr(new Error(`failed to claim task '${id}'`)))) - .forEach(this.emitEvent); + this.emitEvents( + difference( + claimTasksById, + docs.map(doc => doc.id) + ).map(id => asTaskClaimEvent(id, asErr(new Error(`failed to claim task '${id}'`)))) + ); } return { - claimedTasks, + claimedTasks: numberOfTasksClaimed, docs, }; }; @@ -247,7 +244,7 @@ export class TaskStore { // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), // Either task has a schedule or the attempts < the maximum configured - shouldBeOneOf( + shouldBeOneOf( TaskWithSchedule, ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) @@ -255,36 +252,33 @@ export class TaskStore { ) ); - const { query, sort } = - claimTasksById && claimTasksById.length - ? { - query: shouldBeOneOf< - | ExistsBoolClause - | TermBoolClause - | RangeBoolClause - | BoolClause - >(queryForScheduledTasks, taskWithIDsAndRunnableStatus(claimTasksById)), - sort: sortByIdsThenByScheduling(claimTasksById), - } - : { - query: queryForScheduledTasks, - sort: SortByRunAtAndRetryAt, - }; - const { updated } = await this.updateByQuery( asUpdateByQuery({ - query, + query: matchesClauses( + mustBeAllOf( + claimTasksById && claimTasksById.length + ? asPinnedQuery(claimTasksById, queryForScheduledTasks) + : queryForScheduledTasks + ), + filterDownBy(InactiveTasks) + ), update: updateFields({ ownerId: this.taskManagerId, status: 'claiming', retryAt: claimOwnershipUntil, }), - sort, + sort: [ + // sort by score first, so the "pinned" Tasks are first + '_score', + // the nsort by other fields + SortByRunAtAndRetryAt, + ], }), { max_docs: size, } ); + return updated; } @@ -295,24 +289,14 @@ export class TaskStore { claimTasksById: OwnershipClaimingOpts['claimTasksById'], size: OwnershipClaimingOpts['size'] ): Promise { + const claimedTasksQuery = tasksClaimedByOwner(this.taskManagerId); const { docs } = await this.search({ - query: { - bool: { - must: [ - { - term: { - 'task.ownerId': this.taskManagerId, - }, - }, - { term: { 'task.status': 'claiming' } }, - ], - }, - }, - size, - sort: + query: claimTasksById && claimTasksById.length - ? sortByIdsThenByScheduling(claimTasksById) - : SortByRunAtAndRetryAt, + ? asPinnedQuery(claimTasksById, claimedTasksQuery) + : claimedTasksQuery, + size, + sort: SortByRunAtAndRetryAt, seq_no_primary_term: true, }); @@ -397,7 +381,6 @@ export class TaskStore { .map(doc => this.serializer.rawToSavedObject(doc)) .map(doc => omit(doc, 'namespace') as SavedObject) .map(savedObjectToConcreteTaskInstance), - searchAfter: (rawDocs.length && rawDocs[rawDocs.length - 1].sort) || [], }; } @@ -427,20 +410,6 @@ export class TaskStore { } } -function paginatableSort(sort: any[] = []) { - const sortById = { _id: 'desc' }; - - if (!sort.length) { - return [{ 'task.runAt': 'asc' }, sortById]; - } - - if (sort.find(({ _id }) => !!_id)) { - return sort; - } - - return [...sort, sortById]; -} - function taskInstanceToAttributes(doc: TaskInstance): SavedObjectAttributes { return { ...omit(doc, 'id', 'version'), diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js index 50fb9571c2687..e5b645367b8b7 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +const { DEFAULT_MAX_WORKERS } = require('../../../../plugins/task_manager/server/config.ts'); const { EventEmitter } = require('events'); import { initRoutes } from './init_routes'; @@ -16,6 +17,7 @@ const once = function(emitter, event) { export default function TaskTestingAPI(kibana) { const taskTestingEvents = new EventEmitter(); + taskTestingEvents.setMaxListeners(DEFAULT_MAX_WORKERS * 2); return new kibana.Plugin({ name: 'sampleTask', diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index c0dcd99525915..9e818f050c929 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -193,15 +193,44 @@ export function initRoutes(server, taskManager, legacyTaskManager, taskTestingEv }, }); + server.route({ + path: '/api/sample_tasks/task/{taskId}', + method: 'GET', + async handler(request) { + try { + return taskManager.fetch({ + query: { + bool: { + must: [ + { + ids: { + values: [`task:${request.params.taskId}`], + }, + }, + ], + }, + }, + }); + } catch (err) { + return err; + } + }, + }); + server.route({ path: '/api/sample_tasks', method: 'DELETE', async handler() { try { - const { docs: tasks } = await taskManager.fetch({ - query: taskManagerQuery, - }); - return Promise.all(tasks.map(task => taskManager.remove(task.id))); + let tasksFound = 0; + do { + const { docs: tasks } = await taskManager.fetch({ + query: taskManagerQuery, + }); + tasksFound = tasks.length; + await Promise.all(tasks.map(task => taskManager.remove(task.id))); + } while (tasksFound > 0); + return 'OK'; } catch (err) { return err; } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index 0b1c1cbb5af29..7ec0e9b5efa5b 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -13,6 +13,13 @@ const { task: { properties: taskManagerIndexMapping }, } = require('../../../../legacy/plugins/task_manager/server/mappings.json'); +const { + DEFAULT_MAX_WORKERS, + DEFAULT_POLL_INTERVAL, +} = require('../../../../plugins/task_manager/server/config.ts'); + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); + export default function({ getService }) { const es = getService('legacyEs'); const log = getService('log'); @@ -22,11 +29,12 @@ export default function({ getService }) { const supertest = supertestAsPromised(url.format(config.get('servers.kibana'))); describe('scheduling and running tasks', () => { - beforeEach(() => - supertest - .delete('/api/sample_tasks') - .set('kbn-xsrf', 'xxx') - .expect(200) + beforeEach( + async () => + await supertest + .delete('/api/sample_tasks') + .set('kbn-xsrf', 'xxx') + .expect(200) ); beforeEach(async () => { @@ -56,11 +64,19 @@ export default function({ getService }) { .then(response => response.body); } - function historyDocs() { + function currentTask(task) { + return supertest + .get(`/api/sample_tasks/task/${task}`) + .send({ task }) + .expect(200) + .then(response => response.body.docs[0]); + } + + function historyDocs(taskId) { return es .search({ index: testHistoryIndex, - q: 'type:task', + q: taskId ? `taskId:${taskId}` : 'type:task', }) .then(result => result.hits.hits); } @@ -223,7 +239,7 @@ export default function({ getService }) { }); await retry.try(async () => { - expect((await historyDocs()).length).to.eql(1); + expect((await historyDocs(originalTask.id)).length).to.eql(1); const [task] = (await currentTasks()).docs; expect(task.attempts).to.eql(0); @@ -318,6 +334,81 @@ export default function({ getService }) { }); }); + it('should prioritize tasks which are called using runNow', async () => { + const originalTask = await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: `30m` }, + params: {}, + }); + + await retry.try(async () => { + const docs = await historyDocs(originalTask.id); + expect(docs.length).to.eql(1); + + const task = await currentTask(originalTask.id); + + expect(task.state.count).to.eql(1); + + // ensure this task shouldnt run for another half hour + expectReschedule(Date.parse(originalTask.runAt), task, 30 * 60000); + }); + + const taskToBeReleased = await scheduleTask({ + taskType: 'sampleTask', + params: { waitForEvent: 'releaseSingleTask' }, + }); + + await retry.try(async () => { + // wait for taskToBeReleased to stall + expect((await historyDocs(taskToBeReleased.id)).length).to.eql(1); + }); + + // schedule multiple tasks that should force + // Task Manager to use up its worker capacity + // causing tasks to pile up + await Promise.all( + _.times(DEFAULT_MAX_WORKERS + _.random(1, DEFAULT_MAX_WORKERS), () => + scheduleTask({ + taskType: 'sampleTask', + params: { + waitForEvent: 'releaseTheOthers', + }, + }) + ) + ); + + // we need to ensure that TM has a chance to fill its queue with the stalling tasks + await delay(DEFAULT_POLL_INTERVAL); + + // call runNow for our task + const runNowResult = runTaskNow({ + id: originalTask.id, + }); + + // we need to ensure that TM has a chance to push the runNow task into the queue + // before we release the stalled task, so lets give it a chance + await delay(DEFAULT_POLL_INTERVAL); + + // and release only one slot in our worker queue + await releaseTasksWaitingForEventToComplete('releaseSingleTask'); + + expect(await runNowResult).to.eql({ id: originalTask.id }); + + await retry.try(async () => { + const task = await currentTask(originalTask.id); + expect(task.state.count).to.eql(2); + }); + + // drain tasks, othrwise they'll keep Task Manager stalled + await retry.try(async () => { + await releaseTasksWaitingForEventToComplete('releaseTheOthers'); + const tasks = (await currentTasks()).docs.filter( + task => task.params.originalParams.waitForEvent === 'releaseTheOthers' + ); + expect(tasks.length).to.eql(0); + }); + }); + it('should return a task run error result when running a task now fails', async () => { const originalTask = await scheduleTask({ taskType: 'sampleTask', @@ -329,10 +420,7 @@ export default function({ getService }) { const docs = await historyDocs(); expect(docs.filter(taskDoc => taskDoc._source.taskId === originalTask.id).length).to.eql(1); - const [task] = (await currentTasks()).docs.filter( - taskDoc => taskDoc.id === originalTask.id - ); - + const task = await currentTask(originalTask.id); expect(task.state.count).to.eql(1); // ensure this task shouldnt run for another half hour @@ -364,9 +452,7 @@ export default function({ getService }) { (await historyDocs()).filter(taskDoc => taskDoc._source.taskId === originalTask.id).length ).to.eql(2); - const [task] = (await currentTasks()).docs.filter( - taskDoc => taskDoc.id === originalTask.id - ); + const task = await currentTask(originalTask.id); expect(task.attempts).to.eql(1); }); }); From ea9351aaaa003bc4100ecf282791317a3dd80a2d Mon Sep 17 00:00:00 2001 From: MadameSheema Date: Thu, 16 Jan 2020 11:10:49 +0100 Subject: [PATCH 2/5] [SIEM] Improves data providers Cypress tests execution (#54462) * refactor * replaces 'clearTimeline' for 'createNewTimeline' * removes unused data-test-subj Co-authored-by: Elastic Machine --- .../timeline/data_providers.spec.ts | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/data_providers.spec.ts b/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/data_providers.spec.ts index 824e403185238..24c1974cf8343 100644 --- a/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/data_providers.spec.ts +++ b/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/data_providers.spec.ts @@ -9,7 +9,11 @@ import { TIMELINE_DROPPED_DATA_PROVIDERS, TIMELINE_DATA_PROVIDERS_EMPTY, } from '../../lib/timeline/selectors'; -import { dragFromAllHostsToTimeline, toggleTimelineVisibility } from '../../lib/timeline/helpers'; +import { + createNewTimeline, + dragFromAllHostsToTimeline, + toggleTimelineVisibility, +} from '../../lib/timeline/helpers'; import { ALL_HOSTS_WIDGET_DRAGGABLE_HOSTS } from '../../lib/hosts/selectors'; import { HOSTS_PAGE } from '../../lib/urls'; import { waitForAllHostsWidget } from '../../lib/hosts/helpers'; @@ -17,15 +21,20 @@ import { DEFAULT_TIMEOUT, loginAndWaitForPage } from '../../lib/util/helpers'; import { drag, dragWithoutDrop } from '../../lib/drag_n_drop/helpers'; describe('timeline data providers', () => { - beforeEach(() => { + before(() => { loginAndWaitForPage(HOSTS_PAGE); - }); - - it('renders the data provider of a host dragged from the All Hosts widget on the hosts page', () => { waitForAllHostsWidget(); + }); + beforeEach(() => { toggleTimelineVisibility(); + }); + + afterEach(() => { + createNewTimeline(); + }); + it('renders the data provider of a host dragged from the All Hosts widget on the hosts page', () => { dragFromAllHostsToTimeline(); cy.get(TIMELINE_DROPPED_DATA_PROVIDERS, { @@ -45,10 +54,6 @@ describe('timeline data providers', () => { }); it('sets the background to euiColorSuccess with a 10% alpha channel when the user starts dragging a host, but is not hovering over the data providers', () => { - waitForAllHostsWidget(); - - toggleTimelineVisibility(); - cy.get(ALL_HOSTS_WIDGET_DRAGGABLE_HOSTS) .first() .then(host => drag(host)); @@ -61,10 +66,6 @@ describe('timeline data providers', () => { }); it('sets the background to euiColorSuccess with a 20% alpha channel when the user starts dragging a host AND is hovering over the data providers', () => { - waitForAllHostsWidget(); - - toggleTimelineVisibility(); - cy.get(ALL_HOSTS_WIDGET_DRAGGABLE_HOSTS) .first() .then(host => drag(host)); @@ -81,10 +82,6 @@ describe('timeline data providers', () => { }); it('renders the dashed border color as euiColorSuccess when hovering over the data providers', () => { - waitForAllHostsWidget(); - - toggleTimelineVisibility(); - cy.get(ALL_HOSTS_WIDGET_DRAGGABLE_HOSTS) .first() .then(host => drag(host)); From c2f3c977eb688eb3d149ff4300a4fc05488d0eb0 Mon Sep 17 00:00:00 2001 From: Angela Chuang <6295984+angorayc@users.noreply.github.com> Date: Thu, 16 Jan 2020 18:46:22 +0800 Subject: [PATCH 3/5] [SIEM] Dns histogram enhancement (#54902) * update DNS histogram * fix indent * hide dropdown if only one option provided * update DNS histogram * fix types --- .../components/matrix_histogram/index.tsx | 23 +--- .../components/matrix_histogram/types.ts | 4 +- .../components/matrix_histogram/utils.ts | 2 +- .../matrix_histogram/index.gql_query.ts | 22 +++- .../containers/matrix_histogram/index.tsx | 10 +- .../containers/matrix_histogram/utils.ts | 52 ++------- .../containers/network_dns/index.gql_query.ts | 10 +- .../public/containers/network_dns/index.tsx | 10 +- .../siem/public/graphql/introspection.json | 108 ++++++++++++++++++ .../plugins/siem/public/graphql/types.ts | 65 ++++++++--- .../navigation/events_query_tab_body.tsx | 2 +- .../network/navigation/dns_query_tab_body.tsx | 108 +++++++++--------- .../siem/public/pages/network/translations.ts | 15 ++- .../overview/events_by_dataset/index.tsx | 2 +- .../siem/server/graphql/network/resolvers.ts | 15 ++- .../siem/server/graphql/network/schema.gql.ts | 12 ++ .../plugins/siem/server/graphql/types.ts | 67 +++++++++++ .../lib/network/elasticsearch_adapter.ts | 62 ++++++---- .../plugins/siem/server/lib/network/index.ts | 14 ++- .../lib/network/query_dns_histogram.dsl.ts | 83 ++++++++++++++ .../plugins/siem/server/lib/network/types.ts | 31 ++++- .../api_integration/apis/siem/network_dns.ts | 3 +- 22 files changed, 531 insertions(+), 189 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/network/query_dns_histogram.dsl.ts diff --git a/x-pack/legacy/plugins/siem/public/components/matrix_histogram/index.tsx b/x-pack/legacy/plugins/siem/public/components/matrix_histogram/index.tsx index 32dce0e2e5576..56ebbb06f3eb9 100644 --- a/x-pack/legacy/plugins/siem/public/components/matrix_histogram/index.tsx +++ b/x-pack/legacy/plugins/siem/public/components/matrix_histogram/index.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useState, useEffect, useCallback, useMemo } from 'react'; +import React, { useState, useEffect, useCallback } from 'react'; import { ScaleType } from '@elastic/charts'; import darkTheme from '@elastic/eui/dist/eui_theme_dark.json'; @@ -26,14 +26,12 @@ import { HistogramAggregation, MatrixHistogramQueryProps, } from './types'; -import { generateTablePaginationOptions } from '../paginated_table/helpers'; import { ChartSeriesData } from '../charts/common'; import { InspectButtonContainer } from '../inspect'; export const MatrixHistogramComponent: React.FC = ({ activePage, - dataKey, defaultStackByOption, endDate, @@ -45,12 +43,10 @@ export const MatrixHistogramComponent: React.FC - activePage != null && limit != null - ? generateTablePaginationOptions(activePage, limit) - : undefined; const { data, loading, inspect, totalCount, refetch = noop } = useQuery<{}, HistogramAggregation>( { @@ -118,16 +110,13 @@ export const MatrixHistogramComponent: React.FC getPagination(), [activePage, limit]), stackByField: selectedStackByOption.value, } ); @@ -179,7 +168,7 @@ export const MatrixHistogramComponent: React.FC - {stackByOptions && ( + {stackByOptions?.length > 1 && ( void; - isEventsType?: boolean; errorMessage: string; headerChildren?: React.ReactNode; hideHistogramIfEmpty?: boolean; + isAlertsHistogram?: boolean; + isAnomaliesHistogram?: boolean; + isAuthenticationsHistogram?: boolean; id: string; + isDnsHistogram?: boolean; + isEventsHistogram?: boolean; legendPosition?: Position; mapping?: MatrixHistogramMappingTypes; query: Maybe; setQuery: SetQuery; + showLegend?: boolean; sourceId: string; stackByOptions: MatrixHistogramOption[]; subtitle?: string | GetSubTitle; diff --git a/x-pack/legacy/plugins/siem/public/containers/matrix_histogram/utils.ts b/x-pack/legacy/plugins/siem/public/containers/matrix_histogram/utils.ts index 1fc1fedae9f88..9cda9d8f6115f 100644 --- a/x-pack/legacy/plugins/siem/public/containers/matrix_histogram/utils.ts +++ b/x-pack/legacy/plugins/siem/public/containers/matrix_histogram/utils.ts @@ -16,7 +16,7 @@ import { useUiSetting$ } from '../../lib/kibana'; import { createFilter } from '../helpers'; import { useApolloClient } from '../../utils/apollo_context'; import { inputsModel } from '../../store'; -import { GetMatrixHistogramQuery, GetNetworkDnsQuery } from '../../graphql/types'; +import { GetMatrixHistogramQuery } from '../../graphql/types'; export const useQuery = ({ dataKey, @@ -26,15 +26,12 @@ export const useQuery = ({ isAlertsHistogram = false, isAnomaliesHistogram = false, isAuthenticationsHistogram = false, - isEventsType = false, - isDNSHistogram, - isPtrIncluded, + isEventsHistogram = false, + isDnsHistogram = false, isInspected, query, stackByField, startDate, - sort, - pagination, }: MatrixHistogramQueryProps) => { const [defaultIndex] = useUiSetting$(DEFAULT_INDEX_KEY); const [, dispatchToaster] = useStateToaster(); @@ -45,21 +42,7 @@ export const useQuery = ({ const [totalCount, setTotalCount] = useState(-1); const apolloClient = useApolloClient(); - const isDNSQuery = ( - variable: Pick< - MatrixHistogramQueryProps, - 'isDNSHistogram' | 'isPtrIncluded' | 'sort' | 'pagination' - > - ): variable is GetNetworkDnsQuery.Variables => { - return ( - !!isDNSHistogram && - variable.isDNSHistogram !== undefined && - variable.isPtrIncluded !== undefined && - variable.sort !== undefined && - variable.pagination !== undefined - ); - }; - const basicVariables = { + const matrixHistogramVariables: GetMatrixHistogramQuery.Variables = { filterQuery: createFilter(filterQuery), sourceId: 'default', timerange: { @@ -70,20 +53,11 @@ export const useQuery = ({ defaultIndex, inspect: isInspected, stackByField, - }; - const dnsVariables = { - ...basicVariables, - isDNSHistogram, - isPtrIncluded, - sort, - pagination, - }; - const matrixHistogramVariables: GetMatrixHistogramQuery.Variables = { - ...basicVariables, isAlertsHistogram, isAnomaliesHistogram, isAuthenticationsHistogram, - isEventsType, + isDnsHistogram, + isEventsHistogram, }; useEffect(() => { @@ -92,16 +66,13 @@ export const useQuery = ({ const abortSignal = abortCtrl.signal; async function fetchData() { - if (!apolloClient || (pagination != null && pagination.querySize < 0)) return null; + if (!apolloClient) return null; setLoading(true); return apolloClient - .query< - GetMatrixHistogramQuery.Query | GetNetworkDnsQuery.Query, - GetMatrixHistogramQuery.Variables | GetNetworkDnsQuery.Variables - >({ + .query({ query, fetchPolicy: 'cache-first', - variables: isDNSQuery(dnsVariables) ? dnsVariables : matrixHistogramVariables, + variables: matrixHistogramVariables, context: { fetchOptions: { abortSignal, @@ -145,11 +116,8 @@ export const useQuery = ({ query, filterQuery, isInspected, - isDNSHistogram, + isDnsHistogram, stackByField, - sort, - isPtrIncluded, - pagination, startDate, endDate, ]); diff --git a/x-pack/legacy/plugins/siem/public/containers/network_dns/index.gql_query.ts b/x-pack/legacy/plugins/siem/public/containers/network_dns/index.gql_query.ts index 9d82705e9524b..a81d112fa4c50 100644 --- a/x-pack/legacy/plugins/siem/public/containers/network_dns/index.gql_query.ts +++ b/x-pack/legacy/plugins/siem/public/containers/network_dns/index.gql_query.ts @@ -11,7 +11,6 @@ export const networkDnsQuery = gql` $defaultIndex: [String!]! $filterQuery: String $inspect: Boolean! - $isDNSHistogram: Boolean! $isPtrIncluded: Boolean! $pagination: PaginationInputPaginated! $sort: NetworkDnsSortField! @@ -31,7 +30,7 @@ export const networkDnsQuery = gql` stackByField: $stackByField ) { totalCount - edges @skip(if: $isDNSHistogram) { + edges { node { _id dnsBytesIn @@ -44,7 +43,7 @@ export const networkDnsQuery = gql` value } } - pageInfo @skip(if: $isDNSHistogram) { + pageInfo { activePage fakeTotalCount showMorePagesIndicator @@ -53,11 +52,6 @@ export const networkDnsQuery = gql` dsl response } - histogram @include(if: $isDNSHistogram) { - x - y - g - } } } } diff --git a/x-pack/legacy/plugins/siem/public/containers/network_dns/index.tsx b/x-pack/legacy/plugins/siem/public/containers/network_dns/index.tsx index 5c5552edcc4ba..04c8783c30a0f 100644 --- a/x-pack/legacy/plugins/siem/public/containers/network_dns/index.tsx +++ b/x-pack/legacy/plugins/siem/public/containers/network_dns/index.tsx @@ -26,7 +26,7 @@ import { generateTablePaginationOptions } from '../../components/paginated_table import { createFilter, getDefaultFetchPolicy } from '../helpers'; import { QueryTemplatePaginated, QueryTemplatePaginatedProps } from '../query_template_paginated'; import { networkDnsQuery } from './index.gql_query'; -import { DEFAULT_TABLE_ACTIVE_PAGE } from '../../store/constants'; +import { DEFAULT_TABLE_ACTIVE_PAGE, DEFAULT_TABLE_LIMIT } from '../../store/constants'; import { MatrixHistogram } from '../../components/matrix_histogram'; import { MatrixHistogramOption, GetSubTitle } from '../../components/matrix_histogram/types'; import { UpdateDateRange } from '../../components/charts/common'; @@ -57,8 +57,7 @@ interface DnsHistogramOwnProps extends QueryTemplatePaginatedProps { dataKey: string | string[]; defaultStackByOption: MatrixHistogramOption; errorMessage: string; - isDNSHistogram?: boolean; - limit: number; + isDnsHistogram?: boolean; query: DocumentNode; scaleType: ScaleType; setQuery: SetQuery; @@ -105,7 +104,6 @@ export class NetworkDnsComponentQuery extends QueryTemplatePaginated< const variables: GetNetworkDnsQuery.Variables = { defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), filterQuery: createFilter(filterQuery), - isDNSHistogram: false, inspect: isInspected, isPtrIncluded, pagination: generateTablePaginationOptions(activePage, limit), @@ -186,12 +184,12 @@ const makeMapStateToProps = () => { const makeMapHistogramStateToProps = () => { const getNetworkDnsSelector = networkSelectors.dnsSelector(); const getQuery = inputsSelectors.globalQueryByIdSelector(); - const mapStateToProps = (state: State, { id = HISTOGRAM_ID, limit }: DnsHistogramOwnProps) => { + const mapStateToProps = (state: State, { id = HISTOGRAM_ID }: DnsHistogramOwnProps) => { const { isInspected } = getQuery(state, id); return { ...getNetworkDnsSelector(state), activePage: DEFAULT_TABLE_ACTIVE_PAGE, - limit, + limit: DEFAULT_TABLE_LIMIT, isInspected, id, }; diff --git a/x-pack/legacy/plugins/siem/public/graphql/introspection.json b/x-pack/legacy/plugins/siem/public/graphql/introspection.json index d73755fb92185..7b9842fa2c2bc 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/introspection.json +++ b/x-pack/legacy/plugins/siem/public/graphql/introspection.json @@ -1901,6 +1901,59 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "NetworkDnsHistogram", + "description": "", + "args": [ + { + "name": "filterQuery", + "description": "", + "type": { "kind": "SCALAR", "name": "String", "ofType": null }, + "defaultValue": null + }, + { + "name": "defaultIndex", + "description": "", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } + } + } + }, + "defaultValue": null + }, + { + "name": "timerange", + "description": "", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "INPUT_OBJECT", "name": "TimerangeInput", "ofType": null } + }, + "defaultValue": null + }, + { + "name": "stackByField", + "description": "", + "type": { "kind": "SCALAR", "name": "String", "ofType": null }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "OBJECT", "name": "NetworkDsOverTimeData", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "NetworkHttp", "description": "", @@ -8744,6 +8797,61 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "NetworkDsOverTimeData", + "description": "", + "fields": [ + { + "name": "inspect", + "description": "", + "args": [], + "type": { "kind": "OBJECT", "name": "Inspect", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "matrixHistogramData", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "MatrixOverTimeHistogramData", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalCount", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, { "kind": "INPUT_OBJECT", "name": "NetworkHttpSortField", diff --git a/x-pack/legacy/plugins/siem/public/graphql/types.ts b/x-pack/legacy/plugins/siem/public/graphql/types.ts index 73049e26f1581..b13e295a8e168 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/public/graphql/types.ts @@ -499,6 +499,8 @@ export interface Source { NetworkDns: NetworkDnsData; + NetworkDnsHistogram: NetworkDsOverTimeData; + NetworkHttp: NetworkHttpData; OverviewNetwork?: Maybe; @@ -1752,6 +1754,14 @@ export interface MatrixOverOrdinalHistogramData { g: string; } +export interface NetworkDsOverTimeData { + inspect?: Maybe; + + matrixHistogramData: MatrixOverTimeHistogramData[]; + + totalCount: number; +} + export interface NetworkHttpData { edges: NetworkHttpEdges[]; @@ -2430,6 +2440,15 @@ export interface NetworkDnsSourceArgs { defaultIndex: string[]; } +export interface NetworkDnsHistogramSourceArgs { + filterQuery?: Maybe; + + defaultIndex: string[]; + + timerange: TimerangeInput; + + stackByField?: Maybe; +} export interface NetworkHttpSourceArgs { id?: Maybe; @@ -3306,8 +3325,9 @@ export namespace GetMatrixHistogramQuery { isAlertsHistogram: boolean; isAnomaliesHistogram: boolean; isAuthenticationsHistogram: boolean; + isDnsHistogram: boolean; defaultIndex: string[]; - isEventsType: boolean; + isEventsHistogram: boolean; filterQuery?: Maybe; inspect: boolean; sourceId: string; @@ -3333,6 +3353,8 @@ export namespace GetMatrixHistogramQuery { AuthenticationsHistogram: AuthenticationsHistogram; EventsHistogram: EventsHistogram; + + NetworkDnsHistogram: NetworkDnsHistogram; }; export type AlertsHistogram = { @@ -3446,6 +3468,34 @@ export namespace GetMatrixHistogramQuery { response: string[]; }; + + export type NetworkDnsHistogram = { + __typename?: 'NetworkDsOverTimeData'; + + matrixHistogramData: ____MatrixHistogramData[]; + + totalCount: number; + + inspect: Maybe<____Inspect>; + }; + + export type ____MatrixHistogramData = { + __typename?: 'MatrixOverTimeHistogramData'; + + x: number; + + y: number; + + g: string; + }; + + export type ____Inspect = { + __typename?: 'Inspect'; + + dsl: string[]; + + response: string[]; + }; } export namespace GetNetworkDnsQuery { @@ -3453,7 +3503,6 @@ export namespace GetNetworkDnsQuery { defaultIndex: string[]; filterQuery?: Maybe; inspect: boolean; - isDNSHistogram: boolean; isPtrIncluded: boolean; pagination: PaginationInputPaginated; sort: NetworkDnsSortField; @@ -3486,8 +3535,6 @@ export namespace GetNetworkDnsQuery { pageInfo: PageInfo; inspect: Maybe; - - histogram: Maybe; }; export type Edges = { @@ -3537,16 +3584,6 @@ export namespace GetNetworkDnsQuery { response: string[]; }; - - export type Histogram = { - __typename?: 'MatrixOverOrdinalHistogramData'; - - x: string; - - y: number; - - g: string; - }; } export namespace GetNetworkHttpQuery { diff --git a/x-pack/legacy/plugins/siem/public/pages/hosts/navigation/events_query_tab_body.tsx b/x-pack/legacy/plugins/siem/public/pages/hosts/navigation/events_query_tab_body.tsx index df83ad056943a..a07cbc8484a1b 100644 --- a/x-pack/legacy/plugins/siem/public/pages/hosts/navigation/events_query_tab_body.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/hosts/navigation/events_query_tab_body.tsx @@ -52,7 +52,7 @@ export const EventsQueryTabBody = ({ defaultStackByOption={eventsStackByOptions[0]} deleteQuery={deleteQuery} endDate={endDate} - isEventsType={true} + isEventsHistogram={true} errorMessage={i18n.ERROR_FETCHING_EVENTS_DATA} filterQuery={filterQuery} query={MatrixHistogramGqlQuery} diff --git a/x-pack/legacy/plugins/siem/public/pages/network/navigation/dns_query_tab_body.tsx b/x-pack/legacy/plugins/siem/public/pages/network/navigation/dns_query_tab_body.tsx index 53d68e44f7843..35d6eb15416c9 100644 --- a/x-pack/legacy/plugins/siem/public/pages/network/navigation/dns_query_tab_body.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/network/navigation/dns_query_tab_body.tsx @@ -4,31 +4,27 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useEffect } from 'react'; +import React, { useEffect, useCallback } from 'react'; import { getOr } from 'lodash/fp'; import { EuiSpacer } from '@elastic/eui'; -import { ScaleType } from '@elastic/charts'; import { NetworkDnsTable } from '../../../components/page/network/network_dns_table'; -import { - NetworkDnsQuery, - NetworkDnsHistogramQuery, - HISTOGRAM_ID, -} from '../../../containers/network_dns'; +import { NetworkDnsQuery, HISTOGRAM_ID } from '../../../containers/network_dns'; import { manageQuery } from '../../../components/page/manage_query'; import { NetworkComponentQueryProps } from './types'; import { networkModel } from '../../../store'; + import { MatrixHistogramOption } from '../../../components/matrix_histogram/types'; -import { networkDnsQuery } from '../../../containers/network_dns/index.gql_query'; import * as i18n from '../translations'; -import { useFormatBytes } from '../../../components/formatted_bytes'; +import { MatrixHistogramGqlQuery } from '../../../containers/matrix_histogram/index.gql_query'; +import { MatrixHistogramContainer } from '../../../containers/matrix_histogram'; const NetworkDnsTableManage = manageQuery(NetworkDnsTable); const dnsStackByOptions: MatrixHistogramOption[] = [ { - text: i18n.NAVIGATION_DNS_STACK_BY_DOMAIN, + text: i18n.STACK_BY_DOMAIN, value: 'dns.question.registered_domain', }, ]; @@ -50,50 +46,52 @@ export const DnsQueryTabBody = ({ } }; }, [deleteQuery]); - const formatBytes = useFormatBytes(); + + const getTitle = useCallback( + (option: MatrixHistogramOption) => i18n.DOMAINS_COUNT_BY(option.text), + [] + ); return ( - - {({ - totalCount, - loading, - networkDns, - pageInfo, - loadPage, - id, - inspect, - isInspected, - refetch, - }) => ( - <> - - + <> + + + + {({ + totalCount, + loading, + networkDns, + pageInfo, + loadPage, + id, + inspect, + isInspected, + refetch, + }) => ( - - )} - + )} + + ); }; diff --git a/x-pack/legacy/plugins/siem/public/pages/network/translations.ts b/x-pack/legacy/plugins/siem/public/pages/network/translations.ts index 00adce9b7ad8a..6446cde26796e 100644 --- a/x-pack/legacy/plugins/siem/public/pages/network/translations.ts +++ b/x-pack/legacy/plugins/siem/public/pages/network/translations.ts @@ -22,12 +22,9 @@ export const NAVIGATION_DNS_TITLE = i18n.translate('xpack.siem.network.navigatio defaultMessage: 'DNS', }); -export const NAVIGATION_DNS_STACK_BY_DOMAIN = i18n.translate( - 'xpack.siem.hosts.navigation.dns.stackByDomain', - { - defaultMessage: 'domain', - } -); +export const STACK_BY_DOMAIN = i18n.translate('xpack.siem.hosts.dns.stackByDomain', { + defaultMessage: 'unique domains', +}); export const ERROR_FETCHING_DNS_DATA = i18n.translate( 'xpack.siem.hosts.navigation.dns.histogram.errorFetchingDnsData', @@ -54,3 +51,9 @@ export const NAVIGATION_ANOMALIES_TITLE = i18n.translate( export const NAVIGATION_ALERTS_TITLE = i18n.translate('xpack.siem.network.navigation.alertsTitle', { defaultMessage: 'Alerts', }); + +export const DOMAINS_COUNT_BY = (groupByField: string) => + i18n.translate('xpack.siem.network.dns.stackByUniqueSubdomain', { + values: { groupByField }, + defaultMessage: 'Top domains by {groupByField}', + }); diff --git a/x-pack/legacy/plugins/siem/public/pages/overview/events_by_dataset/index.tsx b/x-pack/legacy/plugins/siem/public/pages/overview/events_by_dataset/index.tsx index b4f945c802e56..52084c4bfc280 100644 --- a/x-pack/legacy/plugins/siem/public/pages/overview/events_by_dataset/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/overview/events_by_dataset/index.tsx @@ -108,7 +108,7 @@ export const EventsByDataset = React.memo( })} headerChildren={eventsCountViewEventsButton} id={ID} - isEventsType={true} + isEventsHistogram={true} legendPosition={'right'} query={MatrixHistogramGqlQuery} setQuery={setQuery} diff --git a/x-pack/legacy/plugins/siem/server/graphql/network/resolvers.ts b/x-pack/legacy/plugins/siem/server/graphql/network/resolvers.ts index 2dabd51c198f7..06d6b8c516d8b 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/network/resolvers.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/network/resolvers.ts @@ -7,7 +7,7 @@ import { SourceResolvers } from '../../graphql/types'; import { AppResolverOf, ChildResolverOf } from '../../lib/framework'; import { Network } from '../../lib/network'; -import { createOptionsPaginated } from '../../utils/build_query/create_options'; +import { createOptionsPaginated, createOptions } from '../../utils/build_query/create_options'; import { QuerySourceResolver } from '../sources/resolvers'; type QueryNetworkTopCountriesResolver = ChildResolverOf< @@ -30,6 +30,10 @@ type QueryDnsResolver = ChildResolverOf< QuerySourceResolver >; +type QueryDnsHistogramResolver = ChildResolverOf< + AppResolverOf, + QuerySourceResolver +>; export interface NetworkResolversDeps { network: Network; } @@ -42,6 +46,7 @@ export const createNetworkResolvers = ( NetworkTopCountries: QueryNetworkTopCountriesResolver; NetworkTopNFlow: QueryNetworkTopNFlowResolver; NetworkDns: QueryDnsResolver; + NetworkDnsHistogram: QueryDnsHistogramResolver; }; } => ({ Source: { @@ -76,9 +81,15 @@ export const createNetworkResolvers = ( ...createOptionsPaginated(source, args, info), networkDnsSortField: args.sort, isPtrIncluded: args.isPtrIncluded, - stackByField: args.stackByField, }; return libs.network.getNetworkDns(req, options); }, + async NetworkDnsHistogram(source, args, { req }, info) { + const options = { + ...createOptions(source, args, info), + stackByField: args.stackByField, + }; + return libs.network.getNetworkDnsHistogramData(req, options); + }, }, }); diff --git a/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts b/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts index a5bca68fb30b9..15e2d832a73c9 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts @@ -196,6 +196,12 @@ export const networkSchema = gql` inspect: Inspect } + type NetworkDsOverTimeData { + inspect: Inspect + matrixHistogramData: [MatrixOverTimeHistogramData!]! + totalCount: Float! + } + extend type Source { NetworkTopCountries( id: String @@ -227,6 +233,12 @@ export const networkSchema = gql` timerange: TimerangeInput! defaultIndex: [String!]! ): NetworkDnsData! + NetworkDnsHistogram( + filterQuery: String + defaultIndex: [String!]! + timerange: TimerangeInput! + stackByField: String + ): NetworkDsOverTimeData! NetworkHttp( id: String filterQuery: String diff --git a/x-pack/legacy/plugins/siem/server/graphql/types.ts b/x-pack/legacy/plugins/siem/server/graphql/types.ts index 48ca32874dda2..4a2119b6f7631 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/types.ts @@ -501,6 +501,8 @@ export interface Source { NetworkDns: NetworkDnsData; + NetworkDnsHistogram: NetworkDsOverTimeData; + NetworkHttp: NetworkHttpData; OverviewNetwork?: Maybe; @@ -1754,6 +1756,14 @@ export interface MatrixOverOrdinalHistogramData { g: string; } +export interface NetworkDsOverTimeData { + inspect?: Maybe; + + matrixHistogramData: MatrixOverTimeHistogramData[]; + + totalCount: number; +} + export interface NetworkHttpData { edges: NetworkHttpEdges[]; @@ -2432,6 +2442,15 @@ export interface NetworkDnsSourceArgs { defaultIndex: string[]; } +export interface NetworkDnsHistogramSourceArgs { + filterQuery?: Maybe; + + defaultIndex: string[]; + + timerange: TimerangeInput; + + stackByField?: Maybe; +} export interface NetworkHttpSourceArgs { id?: Maybe; @@ -2930,6 +2949,8 @@ export namespace SourceResolvers { NetworkDns?: NetworkDnsResolver; + NetworkDnsHistogram?: NetworkDnsHistogramResolver; + NetworkHttp?: NetworkHttpResolver; OverviewNetwork?: OverviewNetworkResolver, TypeParent, TContext>; @@ -3281,6 +3302,21 @@ export namespace SourceResolvers { defaultIndex: string[]; } + export type NetworkDnsHistogramResolver< + R = NetworkDsOverTimeData, + Parent = Source, + TContext = SiemContext + > = Resolver; + export interface NetworkDnsHistogramArgs { + filterQuery?: Maybe; + + defaultIndex: string[]; + + timerange: TimerangeInput; + + stackByField?: Maybe; + } + export type NetworkHttpResolver< R = NetworkHttpData, Parent = Source, @@ -7547,6 +7583,36 @@ export namespace MatrixOverOrdinalHistogramDataResolvers { > = Resolver; } +export namespace NetworkDsOverTimeDataResolvers { + export interface Resolvers { + inspect?: InspectResolver, TypeParent, TContext>; + + matrixHistogramData?: MatrixHistogramDataResolver< + MatrixOverTimeHistogramData[], + TypeParent, + TContext + >; + + totalCount?: TotalCountResolver; + } + + export type InspectResolver< + R = Maybe, + Parent = NetworkDsOverTimeData, + TContext = SiemContext + > = Resolver; + export type MatrixHistogramDataResolver< + R = MatrixOverTimeHistogramData[], + Parent = NetworkDsOverTimeData, + TContext = SiemContext + > = Resolver; + export type TotalCountResolver< + R = number, + Parent = NetworkDsOverTimeData, + TContext = SiemContext + > = Resolver; +} + export namespace NetworkHttpDataResolvers { export interface Resolvers { edges?: EdgesResolver; @@ -9227,6 +9293,7 @@ export type IResolvers = { NetworkDnsEdges?: NetworkDnsEdgesResolvers.Resolvers; NetworkDnsItem?: NetworkDnsItemResolvers.Resolvers; MatrixOverOrdinalHistogramData?: MatrixOverOrdinalHistogramDataResolvers.Resolvers; + NetworkDsOverTimeData?: NetworkDsOverTimeDataResolvers.Resolvers; NetworkHttpData?: NetworkHttpDataResolvers.Resolvers; NetworkHttpEdges?: NetworkHttpEdgesResolvers.Resolvers; NetworkHttpItem?: NetworkHttpItemResolvers.Resolvers; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts index 07b748024743c..4bd980fd2ff80 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts @@ -18,10 +18,16 @@ import { NetworkHttpData, NetworkHttpEdges, NetworkTopNFlowEdges, - MatrixOverOrdinalHistogramData, + NetworkDsOverTimeData, + MatrixOverTimeHistogramData, } from '../../graphql/types'; import { inspectStringifyObject } from '../../utils/build_query'; -import { DatabaseSearchResponse, FrameworkAdapter, FrameworkRequest } from '../framework'; +import { + DatabaseSearchResponse, + FrameworkAdapter, + FrameworkRequest, + MatrixHistogramRequestOptions, +} from '../framework'; import { TermAggregation } from '../types'; import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../common/constants'; @@ -32,6 +38,7 @@ import { NetworkTopNFlowRequestOptions, } from './index'; import { buildDnsQuery } from './query_dns.dsl'; +import { buildDnsHistogramQuery } from './query_dns_histogram.dsl'; import { buildTopNFlowQuery, getOppositeField } from './query_top_n_flow.dsl'; import { buildHttpQuery } from './query_http.dsl'; import { buildTopCountriesQuery } from './query_top_countries.dsl'; @@ -41,7 +48,9 @@ import { NetworkTopCountriesBuckets, NetworkHttpBuckets, NetworkTopNFlowBuckets, + DnsHistogramGroupData, } from './types'; +import { EventHit } from '../events/types'; export class ElasticsearchNetworkAdapter implements NetworkAdapter { constructor(private readonly framework: FrameworkAdapter) {} @@ -141,7 +150,6 @@ export class ElasticsearchNetworkAdapter implements NetworkAdapter { ); const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; const edges = networkDnsEdges.splice(cursorStart, querySize - cursorStart); - const histogram = getHistogramData(edges); const inspect = { dsl: [inspectStringifyObject(dsl)], response: [inspectStringifyObject(response)], @@ -156,7 +164,6 @@ export class ElasticsearchNetworkAdapter implements NetworkAdapter { showMorePagesIndicator, }, totalCount, - histogram, }; } @@ -195,29 +202,36 @@ export class ElasticsearchNetworkAdapter implements NetworkAdapter { totalCount, }; } + + public async getNetworkDnsHistogramData( + request: FrameworkRequest, + options: MatrixHistogramRequestOptions + ): Promise { + const dsl = buildDnsHistogramQuery(options); + const response = await this.framework.callWithRequest( + request, + 'search', + dsl + ); + const totalCount = getOr(0, 'hits.total.value', response); + const matrixHistogramData = getOr([], 'aggregations.NetworkDns.buckets', response); + const inspect = { + dsl: [inspectStringifyObject(dsl)], + response: [inspectStringifyObject(response)], + }; + return { + inspect, + matrixHistogramData: getHistogramData(matrixHistogramData), + totalCount, + }; + } } -const getHistogramData = ( - data: NetworkDnsEdges[] -): MatrixOverOrdinalHistogramData[] | undefined => { - if (!Array.isArray(data)) return undefined; +const getHistogramData = (data: DnsHistogramGroupData[]): MatrixOverTimeHistogramData[] => { return data.reduce( - (acc: MatrixOverOrdinalHistogramData[], { node: { dnsBytesOut, dnsBytesIn, _id } }) => { - if (_id != null && dnsBytesOut != null && dnsBytesIn != null) - return [ - ...acc, - { - x: _id, - y: dnsBytesOut, - g: 'DNS Bytes Out', - }, - { - x: _id, - y: dnsBytesIn, - g: 'DNS Bytes In', - }, - ]; - return acc; + (acc: MatrixOverTimeHistogramData[], { key: time, histogram: { buckets } }) => { + const temp = buckets.map(({ key, doc_count }) => ({ x: time, y: doc_count, g: key })); + return [...acc, ...temp]; }, [] ); diff --git a/x-pack/legacy/plugins/siem/server/lib/network/index.ts b/x-pack/legacy/plugins/siem/server/lib/network/index.ts index 42ce9f0726ddb..cbcd33b753d8a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/index.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/index.ts @@ -14,8 +14,13 @@ import { NetworkTopCountriesData, NetworkTopNFlowData, NetworkTopTablesSortField, + NetworkDsOverTimeData, } from '../../graphql/types'; -import { FrameworkRequest, RequestOptionsPaginated } from '../framework'; +import { + FrameworkRequest, + RequestOptionsPaginated, + MatrixHistogramRequestOptions, +} from '../framework'; export * from './elasticsearch_adapter'; import { NetworkAdapter } from './types'; @@ -68,6 +73,13 @@ export class Network { return this.adapter.getNetworkDns(req, options); } + public async getNetworkDnsHistogramData( + req: FrameworkRequest, + options: MatrixHistogramRequestOptions + ): Promise { + return this.adapter.getNetworkDnsHistogramData(req, options); + } + public async getNetworkHttp( req: FrameworkRequest, options: NetworkHttpRequestOptions diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_dns_histogram.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_dns_histogram.dsl.ts new file mode 100644 index 0000000000000..67457ab4840ac --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_dns_histogram.dsl.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createQueryFilterClauses, calculateTimeseriesInterval } from '../../utils/build_query'; +import { MatrixHistogramRequestOptions } from '../framework'; + +export const buildDnsHistogramQuery = ({ + filterQuery, + timerange: { from, to }, + defaultIndex, + sourceConfiguration: { + fields: { timestamp }, + }, + stackByField, +}: MatrixHistogramRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + [timestamp]: { + gte: from, + lte: to, + }, + }, + }, + ]; + + const getHistogramAggregation = () => { + const interval = calculateTimeseriesInterval(from, to); + const histogramTimestampField = '@timestamp'; + const dateHistogram = { + date_histogram: { + field: histogramTimestampField, + fixed_interval: `${interval}s`, + }, + }; + + return { + NetworkDns: { + ...dateHistogram, + aggs: { + histogram: { + terms: { + field: stackByField, + order: { + orderAgg: 'desc', + }, + size: 10, + }, + aggs: { + orderAgg: { + cardinality: { + field: 'dns.question.name', + }, + }, + }, + }, + }, + }, + }; + }; + + const dslQuery = { + index: defaultIndex, + allowNoIndices: true, + ignoreUnavailable: true, + body: { + aggregations: getHistogramAggregation(), + query: { + bool: { + filter, + }, + }, + size: 0, + track_total_hits: true, + }, + }; + + return dslQuery; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/types.ts b/x-pack/legacy/plugins/siem/server/lib/network/types.ts index 960fbe425b002..b5563f9a2fef1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/types.ts @@ -9,8 +9,13 @@ import { NetworkHttpData, NetworkTopCountriesData, NetworkTopNFlowData, + NetworkDsOverTimeData, } from '../../graphql/types'; -import { FrameworkRequest, RequestOptionsPaginated } from '../framework'; +import { + FrameworkRequest, + RequestOptionsPaginated, + MatrixHistogramRequestOptions, +} from '../framework'; import { TotalValue } from '../types'; import { NetworkDnsRequestOptions } from '.'; @@ -24,6 +29,10 @@ export interface NetworkAdapter { options: RequestOptionsPaginated ): Promise; getNetworkDns(req: FrameworkRequest, options: NetworkDnsRequestOptions): Promise; + getNetworkDnsHistogramData( + request: FrameworkRequest, + options: MatrixHistogramRequestOptions + ): Promise; getNetworkHttp(req: FrameworkRequest, options: RequestOptionsPaginated): Promise; } @@ -143,3 +152,23 @@ export interface NetworkHttpBuckets { buckets: GenericBuckets[]; }; } + +interface DnsHistogramSubBucket { + key: string; + doc_count: number; + orderAgg: { + value: number; + }; +} +interface DnsHistogramBucket { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; + buckets: DnsHistogramSubBucket[]; +} + +export interface DnsHistogramGroupData { + key: number; + doc_count: number; + key_as_string: string; + histogram: DnsHistogramBucket; +} diff --git a/x-pack/test/api_integration/apis/siem/network_dns.ts b/x-pack/test/api_integration/apis/siem/network_dns.ts index 5de7ea3a67087..13e98f09d072b 100644 --- a/x-pack/test/api_integration/apis/siem/network_dns.ts +++ b/x-pack/test/api_integration/apis/siem/network_dns.ts @@ -30,7 +30,6 @@ export default function({ getService }: FtrProviderContext) { query: networkDnsQuery, variables: { defaultIndex: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - isDNSHistogram: false, inspect: false, isPtrIncluded: false, pagination: { @@ -66,7 +65,7 @@ export default function({ getService }: FtrProviderContext) { query: networkDnsQuery, variables: { defaultIndex: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - isDNSHistogram: false, + isDnsHistogram: false, inspect: false, isPtrIncluded: false, pagination: { From 6d3c284d113f5f9005b360155cc701bf7a1c365c Mon Sep 17 00:00:00 2001 From: Shahzad Date: Thu, 16 Jan 2020 12:14:38 +0100 Subject: [PATCH 4/5] [Uptime] Fix/embedded map console warning (#54926) * update it * type * refresh app when it is out of focus * updated * unused code --- .../location_map/embeddables/embedded_map.tsx | 8 ++----- .../monitor_status_details.tsx | 24 ++++++++++++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/x-pack/legacy/plugins/uptime/public/components/functional/location_map/embeddables/embedded_map.tsx b/x-pack/legacy/plugins/uptime/public/components/functional/location_map/embeddables/embedded_map.tsx index 63a054c0c4889..98780d23c5a62 100644 --- a/x-pack/legacy/plugins/uptime/public/components/functional/location_map/embeddables/embedded_map.tsx +++ b/x-pack/legacy/plugins/uptime/public/components/functional/location_map/embeddables/embedded_map.tsx @@ -45,7 +45,7 @@ const EmbeddedPanel = styled.div` } `; -export const EmbeddedMap = ({ upPoints, downPoints }: EmbeddedMapProps) => { +export const EmbeddedMap = React.memo(({ upPoints, downPoints }: EmbeddedMapProps) => { const { colors } = useContext(UptimeThemeContext); const [embeddable, setEmbeddable] = useState(); const embeddableRoot: React.RefObject = useRef(null); @@ -55,10 +55,6 @@ export const EmbeddedMap = ({ upPoints, downPoints }: EmbeddedMapProps) => { id: uuid.v4(), filters: [], hidePanelTitles: true, - query: { - query: '', - language: 'kuery', - }, refreshConfig: { value: 0, pause: false, @@ -116,6 +112,6 @@ export const EmbeddedMap = ({ upPoints, downPoints }: EmbeddedMapProps) => {
); -}; +}); EmbeddedMap.displayName = 'EmbeddedMap'; diff --git a/x-pack/legacy/plugins/uptime/public/components/functional/monitor_status_details/monitor_status_details.tsx b/x-pack/legacy/plugins/uptime/public/components/functional/monitor_status_details/monitor_status_details.tsx index bb87497d335ef..63abb6fc4823c 100644 --- a/x-pack/legacy/plugins/uptime/public/components/functional/monitor_status_details/monitor_status_details.tsx +++ b/x-pack/legacy/plugins/uptime/public/components/functional/monitor_status_details/monitor_status_details.tsx @@ -4,10 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useEffect } from 'react'; +import React, { useContext, useEffect, useState } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; import { LocationMap } from '../location_map'; import { MonitorStatusBar } from './monitor_status_bar'; +import { UptimeRefreshContext } from '../../../contexts'; interface MonitorStatusBarProps { monitorId: string; @@ -29,6 +30,27 @@ export const MonitorStatusDetailsComponent = ({ useEffect(() => { loadMonitorLocations(monitorId); }, [loadMonitorLocations, monitorId, dateStart, dateEnd]); + const { refreshApp } = useContext(UptimeRefreshContext); + + const [isTabActive] = useState(document.visibilityState); + const onTabActive = () => { + if (document.visibilityState === 'visible' && isTabActive === 'hidden') { + refreshApp(); + } + }; + + // Refreshing application state after Tab becomes active to render latest map state + // If application renders in when tab is not in focus it gives some unexpected behaviors + // Where map is not visible on change + useEffect(() => { + document.addEventListener('visibilitychange', onTabActive); + return () => { + document.removeEventListener('visibilitychange', onTabActive); + }; + + // we want this effect to execute exactly once after the component mounts + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); return ( From 8a07023343c29b12e9a272836ada3db6b34249ea Mon Sep 17 00:00:00 2001 From: MadameSheema Date: Thu, 16 Jan 2020 12:31:46 +0100 Subject: [PATCH 5/5] [SIEM] Improves toggle column Cypress tests execution time (#54475) * refactor * replaces 'clearTimeline' for 'createNewTimeline' * fixes typecheck issue Co-authored-by: Elastic Machine --- .../smoke_tests/timeline/toggle_column.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/toggle_column.spec.ts b/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/toggle_column.spec.ts index 9a915b0e77d44..fbf75e8a854c6 100644 --- a/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/toggle_column.spec.ts +++ b/x-pack/legacy/plugins/siem/cypress/integration/smoke_tests/timeline/toggle_column.spec.ts @@ -6,15 +6,19 @@ import { drag, drop } from '../../lib/drag_n_drop/helpers'; import { populateTimeline } from '../../lib/fields_browser/helpers'; -import { toggleFirstTimelineEventDetails } from '../../lib/timeline/helpers'; +import { createNewTimeline, toggleFirstTimelineEventDetails } from '../../lib/timeline/helpers'; import { HOSTS_PAGE } from '../../lib/urls'; import { loginAndWaitForPage, DEFAULT_TIMEOUT } from '../../lib/util/helpers'; describe('toggle column in timeline', () => { - beforeEach(() => { + before(() => { loginAndWaitForPage(HOSTS_PAGE); }); + afterEach(() => { + createNewTimeline(); + }); + const timestampField = '@timestamp'; const idField = '_id';