From 637f39a4d270dd37c6c308d9bff4386d3a901a46 Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Tue, 2 Jul 2019 10:21:54 -0600 Subject: [PATCH 01/11] dns --- .../siem/public/graphql/introspection.json | 8 ------ .../plugins/siem/public/graphql/types.ts | 2 -- .../siem/server/graphql/network/schema.gql.ts | 1 - .../plugins/siem/server/graphql/types.ts | 9 ------- .../lib/network/elasticsearch_adapter.ts | 2 -- .../siem/server/lib/network/query_dns.dsl.ts | 25 ++++++------------- .../plugins/siem/server/lib/network/types.ts | 4 --- 7 files changed, 8 insertions(+), 43 deletions(-) diff --git a/x-pack/legacy/plugins/siem/public/graphql/introspection.json b/x-pack/legacy/plugins/siem/public/graphql/introspection.json index ef4582fec4043..547c35ce78b00 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/introspection.json +++ b/x-pack/legacy/plugins/siem/public/graphql/introspection.json @@ -7776,14 +7776,6 @@ "isDeprecated": false, "deprecationReason": null }, - { - "name": "timestamp", - "description": "", - "args": [], - "type": { "kind": "SCALAR", "name": "Date", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, { "name": "uniqueDomains", "description": "", diff --git a/x-pack/legacy/plugins/siem/public/graphql/types.ts b/x-pack/legacy/plugins/siem/public/graphql/types.ts index b3b4729c85312..2ca9b1d10840d 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/public/graphql/types.ts @@ -1194,8 +1194,6 @@ export interface NetworkDnsItem { queryCount?: number | null; - timestamp?: Date | null; - uniqueDomains?: number | null; } diff --git a/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts b/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts index 69fbdb82098a9..44711e542bed9 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts @@ -82,7 +82,6 @@ export const networkSchema = gql` dnsBytesOut: Float dnsName: String queryCount: Float - timestamp: Date uniqueDomains: Float } diff --git a/x-pack/legacy/plugins/siem/server/graphql/types.ts b/x-pack/legacy/plugins/siem/server/graphql/types.ts index abd6c91fd390c..00fdc640164f7 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/types.ts @@ -1223,8 +1223,6 @@ export interface NetworkDnsItem { queryCount?: number | null; - timestamp?: Date | null; - uniqueDomains?: number | null; } @@ -6337,8 +6335,6 @@ export namespace NetworkDnsItemResolvers { queryCount?: QueryCountResolver; - timestamp?: TimestampResolver; - uniqueDomains?: UniqueDomainsResolver; } @@ -6367,11 +6363,6 @@ export namespace NetworkDnsItemResolvers { Parent = NetworkDnsItem, Context = SiemContext > = Resolver; - export type TimestampResolver< - R = Date | null, - Parent = NetworkDnsItem, - Context = SiemContext - > = Resolver; export type UniqueDomainsResolver< R = number | null, Parent = NetworkDnsItem, diff --git a/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts index 8ef71df5fc1d3..a5cdd9a9d732f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts @@ -108,7 +108,6 @@ const formatTopNFlowEdges = ( buckets.map((bucket: NetworkTopNFlowBuckets) => ({ node: { _id: bucket.key, - timestamp: bucket.timestamp.value_as_string, [flowTarget]: { count: getOrNumber('ip_count.value', bucket), domain: bucket.domain.buckets.map(bucketDomain => bucketDomain.key), @@ -130,7 +129,6 @@ const formatDnsEdges = (buckets: NetworkDnsBuckets[]): NetworkDnsEdges[] => buckets.map((bucket: NetworkDnsBuckets) => ({ node: { _id: bucket.key, - timestamp: bucket.timestamp.value_as_string, dnsBytesIn: getOrNumber('dns_bytes_in.value', bucket), dnsBytesOut: getOrNumber('dns_bytes_out.value', bucket), dnsName: bucket.key, diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts index 8740a2d37563c..b4c5f6df4a982 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts @@ -46,9 +46,9 @@ const createIncludePTRFilter = (isPtrIncluded: boolean) => : { must_not: [ { - match_phrase: { + term: { 'dns.question.type': { - query: 'PTR', + value: 'PTR', }, }, }, @@ -56,19 +56,14 @@ const createIncludePTRFilter = (isPtrIncluded: boolean) => }; const getDnsFilter = () => ({ - must: [ - { - match_phrase: { - 'network.protocol': { - query: 'dns', - }, - }, + term: { + 'network.protocol': { + value: 'dns', }, - ], + }, }); export const buildDnsQuery = ({ - fields, filterQuery, isPtrIncluded, networkDnsSortField, @@ -88,6 +83,7 @@ export const buildDnsQuery = ({ lte: to, }, }, + ...getDnsFilter(), }, ]; @@ -122,18 +118,12 @@ export const buildDnsQuery = ({ field: 'destination.bytes', }, }, - timestamp: { - max: { - field: '@timestamp', - }, - }, }, }, }, query: { bool: { filter, - ...getDnsFilter(), ...createIncludePTRFilter(isPtrIncluded), }, }, @@ -141,5 +131,6 @@ export const buildDnsQuery = ({ size: 0, track_total_hits: false, }; + console.log(JSON.stringify(dslQuery.body)); return dslQuery; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/types.ts b/x-pack/legacy/plugins/siem/server/lib/network/types.ts index 4e1b92f55042f..07b38b718dd9b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/types.ts @@ -62,10 +62,6 @@ export interface NetworkTopNFlowData extends SearchHit { export interface NetworkDnsBuckets { key: string; doc_count: number; - timestamp: { - value: number; - value_as_string: string; - }; unique_domains: { value: number; }; From 4891d3ee859a47c43da8645582da64a03e9fe7cf Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Tue, 2 Jul 2019 12:08:17 -0600 Subject: [PATCH 02/11] remove all must --- .../siem/server/lib/events/query.dsl.ts | 13 +++-- .../lib/ip_details/query_domains.dsl.ts | 46 +++++++++--------- .../siem/server/lib/network/query_dns.dsl.ts | 4 +- .../lib/network/query_top_n_flow.dsl.ts | 47 +++++++++---------- .../lib/uncommon_processes/query.dsl.ts | 8 ++-- x-pack/test/api_integration/apis/index.js | 36 +++++++------- .../test/api_integration/apis/siem/index.js | 30 ++++++------ 7 files changed, 87 insertions(+), 97 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts index 58c3bd09c7375..26ee667efd44b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts @@ -34,8 +34,6 @@ export const buildQuery = (options: RequestOptions) => { return []; }; - const filter = [...filterClause, ...getTimerangeFilter(options.timerange)]; - const agg = options.fields.includes('kpiEventType') ? { count_event_type: { @@ -54,6 +52,8 @@ export const buildQuery = (options: RequestOptions) => { ? [{ match_all: {} }, { exists: { field: 'event.action' } }] : [{ match_all: {} }]; + const filter = [...filterClause, ...getTimerangeFilter(options.timerange), ...queryMust]; + const getSortField = (sortField: SortField) => { if (sortField.sortFieldId) { const field: string = @@ -69,7 +69,7 @@ export const buildQuery = (options: RequestOptions) => { const sort: SortRequest = getSortField(options.sortField!); - const queryDsl = { + const dslQuery = { allowNoIndices: true, index: defaultIndex, ignoreUnavailable: true, @@ -77,7 +77,6 @@ export const buildQuery = (options: RequestOptions) => { aggregations: agg, query: { bool: { - must: queryMust, filter, }, }, @@ -90,15 +89,15 @@ export const buildQuery = (options: RequestOptions) => { if (cursor && tiebreaker) { return { - ...queryDsl, + ...dslQuery, body: { - ...queryDsl.body, + ...dslQuery.body, search_after: [cursor, tiebreaker], }, }; } - return queryDsl; + return dslQuery; }; export const buildDetailsQuery = (indexName: string, id: string) => ({ diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts index ca9df87fe5caf..34ea6d3f1fb4a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts @@ -96,38 +96,34 @@ const getBiDirectionalFilter = (flowDirection: FlowDirection, flowTarget: FlowTa flowDirection === FlowDirection.biDirectional && [FlowTarget.source, FlowTarget.destination].includes(flowTarget) ) { - return { - must: [ - { - exists: { - field: 'source.bytes', - }, + return [ + { + exists: { + field: 'source.bytes', }, - { - exists: { - field: 'destination.bytes', - }, + }, + { + exists: { + field: 'destination.bytes', }, - ], - }; + }, + ]; } else if ( flowDirection === FlowDirection.biDirectional && [FlowTarget.client, FlowTarget.server].includes(flowTarget) ) { - return { - must: [ - { - exists: { - field: 'client.bytes', - }, + return [ + { + exists: { + field: 'client.bytes', }, - { - exists: { - field: 'server.bytes', - }, + }, + { + exists: { + field: 'server.bytes', }, - ], - }; + }, + ]; } return []; }; @@ -149,6 +145,7 @@ export const buildDomainsQuery = ({ ...createQueryFilterClauses(filterQuery), { range: { [timestamp]: { gte: from, lte: to } } }, { term: { [`${flowTarget}.ip`]: ip } }, + ...getBiDirectionalFilter(flowDirection, flowTarget), ]; const dslQuery = { @@ -163,7 +160,6 @@ export const buildDomainsQuery = ({ bool: { filter, ...getUniDirectionalFilter(flowDirection), - ...getBiDirectionalFilter(flowDirection, flowTarget), }, }, size: 0, diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts index b4c5f6df4a982..814b05fe404c5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts @@ -83,8 +83,8 @@ export const buildDnsQuery = ({ lte: to, }, }, - ...getDnsFilter(), }, + getDnsFilter(), ]; const dslQuery = { @@ -131,6 +131,6 @@ export const buildDnsQuery = ({ size: 0, track_total_hits: false, }; - console.log(JSON.stringify(dslQuery.body)); + return dslQuery; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts index c0dec11237de1..9f89ad2c8527c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts @@ -33,38 +33,34 @@ const getBiDirectionalFilter = (flowDirection: FlowDirection, flowTarget: FlowTa flowDirection === FlowDirection.biDirectional && [FlowTarget.source, FlowTarget.destination].includes(flowTarget) ) { - return { - must: [ - { - exists: { - field: 'source.bytes', - }, + return [ + { + exists: { + field: 'source.bytes', }, - { - exists: { - field: 'destination.bytes', - }, + }, + { + exists: { + field: 'destination.bytes', }, - ], - }; + }, + ]; } else if ( flowDirection === FlowDirection.biDirectional && [FlowTarget.client, FlowTarget.server].includes(flowTarget) ) { - return { - must: [ - { - exists: { - field: 'client.bytes', - }, + return [ + { + exists: { + field: 'client.bytes', }, - { - exists: { - field: 'server.bytes', - }, + }, + { + exists: { + field: 'server.bytes', }, - ], - }; + }, + ]; } return []; }; @@ -78,7 +74,6 @@ const getCountAgg = (flowTarget: FlowTarget) => ({ }); export const buildTopNFlowQuery = ({ - fields, filterQuery, flowDirection, networkTopNFlowSort, @@ -93,6 +88,7 @@ export const buildTopNFlowQuery = ({ const filter = [ ...createQueryFilterClauses(filterQuery), { range: { [timestamp]: { gte: from, lte: to } } }, + ...getBiDirectionalFilter(flowDirection, flowTarget), ]; const dslQuery = { @@ -109,7 +105,6 @@ export const buildTopNFlowQuery = ({ bool: { filter, ...getUniDirectionalFilter(flowDirection), - ...getBiDirectionalFilter(flowDirection, flowTarget), }, }, }, diff --git a/x-pack/legacy/plugins/siem/server/lib/uncommon_processes/query.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/uncommon_processes/query.dsl.ts index 060e907144b5f..2f83c839d80cb 100644 --- a/x-pack/legacy/plugins/siem/server/lib/uncommon_processes/query.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/uncommon_processes/query.dsl.ts @@ -98,7 +98,7 @@ export const buildQuery = ({ should: [ { bool: { - must: [ + filter: [ { term: { 'agent.type': 'auditbeat', @@ -119,7 +119,7 @@ export const buildQuery = ({ }, { bool: { - must: [ + filter: [ { term: { 'agent.type': 'auditbeat', @@ -140,7 +140,7 @@ export const buildQuery = ({ }, { bool: { - must: [ + filter: [ { term: { 'agent.type': 'winlogbeat', @@ -156,7 +156,7 @@ export const buildQuery = ({ }, { bool: { - must: [ + filter: [ { term: { 'winlog.event_id': 1, diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 4dd380a15c63a..6343f34dc2444 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -8,24 +8,24 @@ export default function ({ loadTestFile }) { describe('apis', function () { this.tags('ciGroup6'); - loadTestFile(require.resolve('./actions')); - loadTestFile(require.resolve('./alerting')); - loadTestFile(require.resolve('./es')); - loadTestFile(require.resolve('./security')); - loadTestFile(require.resolve('./monitoring')); - loadTestFile(require.resolve('./xpack_main')); - loadTestFile(require.resolve('./telemetry')); - loadTestFile(require.resolve('./logstash')); - loadTestFile(require.resolve('./kibana')); - loadTestFile(require.resolve('./infra')); - loadTestFile(require.resolve('./beats')); - loadTestFile(require.resolve('./console')); - loadTestFile(require.resolve('./management')); - loadTestFile(require.resolve('./uptime')); - loadTestFile(require.resolve('./maps')); - loadTestFile(require.resolve('./apm')); + // loadTestFile(require.resolve('./actions')); + // loadTestFile(require.resolve('./alerting')); + // loadTestFile(require.resolve('./es')); + // loadTestFile(require.resolve('./security')); + // loadTestFile(require.resolve('./monitoring')); + // loadTestFile(require.resolve('./xpack_main')); + // loadTestFile(require.resolve('./telemetry')); + // loadTestFile(require.resolve('./logstash')); + // loadTestFile(require.resolve('./kibana')); + // loadTestFile(require.resolve('./infra')); + // loadTestFile(require.resolve('./beats')); + // loadTestFile(require.resolve('./console')); + // loadTestFile(require.resolve('./management')); + // loadTestFile(require.resolve('./uptime')); + // loadTestFile(require.resolve('./maps')); + // loadTestFile(require.resolve('./apm')); loadTestFile(require.resolve('./siem')); - loadTestFile(require.resolve('./code')); - loadTestFile(require.resolve('./short_urls')); + // loadTestFile(require.resolve('./code')); + // loadTestFile(require.resolve('./short_urls')); }); } diff --git a/x-pack/test/api_integration/apis/siem/index.js b/x-pack/test/api_integration/apis/siem/index.js index 2ddfec261aa02..50fa18f5e989c 100644 --- a/x-pack/test/api_integration/apis/siem/index.js +++ b/x-pack/test/api_integration/apis/siem/index.js @@ -6,25 +6,25 @@ export default function ({ loadTestFile }) { describe('Siem GraphQL Endpoints', () => { - loadTestFile(require.resolve('./authentications')); + // loadTestFile(require.resolve('./authentications')); loadTestFile(require.resolve('./domains')); loadTestFile(require.resolve('./events')); - loadTestFile(require.resolve('./hosts')); - loadTestFile(require.resolve('./kpi_network')); - loadTestFile(require.resolve('./kpi_hosts')); + // loadTestFile(require.resolve('./hosts')); + // loadTestFile(require.resolve('./kpi_network')); + // loadTestFile(require.resolve('./kpi_hosts')); loadTestFile(require.resolve('./network_dns')); loadTestFile(require.resolve('./network_top_n_flow')); - loadTestFile(require.resolve('./overview_host')); - loadTestFile(require.resolve('./saved_objects/notes')); - loadTestFile(require.resolve('./saved_objects/pinned_events')); - loadTestFile(require.resolve('./saved_objects/timeline')); - loadTestFile(require.resolve('./sources')); - loadTestFile(require.resolve('./overview_network')); - loadTestFile(require.resolve('./timeline')); - loadTestFile(require.resolve('./timeline_details')); + // loadTestFile(require.resolve('./overview_host')); + // loadTestFile(require.resolve('./saved_objects/notes')); + // loadTestFile(require.resolve('./saved_objects/pinned_events')); + // loadTestFile(require.resolve('./saved_objects/timeline')); + // loadTestFile(require.resolve('./sources')); + // loadTestFile(require.resolve('./overview_network')); + // loadTestFile(require.resolve('./timeline')); + // loadTestFile(require.resolve('./timeline_details')); loadTestFile(require.resolve('./uncommon_processes')); - loadTestFile(require.resolve('./users')); - loadTestFile(require.resolve('./tls')); - loadTestFile(require.resolve('./feature_controls')); + // loadTestFile(require.resolve('./users')); + // loadTestFile(require.resolve('./tls')); + // loadTestFile(require.resolve('./feature_controls')); }); } From ab8fec7ce7b5528edb59829038cbeee02b65fcaa Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Tue, 2 Jul 2019 12:19:06 -0600 Subject: [PATCH 03/11] track_total_hits can be false --- .../lib/ip_details/query_domains.dsl.ts | 2 +- .../query_last_first_seen_domain.dsl.ts | 2 +- .../lib/ip_details/query_overview.dsl.ts | 2 +- .../server/lib/ip_details/query_tls.dsl.ts | 2 +- .../query_unique_private_ips.dsl.ts | 2 +- .../siem/server/lib/overview/query.dsl.ts | 4 +-- .../test/api_integration/apis/siem/index.js | 30 +++++++++---------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts index 34ea6d3f1fb4a..b5f8a6cbc144c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts @@ -163,7 +163,7 @@ export const buildDomainsQuery = ({ }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, }; diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts index 1ce7791bab1bc..c5e0aeb99eeaa 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts @@ -28,7 +28,7 @@ export const buildFirstLastSeenDomainQuery = ({ }, query: { bool: { filter } }, size: 0, - track_total_hits: true, + track_total_hits: false, }, }; diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts index 78f70c7cfbde4..d3ce7cf1ae82a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts @@ -121,7 +121,7 @@ export const buildOverviewQuery = ({ defaultIndex, ip }: IpOverviewRequestOption }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, }; diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts index 55af687aa7a76..179a5b9042ab2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts @@ -86,7 +86,7 @@ export const buildTlsQuery = ({ }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, }; diff --git a/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_private_ips.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_private_ips.dsl.ts index 58d694643bac2..f12ab2a3072ae 100644 --- a/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_private_ips.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_private_ips.dsl.ts @@ -99,7 +99,7 @@ export const buildUniquePrvateIpQuery = ({ }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, ]; diff --git a/x-pack/legacy/plugins/siem/server/lib/overview/query.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/overview/query.dsl.ts index e83e3908b0da8..c9e903ec6c9d9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/overview/query.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/overview/query.dsl.ts @@ -98,7 +98,7 @@ export const buildOverviewNetworkQuery = ({ }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, }; @@ -203,7 +203,7 @@ export const buildOverviewHostQuery = ({ }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, }; diff --git a/x-pack/test/api_integration/apis/siem/index.js b/x-pack/test/api_integration/apis/siem/index.js index 50fa18f5e989c..2ddfec261aa02 100644 --- a/x-pack/test/api_integration/apis/siem/index.js +++ b/x-pack/test/api_integration/apis/siem/index.js @@ -6,25 +6,25 @@ export default function ({ loadTestFile }) { describe('Siem GraphQL Endpoints', () => { - // loadTestFile(require.resolve('./authentications')); + loadTestFile(require.resolve('./authentications')); loadTestFile(require.resolve('./domains')); loadTestFile(require.resolve('./events')); - // loadTestFile(require.resolve('./hosts')); - // loadTestFile(require.resolve('./kpi_network')); - // loadTestFile(require.resolve('./kpi_hosts')); + loadTestFile(require.resolve('./hosts')); + loadTestFile(require.resolve('./kpi_network')); + loadTestFile(require.resolve('./kpi_hosts')); loadTestFile(require.resolve('./network_dns')); loadTestFile(require.resolve('./network_top_n_flow')); - // loadTestFile(require.resolve('./overview_host')); - // loadTestFile(require.resolve('./saved_objects/notes')); - // loadTestFile(require.resolve('./saved_objects/pinned_events')); - // loadTestFile(require.resolve('./saved_objects/timeline')); - // loadTestFile(require.resolve('./sources')); - // loadTestFile(require.resolve('./overview_network')); - // loadTestFile(require.resolve('./timeline')); - // loadTestFile(require.resolve('./timeline_details')); + loadTestFile(require.resolve('./overview_host')); + loadTestFile(require.resolve('./saved_objects/notes')); + loadTestFile(require.resolve('./saved_objects/pinned_events')); + loadTestFile(require.resolve('./saved_objects/timeline')); + loadTestFile(require.resolve('./sources')); + loadTestFile(require.resolve('./overview_network')); + loadTestFile(require.resolve('./timeline')); + loadTestFile(require.resolve('./timeline_details')); loadTestFile(require.resolve('./uncommon_processes')); - // loadTestFile(require.resolve('./users')); - // loadTestFile(require.resolve('./tls')); - // loadTestFile(require.resolve('./feature_controls')); + loadTestFile(require.resolve('./users')); + loadTestFile(require.resolve('./tls')); + loadTestFile(require.resolve('./feature_controls')); }); } From 2ce90cf4208af7121c2002e915bbf18cc6a4b5a3 Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Tue, 2 Jul 2019 12:52:53 -0600 Subject: [PATCH 04/11] rm firstSeen agg from ip details --- .../server/lib/ip_details/elasticsearch_adapter.ts | 1 - .../legacy/plugins/siem/server/lib/ip_details/mock.ts | 6 ------ .../siem/server/lib/ip_details/query_domains.dsl.ts | 5 ----- x-pack/legacy/plugins/siem/server/lib/network/mock.ts | 10 ---------- 4 files changed, 22 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts index 1c78af9118edf..62bd3c6f543bc 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts @@ -235,7 +235,6 @@ export const formatDomainsEdges = ( [flowTarget]: { uniqueIpCount: getOrNumber('uniqueIpCount.value', bucket), domainName: bucket.key, - firstSeen: get('firstSeen.value_as_string', bucket), lastSeen: get('lastSeen.value_as_string', bucket), }, network: { diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts index b3429be2efbd2..3b7d4a44b34c7 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts @@ -268,10 +268,6 @@ export const mockDomainsResponseBuckets: DomainsBuckets[] = [ bytes: { value: 974964465, }, - firstSeen: { - value: 1554146873000, - value_as_string: '2019-04-01T19:27:53.000Z', - }, packets: { value: 16946245, }, @@ -302,7 +298,6 @@ export const mockFormattedSource: DomainsEdges[] = [ }, source: { domainName: 'example.com', - firstSeen: '2019-04-01T19:27:53.000Z', lastSeen: '2019-04-10T18:28:39.000Z', uniqueIpCount: 805, }, @@ -317,7 +312,6 @@ export const mockFormattedDestination: DomainsEdges[] = [ _id: 'example.com', destination: { domainName: 'example.com', - firstSeen: '2019-04-01T19:27:53.000Z', lastSeen: '2019-04-10T18:28:39.000Z', uniqueIpCount: 805, }, diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts index b5f8a6cbc144c..fb6f02bb866e5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_domains.dsl.ts @@ -37,11 +37,6 @@ const getAggs = ( }, }, aggs: { - firstSeen: { - min: { - field: '@timestamp', - }, - }, lastSeen: { max: { field: '@timestamp', diff --git a/x-pack/legacy/plugins/siem/server/lib/network/mock.ts b/x-pack/legacy/plugins/siem/server/lib/network/mock.ts index 5f2ab5b35ba3e..40f837072d4ff 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/mock.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/mock.ts @@ -485,7 +485,6 @@ export const mockResult = { }, node: { _id: '1.1.1.1', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 11276023407, packets: 1025631, @@ -505,7 +504,6 @@ export const mockResult = { }, node: { _id: '2.2.2.2', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 5469323342, packets: 2811441, @@ -525,7 +523,6 @@ export const mockResult = { }, node: { _id: '3.3.3.3', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 3807671322, packets: 4494034, @@ -545,7 +542,6 @@ export const mockResult = { }, node: { _id: '4.4.4.4', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 166517626, packets: 3194782, @@ -565,7 +561,6 @@ export const mockResult = { }, node: { _id: '5.5.5.5', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 104785026, packets: 1838597, @@ -585,7 +580,6 @@ export const mockResult = { }, node: { _id: '6.6.6.6', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 28804250, packets: 482982, @@ -605,7 +599,6 @@ export const mockResult = { }, node: { _id: '7.7.7.7', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 23032363, packets: 400623, @@ -625,7 +618,6 @@ export const mockResult = { }, node: { _id: '8.8.8.8', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 21424889, packets: 344357, @@ -645,7 +637,6 @@ export const mockResult = { }, node: { _id: '9.9.9.9', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 19205000, packets: 355663, @@ -665,7 +656,6 @@ export const mockResult = { }, node: { _id: '10.10.10.10', - timestamp: '2019-02-18T21:14:24.000Z', network: { bytes: 11407633, packets: 199360, From 64febd1b7a64081af37576cce919023c88472a55 Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Tue, 2 Jul 2019 14:42:23 -0600 Subject: [PATCH 05/11] address unused query parts --- .../lib/ip_details/elasticsearch_adapter.ts | 3 +- .../lib/ip_details/query_overview.dsl.ts | 26 ++++------- .../plugins/siem/server/lib/network/mock.ts | 44 ------------------- .../lib/network/query_top_n_flow.dsl.ts | 24 ---------- .../plugins/siem/server/lib/network/types.ts | 4 -- 5 files changed, 9 insertions(+), 92 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts index 62bd3c6f543bc..630b8321f8db6 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts @@ -204,10 +204,9 @@ export const getIpOverviewAgg = (type: string, overviewHit: OverviewHit | {}) => export const getIpOverviewHostAgg = (overviewHostHit: OverviewHostHit | {}) => { const hostFields: HostEcsFields | null = getOr( null, - `host.results.hits.hits[0]._source.host`, + `results.hits.hits[0]._source.host`, overviewHostHit ); - return { host: { ...hostFields, diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts index d3ce7cf1ae82a..0107e358010a4 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_overview.dsl.ts @@ -79,24 +79,15 @@ const getHostAggs = (ip: string) => { }, }, aggs: { - host: { - filter: { - exists: { - field: 'host', - }, - }, - aggs: { - results: { - top_hits: { - size: 1, - _source: ['host'], - sort: [ - { - '@timestamp': 'desc', - }, - ], + results: { + top_hits: { + size: 1, + _source: ['host'], + sort: [ + { + '@timestamp': 'desc', }, - }, + ], }, }, }, @@ -124,6 +115,5 @@ export const buildOverviewQuery = ({ defaultIndex, ip }: IpOverviewRequestOption track_total_hits: false, }, }; - return dslQuery; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/mock.ts b/x-pack/legacy/plugins/siem/server/lib/network/mock.ts index 40f837072d4ff..1a704224743f2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/mock.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/mock.ts @@ -149,10 +149,6 @@ export const mockResponse = { ip_count: { value: 1, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -182,10 +178,6 @@ export const mockResponse = { ip_count: { value: 2, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -212,10 +204,6 @@ export const mockResponse = { ip_count: { value: 5, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -245,10 +233,6 @@ export const mockResponse = { ip_count: { value: 1, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -275,10 +259,6 @@ export const mockResponse = { ip_count: { value: 3, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -305,10 +285,6 @@ export const mockResponse = { ip_count: { value: 2, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { doc_count_error_upper_bound: 0, sum_other_doc_count: 31, @@ -337,10 +313,6 @@ export const mockResponse = { ip_count: { value: 2, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, @@ -369,10 +341,6 @@ export const mockResponse = { ip_count: { value: 2, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -399,10 +367,6 @@ export const mockResponse = { ip_count: { value: 2, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -429,10 +393,6 @@ export const mockResponse = { ip_count: { value: 2, }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, domain: { buckets: [ { @@ -453,10 +413,6 @@ export const mockResponse = { }, ], }, - timestamp: { - value: 155052446412, - value_as_string: '2019-02-18T21:14:24.000Z', - }, ip_count: { value: 2, }, diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts index 9f89ad2c8527c..48dd423179191 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts @@ -148,13 +148,6 @@ const getUniDirectionAggs = ( timestamp: 'desc', }, }, - aggs: { - timestamp: { - max: { - field: '@timestamp', - }, - }, - }, }, ip_count: { cardinality: { @@ -168,11 +161,6 @@ const getUniDirectionAggs = ( field: 'network.packets', }, }, - timestamp: { - max: { - field: '@timestamp', - }, - }, }, }, } @@ -212,13 +200,6 @@ const getBiDirectionAggs = ( timestamp: 'desc', }, }, - aggs: { - timestamp: { - max: { - field: '@timestamp', - }, - }, - }, }, ip_count: { cardinality: { @@ -230,11 +211,6 @@ const getBiDirectionAggs = ( field: `${flowTarget}.packets`, }, }, - timestamp: { - max: { - field: '@timestamp', - }, - }, }, }, } diff --git a/x-pack/legacy/plugins/siem/server/lib/network/types.ts b/x-pack/legacy/plugins/siem/server/lib/network/types.ts index 07b38b718dd9b..ca2800a58f576 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/types.ts @@ -24,10 +24,6 @@ export interface DirectionBuckets { export interface NetworkTopNFlowBuckets { key: string; - timestamp: { - value: number; - value_as_string: string; - }; bytes: { value: number; }; From da01a2972b092e1a8bb22c98d2a732e3225919d9 Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Wed, 3 Jul 2019 09:46:23 -0600 Subject: [PATCH 06/11] tests pass. 17 and 18 still to do --- .../siem/public/graphql/introspection.json | 8 --- .../plugins/siem/public/graphql/types.ts | 2 - .../siem/server/graphql/network/schema.gql.ts | 1 - .../plugins/siem/server/graphql/types.ts | 9 --- .../siem/server/lib/events/query.dsl.ts | 6 +- .../siem/server/lib/ip_details/mock.ts | 67 +++++++++---------- .../server/lib/ip_details/query_tls.dsl.ts | 2 +- .../siem/server/lib/ip_details/types.ts | 16 +---- .../plugins/siem/server/lib/kpi_hosts/mock.ts | 1 + .../lib/kpi_hosts/query_authentication.dsl.ts | 6 +- .../lib/network/query_top_n_flow.dsl.ts | 6 -- 11 files changed, 42 insertions(+), 82 deletions(-) diff --git a/x-pack/legacy/plugins/siem/public/graphql/introspection.json b/x-pack/legacy/plugins/siem/public/graphql/introspection.json index 547c35ce78b00..7135874b737fe 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/introspection.json +++ b/x-pack/legacy/plugins/siem/public/graphql/introspection.json @@ -7421,14 +7421,6 @@ "isDeprecated": false, "deprecationReason": null }, - { - "name": "timestamp", - "description": "", - "args": [], - "type": { "kind": "SCALAR", "name": "Date", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, { "name": "source", "description": "", diff --git a/x-pack/legacy/plugins/siem/public/graphql/types.ts b/x-pack/legacy/plugins/siem/public/graphql/types.ts index 2ca9b1d10840d..dd5aa69d6ddc3 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/public/graphql/types.ts @@ -1138,8 +1138,6 @@ export interface NetworkTopNFlowEdges { export interface NetworkTopNFlowItem { _id?: string | null; - timestamp?: Date | null; - source?: TopNFlowItem | null; destination?: TopNFlowItem | null; diff --git a/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts b/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts index 44711e542bed9..0bae96c202f4f 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts @@ -44,7 +44,6 @@ export const networkSchema = gql` type NetworkTopNFlowItem { _id: String - timestamp: Date source: TopNFlowItem destination: TopNFlowItem client: TopNFlowItem diff --git a/x-pack/legacy/plugins/siem/server/graphql/types.ts b/x-pack/legacy/plugins/siem/server/graphql/types.ts index 00fdc640164f7..2a146725d594c 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/types.ts @@ -1167,8 +1167,6 @@ export interface NetworkTopNFlowEdges { export interface NetworkTopNFlowItem { _id?: string | null; - timestamp?: Date | null; - source?: TopNFlowItem | null; destination?: TopNFlowItem | null; @@ -6169,8 +6167,6 @@ export namespace NetworkTopNFlowItemResolvers { export interface Resolvers { _id?: IdResolver; - timestamp?: TimestampResolver; - source?: SourceResolver; destination?: DestinationResolver; @@ -6187,11 +6183,6 @@ export namespace NetworkTopNFlowItemResolvers { Parent = NetworkTopNFlowItem, Context = SiemContext > = Resolver; - export type TimestampResolver< - R = Date | null, - Parent = NetworkTopNFlowItem, - Context = SiemContext - > = Resolver; export type SourceResolver< R = TopNFlowItem | null, Parent = NetworkTopNFlowItem, diff --git a/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts index 26ee667efd44b..8b8636d292d29 100644 --- a/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts @@ -48,11 +48,7 @@ export const buildQuery = (options: RequestOptions) => { } : {}; - const queryMust = options.fields.includes('kpiEventType') - ? [{ match_all: {} }, { exists: { field: 'event.action' } }] - : [{ match_all: {} }]; - - const filter = [...filterClause, ...getTimerangeFilter(options.timerange), ...queryMust]; + const filter = [...filterClause, ...getTimerangeFilter(options.timerange), { match_all: {} }]; const getSortField = (sortField: SortField) => { if (sortField.sortFieldId) { diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts index 3b7d4a44b34c7..51d74bb4e20ba 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/mock.ts @@ -128,44 +128,41 @@ export const responseAggs: IpOverviewHit = { }, host: { doc_count: 1588091, - host: { - doc_count: 1588091, - results: { - hits: { - total: { - value: 1588091, - relation: 'eq', - }, - max_score: null, - hits: [ - { - _index: 'filebeat-8.0.0-2019.05.20-000004', - _type: '_doc', - _id: 'NU9dD2sB9v5HJNSHMMRc', - _score: null, - _source: { - host: { - hostname: 'suricata-iowa', - os: { - kernel: '4.15.0-1032-gcp', - codename: 'bionic', - name: 'Ubuntu', - family: 'debian', - version: '18.04.2 LTS (Bionic Beaver)', - platform: 'ubuntu', - }, - ip: ['10.128.0.4', 'fe80::4001:aff:fe80:4'], - containerized: false, - name: 'suricata-iowa', - id: 'be1f3d767896212736b880e846876dcb', - mac: ['42:01:0a:80:00:04'], - architecture: 'x86_64', + results: { + hits: { + total: { + value: 1588091, + relation: 'eq', + }, + max_score: null, + hits: [ + { + _index: 'filebeat-8.0.0-2019.05.20-000004', + _type: '_doc', + _id: 'NU9dD2sB9v5HJNSHMMRc', + _score: null, + _source: { + host: { + hostname: 'suricata-iowa', + os: { + kernel: '4.15.0-1032-gcp', + codename: 'bionic', + name: 'Ubuntu', + family: 'debian', + version: '18.04.2 LTS (Bionic Beaver)', + platform: 'ubuntu', }, + ip: ['10.128.0.4', 'fe80::4001:aff:fe80:4'], + containerized: false, + name: 'suricata-iowa', + id: 'be1f3d767896212736b880e846876dcb', + mac: ['42:01:0a:80:00:04'], + architecture: 'x86_64', }, - sort: [1559330892000], }, - ], - }, + sort: [1559330892000], + }, + ], }, }, }, diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts index 179a5b9042ab2..6456f4b21f56e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts @@ -69,7 +69,7 @@ export const buildTlsQuery = ({ ...createQueryFilterClauses(filterQuery), { range: { [timestamp]: { gte: from, lte: to } } }, { term: { [`${flowTarget}.ip`]: ip } }, - { term: { 'event.dataset': 'tls' } }, + { term: { 'network.protocol': 'tls' } }, ]; const dslQuery = { diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts index e6e5e7c1caa67..b33591cd2e0b0 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts @@ -68,25 +68,13 @@ export interface OverviewHit { }; } -export interface OverviewHostHit { - took?: number; - timed_out?: boolean; - _scroll_id?: string; - _shards?: ShardsResponse; - timeout?: number; - hits?: { - total: number; - hits: Hit[]; - }; - doc_count: number; - host: ResultHit; -} +export type OverviewHostHit = ResultHit; export interface IpOverviewHit { aggregations: { destination?: OverviewHit; source?: OverviewHit; - host: OverviewHostHit; + host: ResultHit; }; _shards: { total: number; diff --git a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts index 4e9dcd1b7ecdc..19f0719ad1cb6 100644 --- a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts +++ b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts @@ -520,6 +520,7 @@ const mockAuthFilter = { should: [ { match: { 'event.type': 'authentication_success' } }, { match: { 'event.type': 'authentication_failure' } }, + { match: { 'event.category': 'authentication' } }, ], minimum_should_match: 1, }, diff --git a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts index 7234ea24625f0..887444ecb1eaf 100644 --- a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts @@ -20,6 +20,11 @@ const getAuthQueryFilter = () => [ 'event.type': 'authentication_failure', }, }, + { + match: { + 'event.category': 'authentication', + }, + }, ], minimum_should_match: 1, }, @@ -109,6 +114,5 @@ export const buildAuthQuery = ({ track_total_hits: false, }, ]; - return dslQuery; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts index 48dd423179191..d1c646ef19892 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts @@ -144,9 +144,6 @@ const getUniDirectionAggs = ( domain: { terms: { field: `${flowTarget}.domain`, - order: { - timestamp: 'desc', - }, }, }, ip_count: { @@ -196,9 +193,6 @@ const getBiDirectionAggs = ( domain: { terms: { field: `${flowTarget}.domain`, - order: { - timestamp: 'desc', - }, }, }, ip_count: { From 97a767b227aec228de9068482619395354a09d19 Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Wed, 3 Jul 2019 10:12:55 -0600 Subject: [PATCH 07/11] finish 17 and 18, clean up --- .../first_last_seen_domain/index.test.tsx | 188 ------------------ .../network/first_last_seen_domain/index.tsx | 78 -------- .../first_last_seen.gql_query.ts | 30 --- .../domains/first_last_seen_domain/index.ts | 88 -------- .../domains/first_last_seen_domain/mock.ts | 54 ----- .../siem/public/graphql/introspection.json | 141 ------------- .../plugins/siem/public/graphql/types.ts | 59 ------ .../siem/server/graphql/events/schema.gql.ts | 6 - .../server/graphql/ip_details/resolvers.ts | 17 -- .../server/graphql/ip_details/schema.gql.ts | 25 +-- .../plugins/siem/server/graphql/types.ts | 91 --------- .../lib/events/elasticsearch_adapter.ts | 10 +- .../siem/server/lib/events/query.dsl.ts | 15 -- .../lib/ip_details/elasticsearch_adapter.ts | 23 +-- .../siem/server/lib/ip_details/index.ts | 10 +- .../query_last_first_seen_domain.dsl.ts | 36 ---- .../siem/server/lib/ip_details/types.ts | 33 +-- .../lib/network/query_top_n_flow.dsl.ts | 20 ++ x-pack/test/api_integration/apis/index.js | 36 ++-- 19 files changed, 43 insertions(+), 917 deletions(-) delete mode 100644 x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.test.tsx delete mode 100644 x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.tsx delete mode 100644 x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/first_last_seen.gql_query.ts delete mode 100644 x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/index.ts delete mode 100644 x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/mock.ts delete mode 100644 x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts diff --git a/x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.test.tsx b/x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.test.tsx deleted file mode 100644 index 0cb6a3e2c7101..0000000000000 --- a/x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.test.tsx +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { cloneDeep } from 'lodash/fp'; -import * as React from 'react'; -import { MockedProvider } from 'react-apollo/test-utils'; -import { render } from 'react-testing-library'; - -import { mockFirstLastSeenDomainQuery } from '../../../../containers/domains/first_last_seen_domain/mock'; -import { FlowTarget } from '../../../../graphql/types'; -import { wait } from '../../../../lib/helpers'; -import { TestProviders } from '../../../../mock'; -import '../../../../mock/ui_settings'; - -import { FirstLastSeenDomain } from './index'; - -describe('FirstLastSeen Component', async () => { - // this is just a little hack to silence a warning that we'll get until react - // fixes this: https://github.com/facebook/react/pull/14853 - // For us that mean we need to upgrade to 16.9.0 - // and we will be able to do that when we are in master - // eslint-disable-next-line no-console - const originalError = console.error; - beforeAll(() => { - // eslint-disable-next-line no-console - console.error = (...args: string[]) => { - if (/Warning.*not wrapped in act/.test(args[0])) { - return; - } - originalError.call(console, ...args); - }; - }); - - afterAll(() => { - // eslint-disable-next-line no-console - console.error = originalError; - }); - - const ip = '10.10.10.10'; - const domainName = 'example.com'; - const firstSeen = 'Apr 8, 2019 @ 16:09:40.692'; - const lastSeen = 'Apr 8, 2019 @ 18:35:45.064'; - - test('Loading', async () => { - const { container } = render( - - - - - - ); - expect(container.innerHTML).toBe( - '' - ); - }); - - test('First Seen', async () => { - const { container } = render( - - - - - - ); - - await wait(); - - expect(container.innerHTML).toBe( - `
${firstSeen}
` - ); - }); - - test('Last Seen', async () => { - const { container } = render( - - - - - - ); - await wait(); - expect(container.innerHTML).toBe( - `
${lastSeen}
` - ); - }); - - test('First Seen is empty but not Last Seen', async () => { - const badDateTime = cloneDeep(mockFirstLastSeenDomainQuery); - badDateTime[0].result.data!.source.DomainFirstLastSeen.firstSeen = null; - const { container } = render( - - - - - - ); - - await wait(); - - expect(container.innerHTML).toBe( - `
${lastSeen}
` - ); - }); - - test('Last Seen is empty but not First Seen', async () => { - const badDateTime = cloneDeep(mockFirstLastSeenDomainQuery); - badDateTime[0].result.data!.source.DomainFirstLastSeen.lastSeen = null; - const { container } = render( - - - - - - ); - - await wait(); - - expect(container.innerHTML).toBe( - `
${firstSeen}
` - ); - }); - - test('First Seen With a bad date time string', async () => { - const badDateTime = cloneDeep(mockFirstLastSeenDomainQuery); - badDateTime[0].result.data!.source.DomainFirstLastSeen.firstSeen = 'something-invalid'; - const { container } = render( - - - - - - ); - await wait(); - expect(container.textContent).toBe('something-invalid'); - }); - - test('Last Seen With a bad date time string', async () => { - const badDateTime = cloneDeep(mockFirstLastSeenDomainQuery); - badDateTime[0].result.data!.source.DomainFirstLastSeen.lastSeen = 'something-invalid'; - const { container } = render( - - - - - - ); - await wait(); - expect(container.textContent).toBe('something-invalid'); - }); -}); diff --git a/x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.tsx b/x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.tsx deleted file mode 100644 index 0a5713d002c7f..0000000000000 --- a/x-pack/legacy/plugins/siem/public/components/page/network/first_last_seen_domain/index.tsx +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { EuiIcon, EuiLoadingSpinner, EuiText, EuiToolTip } from '@elastic/eui'; -import moment from 'moment'; -import React from 'react'; -import { ApolloConsumer } from 'react-apollo'; -import { pure } from 'recompose'; - -import { useFirstLastSeenDomainQuery } from '../../../../containers/domains/first_last_seen_domain'; -import { FlowTarget } from '../../../../graphql/types'; -import { getEmptyTagValue } from '../../../empty_value'; -import { PreferenceFormattedDate } from '../../../formatted_date'; -import { LocalizedDateTooltip } from '../../../localized_date_tooltip'; - -export type FirstLastSeenType = 'first-seen' | 'last-seen'; - -export interface FirstLastSeenProps { - ip: string; - domainName: string; - flowTarget: FlowTarget; - type: FirstLastSeenType; -} - -export const FirstLastSeenDomain = pure( - ({ ip, domainName, flowTarget, type }) => { - return ( - - {client => { - const { loading, firstSeen, lastSeen, errorMessage } = useFirstLastSeenDomainQuery( - ip, - domainName, - flowTarget, - 'default', - client - ); - - if (errorMessage != null) { - return ( - - - - ); - } - const valueSeen = type === 'first-seen' ? firstSeen : lastSeen; - return ( - <> - {loading && } - {!loading && valueSeen != null && new Date(valueSeen).toString() === 'Invalid Date' - ? valueSeen - : !loading && - valueSeen != null && ( - - - - - - )} - {!loading && valueSeen == null && getEmptyTagValue()} - - ); - }} - - ); - } -); diff --git a/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/first_last_seen.gql_query.ts b/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/first_last_seen.gql_query.ts deleted file mode 100644 index 73eeffd4bcf6a..0000000000000 --- a/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/first_last_seen.gql_query.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import gql from 'graphql-tag'; - -export const DomainFirstLastSeenGqlQuery = gql` - query GetDomainFirstLastSeenQuery( - $sourceId: ID! - $ip: String! - $domainName: String! - $flowTarget: FlowTarget! - $defaultIndex: [String!]! - ) { - source(id: $sourceId) { - id - DomainFirstLastSeen( - ip: $ip - domainName: $domainName - flowTarget: $flowTarget - defaultIndex: $defaultIndex - ) { - firstSeen - lastSeen - } - } - } -`; diff --git a/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/index.ts b/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/index.ts deleted file mode 100644 index bd216e926ddd7..0000000000000 --- a/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/index.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import ApolloClient from 'apollo-client'; -import { get } from 'lodash/fp'; -import React, { useEffect, useState } from 'react'; - -import chrome from 'ui/chrome'; -import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; -import { FlowTarget, GetDomainFirstLastSeenQuery } from '../../../graphql/types'; -import { inputsModel } from '../../../store'; -import { QueryTemplateProps } from '../../query_template'; - -import { DomainFirstLastSeenGqlQuery } from './first_last_seen.gql_query'; - -export interface DomainFirstLastSeenArgs { - id: string; - errorMessage: string; - firstSeen: Date; - lastSeen: Date; - loading: boolean; - refetch: inputsModel.Refetch; -} - -export interface OwnProps extends QueryTemplateProps { - children: (args: DomainFirstLastSeenArgs) => React.ReactNode; - ip: string; - domainName: string; - flowTarget: FlowTarget; -} - -export function useFirstLastSeenDomainQuery( - ip: string, - domainName: string, - flowTarget: FlowTarget, - sourceId: string, - apolloClient: ApolloClient -) { - const [loading, updateLoading] = useState(false); - const [firstSeen, updateFirstSeen] = useState(null); - const [lastSeen, updateLastSeen] = useState(null); - const [errorMessage, updateErrorMessage] = useState(null); - - async function fetchDomainFirstLastSeen() { - updateLoading(true); - return apolloClient - .query({ - query: DomainFirstLastSeenGqlQuery, - fetchPolicy: 'cache-first', - variables: { - sourceId, - ip, - domainName, - flowTarget, - defaultIndex: chrome.getUiSettingsClient().get(DEFAULT_INDEX_KEY), - }, - }) - .then( - result => { - updateLoading(false); - updateFirstSeen(get('data.source.DomainFirstLastSeen.firstSeen', result)); - updateLastSeen(get('data.source.DomainFirstLastSeen.lastSeen', result)); - updateErrorMessage(null); - return result; - }, - error => { - updateLoading(false); - updateErrorMessage(error.message); - return error; - } - ); - } - - useEffect(() => { - try { - fetchDomainFirstLastSeen(); - } catch (err) { - updateFirstSeen(null); - updateLastSeen(null); - updateErrorMessage(err.toString()); - } - }, []); - - return { firstSeen, lastSeen, loading, errorMessage }; -} diff --git a/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/mock.ts b/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/mock.ts deleted file mode 100644 index 2bb06ed8d8efb..0000000000000 --- a/x-pack/legacy/plugins/siem/public/containers/domains/first_last_seen_domain/mock.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { FlowTarget, GetDomainFirstLastSeenQuery } from '../../../graphql/types'; - -import { DomainFirstLastSeenGqlQuery } from './first_last_seen.gql_query'; - -interface MockFirstLastSeenDomainQuery { - request: { - query: GetDomainFirstLastSeenQuery.Query; - variables: GetDomainFirstLastSeenQuery.Variables; - }; - result: { - data?: { - source: { - id: string; - DomainFirstLastSeen: { - firstSeen: string | null; - lastSeen: string | null; - }; - }; - }; - errors?: [{ message: string }]; - }; -} - -export const mockFirstLastSeenDomainQuery: MockFirstLastSeenDomainQuery[] = [ - { - request: { - query: DomainFirstLastSeenGqlQuery, - variables: { - sourceId: 'default', - ip: '10.10.10.10', - domainName: 'example.com', - flowTarget: FlowTarget.source, - defaultIndex: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - }, - }, - result: { - data: { - source: { - id: 'default', - DomainFirstLastSeen: { - firstSeen: '2019-04-08T16:09:40.692Z', - lastSeen: '2019-04-08T18:35:45.064Z', - }, - }, - }, - }, - }, -]; diff --git a/x-pack/legacy/plugins/siem/public/graphql/introspection.json b/x-pack/legacy/plugins/siem/public/graphql/introspection.json index 7135874b737fe..5ca2a1b5c1224 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/introspection.json +++ b/x-pack/legacy/plugins/siem/public/graphql/introspection.json @@ -1288,73 +1288,6 @@ "isDeprecated": false, "deprecationReason": null }, - { - "name": "DomainFirstLastSeen", - "description": "", - "args": [ - { - "name": "id", - "description": "", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - }, - { - "name": "ip", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "domainName", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "flowTarget", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "ENUM", "name": "FlowTarget", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "defaultIndex", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "FirstLastSeenDomain", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, { "name": "Tls", "description": "", @@ -3089,22 +3022,6 @@ "name": "EventsData", "description": "", "fields": [ - { - "name": "kpiEventType", - "description": "", - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "KpiItem", "ofType": null } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, { "name": "edges", "description": "", @@ -3155,37 +3072,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "KpiItem", - "description": "", - "fields": [ - { - "name": "value", - "description": "", - "args": [], - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "count", - "description": "", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, { "kind": "OBJECT", "name": "EcsEdges", @@ -6398,33 +6284,6 @@ ], "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "FirstLastSeenDomain", - "description": "", - "fields": [ - { - "name": "firstSeen", - "description": "", - "args": [], - "type": { "kind": "SCALAR", "name": "Date", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "lastSeen", - "description": "", - "args": [], - "type": { "kind": "SCALAR", "name": "Date", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, { "kind": "INPUT_OBJECT", "name": "TlsSortField", diff --git a/x-pack/legacy/plugins/siem/public/graphql/types.ts b/x-pack/legacy/plugins/siem/public/graphql/types.ts index dd5aa69d6ddc3..1eb78887a2f93 100644 --- a/x-pack/legacy/plugins/siem/public/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/public/graphql/types.ts @@ -126,8 +126,6 @@ export interface Source { Domains: DomainsData; - DomainFirstLastSeen: FirstLastSeenDomain; - Tls: TlsData; Users: UsersData; @@ -326,8 +324,6 @@ export interface PageInfo { } export interface EventsData { - kpiEventType?: KpiItem[] | null; - edges: EcsEdges[]; totalCount: number; @@ -335,12 +331,6 @@ export interface EventsData { pageInfo: PageInfo; } -export interface KpiItem { - value?: string | null; - - count: number; -} - export interface EcsEdges { node: Ecs; @@ -981,12 +971,6 @@ export interface DomainsNetworkField { direction?: NetworkDirectionEcs[] | null; } -export interface FirstLastSeenDomain { - firstSeen?: Date | null; - - lastSeen?: Date | null; -} - export interface TlsData { edges: TlsEdges[]; @@ -1828,17 +1812,6 @@ export interface DomainsSourceArgs { defaultIndex: string[]; } -export interface DomainFirstLastSeenSourceArgs { - id?: string | null; - - ip: string; - - domainName: string; - - flowTarget: FlowTarget; - - defaultIndex: string[]; -} export interface TlsSourceArgs { filterQuery?: string | null; @@ -2234,38 +2207,6 @@ export namespace GetAuthenticationsQuery { }; } -export namespace GetDomainFirstLastSeenQuery { - export type Variables = { - sourceId: string; - ip: string; - domainName: string; - flowTarget: FlowTarget; - defaultIndex: string[]; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'Source'; - - id: string; - - DomainFirstLastSeen: DomainFirstLastSeen; - }; - - export type DomainFirstLastSeen = { - __typename?: 'FirstLastSeenDomain'; - - firstSeen?: Date | null; - - lastSeen?: Date | null; - }; -} - export namespace GetDomainsQuery { export type Variables = { sourceId: string; diff --git a/x-pack/legacy/plugins/siem/server/graphql/events/schema.gql.ts b/x-pack/legacy/plugins/siem/server/graphql/events/schema.gql.ts index 87a42ede1433b..031488ff341db 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/events/schema.gql.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/events/schema.gql.ts @@ -9,13 +9,7 @@ import gql from 'graphql-tag'; export const eventsSchema = gql` scalar EsValue - type KpiItem { - value: String - count: Float! - } - type EventsData { - kpiEventType: [KpiItem!] edges: [EcsEdges!]! totalCount: Float! pageInfo: PageInfo! diff --git a/x-pack/legacy/plugins/siem/server/graphql/ip_details/resolvers.ts b/x-pack/legacy/plugins/siem/server/graphql/ip_details/resolvers.ts index 56a63abda431f..3e962f887b962 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/ip_details/resolvers.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/ip_details/resolvers.ts @@ -14,7 +14,6 @@ import { } from '../../lib/ip_details'; import { createOptions } from '../../utils/build_query/create_options'; import { QuerySourceResolver } from '../sources/resolvers'; -import { DomainFirstLastSeenRequestOptions } from '../../lib/ip_details/types'; export type QueryIpOverviewResolver = ChildResolverOf< AppResolverOf, @@ -31,11 +30,6 @@ export type QueryTlsResolver = ChildResolverOf< QuerySourceResolver >; -type QueryDomainFirstLastSeenResolver = ChildResolverOf< - AppResolverOf, - QuerySourceResolver ->; - export type QueryUsersResolver = ChildResolverOf< AppResolverOf, QuerySourceResolver @@ -52,7 +46,6 @@ export const createIpDetailsResolvers = ( IpOverview: QueryIpOverviewResolver; Domains: QueryDomainsResolver; Tls: QueryTlsResolver; - DomainFirstLastSeen: QueryDomainFirstLastSeenResolver; Users: QueryUsersResolver; }; } => ({ @@ -80,16 +73,6 @@ export const createIpDetailsResolvers = ( }; return libs.ipDetails.getTls(req, options); }, - async DomainFirstLastSeen(source, args, { req }) { - const options: DomainFirstLastSeenRequestOptions = { - sourceConfiguration: source.configuration, - ip: args.ip, - domainName: args.domainName, - flowTarget: args.flowTarget, - defaultIndex: args.defaultIndex, - }; - return libs.ipDetails.getDomainFirstLastSeen(req, options); - }, async Users(source, args, { req }, info) { const options: UsersRequestOptions = { ...createOptions(source, args, info), diff --git a/x-pack/legacy/plugins/siem/server/graphql/ip_details/schema.gql.ts b/x-pack/legacy/plugins/siem/server/graphql/ip_details/schema.gql.ts index 61f36c06b5f9c..dd092724207a4 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/ip_details/schema.gql.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/ip_details/schema.gql.ts @@ -102,23 +102,6 @@ const domainsSchema = gql` } `; -const firstLastSeenSchema = gql` - type FirstLastSeenDomain { - firstSeen: Date - lastSeen: Date - } - - extend type Source { - DomainFirstLastSeen( - id: String - ip: String! - domainName: String! - flowTarget: FlowTarget! - defaultIndex: [String!]! - ): FirstLastSeenDomain! - } -`; - const tlsSchema = gql` enum TlsFields { _id @@ -209,10 +192,4 @@ const usersSchema = gql` } `; -export const ipDetailsSchemas = [ - ipOverviewSchema, - domainsSchema, - firstLastSeenSchema, - tlsSchema, - usersSchema, -]; +export const ipDetailsSchemas = [ipOverviewSchema, domainsSchema, tlsSchema, usersSchema]; diff --git a/x-pack/legacy/plugins/siem/server/graphql/types.ts b/x-pack/legacy/plugins/siem/server/graphql/types.ts index 2a146725d594c..5b2e5e934876a 100644 --- a/x-pack/legacy/plugins/siem/server/graphql/types.ts +++ b/x-pack/legacy/plugins/siem/server/graphql/types.ts @@ -155,8 +155,6 @@ export interface Source { Domains: DomainsData; - DomainFirstLastSeen: FirstLastSeenDomain; - Tls: TlsData; Users: UsersData; @@ -355,8 +353,6 @@ export interface PageInfo { } export interface EventsData { - kpiEventType?: KpiItem[] | null; - edges: EcsEdges[]; totalCount: number; @@ -364,12 +360,6 @@ export interface EventsData { pageInfo: PageInfo; } -export interface KpiItem { - value?: string | null; - - count: number; -} - export interface EcsEdges { node: Ecs; @@ -1010,12 +1000,6 @@ export interface DomainsNetworkField { direction?: NetworkDirectionEcs[] | null; } -export interface FirstLastSeenDomain { - firstSeen?: Date | null; - - lastSeen?: Date | null; -} - export interface TlsData { edges: TlsEdges[]; @@ -1857,17 +1841,6 @@ export interface DomainsSourceArgs { defaultIndex: string[]; } -export interface DomainFirstLastSeenSourceArgs { - id?: string | null; - - ip: string; - - domainName: string; - - flowTarget: FlowTarget; - - defaultIndex: string[]; -} export interface TlsSourceArgs { filterQuery?: string | null; @@ -2438,8 +2411,6 @@ export namespace SourceResolvers { Domains?: DomainsResolver; - DomainFirstLastSeen?: DomainFirstLastSeenResolver; - Tls?: TlsResolver; Users?: UsersResolver; @@ -2649,23 +2620,6 @@ export namespace SourceResolvers { defaultIndex: string[]; } - export type DomainFirstLastSeenResolver< - R = FirstLastSeenDomain, - Parent = Source, - Context = SiemContext - > = Resolver; - export interface DomainFirstLastSeenArgs { - id?: string | null; - - ip: string; - - domainName: string; - - flowTarget: FlowTarget; - - defaultIndex: string[]; - } - export type TlsResolver = Resolver< R, Parent, @@ -3432,8 +3386,6 @@ export namespace PageInfoResolvers { export namespace EventsDataResolvers { export interface Resolvers { - kpiEventType?: KpiEventTypeResolver; - edges?: EdgesResolver; totalCount?: TotalCountResolver; @@ -3441,11 +3393,6 @@ export namespace EventsDataResolvers { pageInfo?: PageInfoResolver; } - export type KpiEventTypeResolver< - R = KpiItem[] | null, - Parent = EventsData, - Context = SiemContext - > = Resolver; export type EdgesResolver = Resolver< R, Parent, @@ -3463,25 +3410,6 @@ export namespace EventsDataResolvers { >; } -export namespace KpiItemResolvers { - export interface Resolvers { - value?: ValueResolver; - - count?: CountResolver; - } - - export type ValueResolver = Resolver< - R, - Parent, - Context - >; - export type CountResolver = Resolver< - R, - Parent, - Context - >; -} - export namespace EcsEdgesResolvers { export interface Resolvers { node?: NodeResolver; @@ -5610,25 +5538,6 @@ export namespace DomainsNetworkFieldResolvers { > = Resolver; } -export namespace FirstLastSeenDomainResolvers { - export interface Resolvers { - firstSeen?: FirstSeenResolver; - - lastSeen?: LastSeenResolver; - } - - export type FirstSeenResolver< - R = Date | null, - Parent = FirstLastSeenDomain, - Context = SiemContext - > = Resolver; - export type LastSeenResolver< - R = Date | null, - Parent = FirstLastSeenDomain, - Context = SiemContext - > = Resolver; -} - export namespace TlsDataResolvers { export interface Resolvers { edges?: EdgesResolver; diff --git a/x-pack/legacy/plugins/siem/server/lib/events/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/events/elasticsearch_adapter.ts index 085c4e3d8ffb5..b5f355a741bd1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/events/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/events/elasticsearch_adapter.ts @@ -22,7 +22,6 @@ import { DetailItem, EcsEdges, EventsData, - KpiItem, LastEventTimeData, TimelineData, TimelineDetailsData, @@ -64,13 +63,6 @@ export class ElasticsearchEventsAdapter implements EventsAdapter { buildQuery(queryOptions) ); - const kpiEventType: KpiItem[] = - response.aggregations && response.aggregations.count_event_type - ? response.aggregations.count_event_type.buckets.map(item => ({ - value: item.key, - count: item.doc_count, - })) - : []; const { limit } = options.pagination; const totalCount = getOr(0, 'hits.total.value', response); const hits = response.hits.hits; @@ -80,7 +72,7 @@ export class ElasticsearchEventsAdapter implements EventsAdapter { const hasNextPage = eventsEdges.length === limit + 1; const edges = hasNextPage ? eventsEdges.splice(0, limit) : eventsEdges; const lastCursor = get('cursor', last(edges)); - return { kpiEventType, edges, totalCount, pageInfo: { hasNextPage, endCursor: lastCursor } }; + return { edges, totalCount, pageInfo: { hasNextPage, endCursor: lastCursor } }; } public async getTimelineData( diff --git a/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts index 8b8636d292d29..6755069d45be1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/events/query.dsl.ts @@ -34,20 +34,6 @@ export const buildQuery = (options: RequestOptions) => { return []; }; - const agg = options.fields.includes('kpiEventType') - ? { - count_event_type: { - terms: { - field: 'event.action', - size: 5, - order: { - _count: 'desc', - }, - }, - }, - } - : {}; - const filter = [...filterClause, ...getTimerangeFilter(options.timerange), { match_all: {} }]; const getSortField = (sortField: SortField) => { @@ -70,7 +56,6 @@ export const buildQuery = (options: RequestOptions) => { index: defaultIndex, ignoreUnavailable: true, body: { - aggregations: agg, query: { bool: { filter, diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts index 630b8321f8db6..8097d7677411b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/elasticsearch_adapter.ts @@ -10,7 +10,6 @@ import { AutonomousSystem, DomainsData, DomainsEdges, - FirstLastSeenDomain, FlowTarget, GeoEcsFields, HostEcsFields, @@ -21,7 +20,7 @@ import { UsersEdges, } from '../../graphql/types'; import { DatabaseSearchResponse, FrameworkAdapter, FrameworkRequest } from '../framework'; -import { SearchHit, TermAggregation } from '../types'; +import { TermAggregation } from '../types'; import { DomainsRequestOptions, @@ -30,11 +29,8 @@ import { UsersRequestOptions, } from './index'; import { buildDomainsQuery } from './query_domains.dsl'; -import { buildFirstLastSeenDomainQuery } from './query_last_first_seen_domain.dsl'; import { buildOverviewQuery } from './query_overview.dsl'; import { - DomainFirstLastSeenItem, - DomainFirstLastSeenRequestOptions, DomainsBuckets, IpDetailsAdapter, IpOverviewHit, @@ -124,23 +120,6 @@ export class ElasticsearchIpOverviewAdapter implements IpDetailsAdapter { }; } - public async getDomainsFirstLastSeen( - request: FrameworkRequest, - options: DomainFirstLastSeenRequestOptions - ): Promise { - const response = await this.framework.callWithRequest( - request, - 'search', - buildFirstLastSeenDomainQuery(options) - ); - - const aggregations: DomainFirstLastSeenItem = get('aggregations', response) || {}; - return { - firstSeen: get('firstSeen.value_as_string', aggregations), - lastSeen: get('lastSeen.value_as_string', aggregations), - }; - } - public async getUsers( request: FrameworkRequest, options: UsersRequestOptions diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/index.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/index.ts index 8722dad45119c..83a58dc5d4e54 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/index.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/index.ts @@ -7,7 +7,6 @@ import { DomainsData, DomainsSortField, - FirstLastSeenDomain, FlowDirection, FlowTarget, IpOverviewData, @@ -18,7 +17,7 @@ import { } from '../../graphql/types'; import { FrameworkRequest, RequestOptions } from '../framework'; -import { DomainFirstLastSeenRequestOptions, IpDetailsAdapter } from './types'; +import { IpDetailsAdapter } from './types'; export * from './elasticsearch_adapter'; @@ -65,13 +64,6 @@ export class IpDetails { return await this.adapter.getTls(req, options); } - public async getDomainFirstLastSeen( - req: FrameworkRequest, - options: DomainFirstLastSeenRequestOptions - ): Promise { - return await this.adapter.getDomainsFirstLastSeen(req, options); - } - public async getUsers(req: FrameworkRequest, options: UsersRequestOptions): Promise { return await this.adapter.getUsers(req, options); } diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts deleted file mode 100644 index c5e0aeb99eeaa..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_last_first_seen_domain.dsl.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { DomainFirstLastSeenRequestOptions } from './types'; - -export const buildFirstLastSeenDomainQuery = ({ - ip, - domainName, - flowTarget, - defaultIndex, -}: DomainFirstLastSeenRequestOptions) => { - const filter = [ - { term: { [`${flowTarget}.ip`]: ip } }, - { term: { [`${flowTarget}.domain`]: domainName } }, - ]; - - const dslQuery = { - allowNoIndices: true, - index: defaultIndex, - ignoreUnavailable: true, - body: { - aggregations: { - firstSeen: { min: { field: '@timestamp' } }, - lastSeen: { max: { field: '@timestamp' } }, - }, - query: { bool: { filter } }, - size: 0, - track_total_hits: false, - }, - }; - - return dslQuery; -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts index b33591cd2e0b0..c97506f19cb9a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/types.ts @@ -4,15 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - DomainsData, - FirstLastSeenDomain, - FlowTarget, - IpOverviewData, - NetworkDirectionEcs, - SourceConfiguration, - UsersData, -} from '../../graphql/types'; +import { DomainsData, IpOverviewData, NetworkDirectionEcs, UsersData } from '../../graphql/types'; import { FrameworkRequest, RequestBasicOptions } from '../framework'; import { Hit, ShardsResponse, TotalValue } from '../types'; @@ -20,10 +12,6 @@ export interface IpDetailsAdapter { getIpDetails(request: FrameworkRequest, options: RequestBasicOptions): Promise; getDomains(request: FrameworkRequest, options: RequestBasicOptions): Promise; getTls(request: FrameworkRequest, options: RequestBasicOptions): Promise; - getDomainsFirstLastSeen( - req: FrameworkRequest, - options: DomainFirstLastSeenRequestOptions - ): Promise; getUsers(request: FrameworkRequest, options: RequestBasicOptions): Promise; } @@ -155,25 +143,6 @@ export interface TlsBuckets { }; } -export interface DomainFirstLastSeenRequestOptions { - ip: string; - domainName: string; - flowTarget: FlowTarget; - sourceConfiguration: SourceConfiguration; - defaultIndex: string[]; -} - -export interface DomainFirstLastSeenItem { - firstSeen?: { - value: number; - value_as_string: string; - }; - lastSeen?: { - value: number; - value_as_string: string; - }; -} - // Users Table export interface UsersResponse { diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts index d1c646ef19892..4afb3c6607ff1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_top_n_flow.dsl.ts @@ -144,6 +144,16 @@ const getUniDirectionAggs = ( domain: { terms: { field: `${flowTarget}.domain`, + order: { + timestamp: 'desc', + }, + }, + aggs: { + timestamp: { + max: { + field: '@timestamp', + }, + }, }, }, ip_count: { @@ -193,6 +203,16 @@ const getBiDirectionAggs = ( domain: { terms: { field: `${flowTarget}.domain`, + order: { + timestamp: 'desc', + }, + }, + aggs: { + timestamp: { + max: { + field: '@timestamp', + }, + }, }, }, ip_count: { diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 6343f34dc2444..4dd380a15c63a 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -8,24 +8,24 @@ export default function ({ loadTestFile }) { describe('apis', function () { this.tags('ciGroup6'); - // loadTestFile(require.resolve('./actions')); - // loadTestFile(require.resolve('./alerting')); - // loadTestFile(require.resolve('./es')); - // loadTestFile(require.resolve('./security')); - // loadTestFile(require.resolve('./monitoring')); - // loadTestFile(require.resolve('./xpack_main')); - // loadTestFile(require.resolve('./telemetry')); - // loadTestFile(require.resolve('./logstash')); - // loadTestFile(require.resolve('./kibana')); - // loadTestFile(require.resolve('./infra')); - // loadTestFile(require.resolve('./beats')); - // loadTestFile(require.resolve('./console')); - // loadTestFile(require.resolve('./management')); - // loadTestFile(require.resolve('./uptime')); - // loadTestFile(require.resolve('./maps')); - // loadTestFile(require.resolve('./apm')); + loadTestFile(require.resolve('./actions')); + loadTestFile(require.resolve('./alerting')); + loadTestFile(require.resolve('./es')); + loadTestFile(require.resolve('./security')); + loadTestFile(require.resolve('./monitoring')); + loadTestFile(require.resolve('./xpack_main')); + loadTestFile(require.resolve('./telemetry')); + loadTestFile(require.resolve('./logstash')); + loadTestFile(require.resolve('./kibana')); + loadTestFile(require.resolve('./infra')); + loadTestFile(require.resolve('./beats')); + loadTestFile(require.resolve('./console')); + loadTestFile(require.resolve('./management')); + loadTestFile(require.resolve('./uptime')); + loadTestFile(require.resolve('./maps')); + loadTestFile(require.resolve('./apm')); loadTestFile(require.resolve('./siem')); - // loadTestFile(require.resolve('./code')); - // loadTestFile(require.resolve('./short_urls')); + loadTestFile(require.resolve('./code')); + loadTestFile(require.resolve('./short_urls')); }); } From 2c7134751b38522266de448bf0c38abe8e8da735 Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Fri, 5 Jul 2019 10:00:27 -0600 Subject: [PATCH 08/11] address PR comments --- .../plugins/siem/server/lib/kpi_hosts/mock.ts | 10 +++--- .../lib/kpi_hosts/query_authentication.dsl.ts | 36 ++++++------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts index 4800a4359c094..36dd50ec2d203 100644 --- a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts +++ b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/mock.ts @@ -373,11 +373,13 @@ const mockAuthAggs = { const mockAuthFilter = { bool: { - should: [ - { match: { 'event.type': 'authentication_success' } }, - { match: { 'event.type': 'authentication_failure' } }, + filter: [ + { + term: { + 'event.category': 'authentication', + }, + }, ], - minimum_should_match: 1, }, }; diff --git a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts index 887444ecb1eaf..5734aa6ee88cc 100644 --- a/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/kpi_hosts/query_authentication.dsl.ts @@ -6,30 +6,6 @@ import { createQueryFilterClauses } from '../../utils/build_query'; import { KpiHostsESMSearchBody } from './types'; import { RequestBasicOptions } from '../framework'; -const getAuthQueryFilter = () => [ - { - bool: { - should: [ - { - match: { - 'event.type': 'authentication_success', - }, - }, - { - match: { - 'event.type': 'authentication_failure', - }, - }, - { - match: { - 'event.category': 'authentication', - }, - }, - ], - minimum_should_match: 1, - }, - }, -]; export const buildAuthQuery = ({ filterQuery, @@ -41,7 +17,17 @@ export const buildAuthQuery = ({ }: RequestBasicOptions): KpiHostsESMSearchBody[] => { const filter = [ ...createQueryFilterClauses(filterQuery), - ...getAuthQueryFilter(), + { + bool: { + filter: [ + { + term: { + 'event.category': 'authentication', + }, + }, + ], + }, + }, { range: { [timestamp]: { From e82ea41b8d7d85bed2f12610aeae3c1cb4ab34ee Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Fri, 5 Jul 2019 15:37:20 -0600 Subject: [PATCH 09/11] authentication adjustment --- .../server/lib/authentications/elasticsearch_adapter.ts | 6 +++++- .../plugins/siem/server/lib/authentications/query.dsl.ts | 7 ------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/authentications/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/authentications/elasticsearch_adapter.ts index 611bb623e5ee3..18c37c3a9e828 100644 --- a/x-pack/legacy/plugins/siem/server/lib/authentications/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/authentications/elasticsearch_adapter.ts @@ -39,7 +39,11 @@ export class ElasticsearchAuthenticationAdapter implements AuthenticationsAdapte 'aggregations.group_by_users.buckets', response ).map((bucket: AuthenticationBucket) => ({ - _id: bucket.authentication.hits.hits[0]._id, + _id: getOr( + `${bucket.key}+${bucket.doc_count}`, + 'failures.lastFailure.hits.hits[0].id', + bucket + ), _source: { lastSuccess: getOr(null, 'successes.lastSuccess.hits.hits[0]._source', bucket), lastFailure: getOr(null, 'failures.lastFailure.hits.hits[0]._source', bucket), diff --git a/x-pack/legacy/plugins/siem/server/lib/authentications/query.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/authentications/query.dsl.ts index b2216861de378..ae184c3212995 100644 --- a/x-pack/legacy/plugins/siem/server/lib/authentications/query.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/authentications/query.dsl.ts @@ -99,13 +99,6 @@ export const buildQuery = ({ }, }, }, - authentication: { - top_hits: { - size: 1, - _source: esFields, - sort: [{ '@timestamp': { order: 'desc' } }], - }, - }, }, }, }, From b724339845679be9152d5d2030059c09bd35f55f Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Fri, 5 Jul 2019 16:03:26 -0600 Subject: [PATCH 10/11] more PR concerns addressed --- .../server/lib/hosts/query.detail_host.dsl.ts | 1 - .../server/lib/ip_details/query_tls.dsl.ts | 1 - .../lib/kpi_network/query_unique_flow.ts | 2 +- .../siem/server/lib/network/query_dns.dsl.ts | 9 ----- x-pack/test/api_integration/apis/index.js | 36 +++++++++---------- 5 files changed, 19 insertions(+), 30 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/hosts/query.detail_host.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/hosts/query.detail_host.dsl.ts index 9d9727130db08..db08fa14e2b44 100644 --- a/x-pack/legacy/plugins/siem/server/lib/hosts/query.detail_host.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/hosts/query.detail_host.dsl.ts @@ -39,7 +39,6 @@ export const buildHostOverviewQuery = ({ ignoreUnavailable: true, body: { aggregations: { - lastSeen: { max: { field: '@timestamp' } }, ...buildFieldsTermAggregation(esFields.filter(field => !['@timestamp'].includes(field))), }, query: { bool: { filter } }, diff --git a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts index 6456f4b21f56e..22292cb1194e3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/ip_details/query_tls.dsl.ts @@ -69,7 +69,6 @@ export const buildTlsQuery = ({ ...createQueryFilterClauses(filterQuery), { range: { [timestamp]: { gte: from, lte: to } } }, { term: { [`${flowTarget}.ip`]: ip } }, - { term: { 'network.protocol': 'tls' } }, ]; const dslQuery = { diff --git a/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_flow.ts b/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_flow.ts index 04f248c052b2f..4581b889cc9ef 100644 --- a/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_flow.ts +++ b/x-pack/legacy/plugins/siem/server/lib/kpi_network/query_unique_flow.ts @@ -50,7 +50,7 @@ export const buildUniqueFlowIdsQuery = ({ }, }, size: 0, - track_total_hits: true, + track_total_hits: false, }, ]; diff --git a/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts b/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts index 814b05fe404c5..7c08aab1e20b5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts +++ b/x-pack/legacy/plugins/siem/server/lib/network/query_dns.dsl.ts @@ -55,14 +55,6 @@ const createIncludePTRFilter = (isPtrIncluded: boolean) => ], }; -const getDnsFilter = () => ({ - term: { - 'network.protocol': { - value: 'dns', - }, - }, -}); - export const buildDnsQuery = ({ filterQuery, isPtrIncluded, @@ -84,7 +76,6 @@ export const buildDnsQuery = ({ }, }, }, - getDnsFilter(), ]; const dslQuery = { diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 4dd380a15c63a..6343f34dc2444 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -8,24 +8,24 @@ export default function ({ loadTestFile }) { describe('apis', function () { this.tags('ciGroup6'); - loadTestFile(require.resolve('./actions')); - loadTestFile(require.resolve('./alerting')); - loadTestFile(require.resolve('./es')); - loadTestFile(require.resolve('./security')); - loadTestFile(require.resolve('./monitoring')); - loadTestFile(require.resolve('./xpack_main')); - loadTestFile(require.resolve('./telemetry')); - loadTestFile(require.resolve('./logstash')); - loadTestFile(require.resolve('./kibana')); - loadTestFile(require.resolve('./infra')); - loadTestFile(require.resolve('./beats')); - loadTestFile(require.resolve('./console')); - loadTestFile(require.resolve('./management')); - loadTestFile(require.resolve('./uptime')); - loadTestFile(require.resolve('./maps')); - loadTestFile(require.resolve('./apm')); + // loadTestFile(require.resolve('./actions')); + // loadTestFile(require.resolve('./alerting')); + // loadTestFile(require.resolve('./es')); + // loadTestFile(require.resolve('./security')); + // loadTestFile(require.resolve('./monitoring')); + // loadTestFile(require.resolve('./xpack_main')); + // loadTestFile(require.resolve('./telemetry')); + // loadTestFile(require.resolve('./logstash')); + // loadTestFile(require.resolve('./kibana')); + // loadTestFile(require.resolve('./infra')); + // loadTestFile(require.resolve('./beats')); + // loadTestFile(require.resolve('./console')); + // loadTestFile(require.resolve('./management')); + // loadTestFile(require.resolve('./uptime')); + // loadTestFile(require.resolve('./maps')); + // loadTestFile(require.resolve('./apm')); loadTestFile(require.resolve('./siem')); - loadTestFile(require.resolve('./code')); - loadTestFile(require.resolve('./short_urls')); + // loadTestFile(require.resolve('./code')); + // loadTestFile(require.resolve('./short_urls')); }); } From 806b45a1aedfd666836f324cbbc8817839840d3b Mon Sep 17 00:00:00 2001 From: stephmilovic Date: Fri, 5 Jul 2019 16:03:39 -0600 Subject: [PATCH 11/11] more PR concerns addressed --- x-pack/test/api_integration/apis/index.js | 36 +++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 6343f34dc2444..4dd380a15c63a 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -8,24 +8,24 @@ export default function ({ loadTestFile }) { describe('apis', function () { this.tags('ciGroup6'); - // loadTestFile(require.resolve('./actions')); - // loadTestFile(require.resolve('./alerting')); - // loadTestFile(require.resolve('./es')); - // loadTestFile(require.resolve('./security')); - // loadTestFile(require.resolve('./monitoring')); - // loadTestFile(require.resolve('./xpack_main')); - // loadTestFile(require.resolve('./telemetry')); - // loadTestFile(require.resolve('./logstash')); - // loadTestFile(require.resolve('./kibana')); - // loadTestFile(require.resolve('./infra')); - // loadTestFile(require.resolve('./beats')); - // loadTestFile(require.resolve('./console')); - // loadTestFile(require.resolve('./management')); - // loadTestFile(require.resolve('./uptime')); - // loadTestFile(require.resolve('./maps')); - // loadTestFile(require.resolve('./apm')); + loadTestFile(require.resolve('./actions')); + loadTestFile(require.resolve('./alerting')); + loadTestFile(require.resolve('./es')); + loadTestFile(require.resolve('./security')); + loadTestFile(require.resolve('./monitoring')); + loadTestFile(require.resolve('./xpack_main')); + loadTestFile(require.resolve('./telemetry')); + loadTestFile(require.resolve('./logstash')); + loadTestFile(require.resolve('./kibana')); + loadTestFile(require.resolve('./infra')); + loadTestFile(require.resolve('./beats')); + loadTestFile(require.resolve('./console')); + loadTestFile(require.resolve('./management')); + loadTestFile(require.resolve('./uptime')); + loadTestFile(require.resolve('./maps')); + loadTestFile(require.resolve('./apm')); loadTestFile(require.resolve('./siem')); - // loadTestFile(require.resolve('./code')); - // loadTestFile(require.resolve('./short_urls')); + loadTestFile(require.resolve('./code')); + loadTestFile(require.resolve('./short_urls')); }); }