From a21bd6a9165af05bc85a4b8364928f02786d3557 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Fri, 8 Sep 2023 17:00:56 +0200 Subject: [PATCH] NETOBSERV-1269 refactor overlapping detection for BNF (#379) This fix allows working on situations as described in https://issues.redhat.com/browse/NETOBSERV-1269 and, hopefully, is a more generic / consistent solution anyway. Changing how overlapping results are managed: work on results rather than on query to remove overlapping parts. When fetching flows, we don't explicitly remove the overlaps, but this is managed implicitly with the existing dedup functions When fetching metrics, we consider that the computed metrics for any label set is the ORIGINAL metrics + the SWAPPED metrics - the OVERLAP metrics. Add tests, fix tests Note that adjustment is necessary in MetricsQuerySummary test as the "endWithTolerance" calculation has changed a little bit, modifying computed rates ... new values are still close to the previous ones Rename callback function --- web/src/components/netflow-traffic.tsx | 44 +- .../__tests__/metrics-query-summary.spec.tsx | 4 +- web/src/model/__tests__/flow-query.spec.ts | 280 ---------- web/src/model/filters.ts | 3 - web/src/model/flow-query.ts | 86 +--- .../utils/__tests__/back-and-forth.spec.ts | 482 ++++++++++++++++++ web/src/utils/back-and-forth.ts | 99 ++++ web/src/utils/filter-definitions.ts | 60 +-- web/src/utils/metrics.ts | 90 +++- 9 files changed, 718 insertions(+), 430 deletions(-) delete mode 100644 web/src/model/__tests__/flow-query.spec.ts create mode 100644 web/src/utils/__tests__/back-and-forth.spec.ts create mode 100644 web/src/utils/back-and-forth.ts diff --git a/web/src/components/netflow-traffic.tsx b/web/src/components/netflow-traffic.tsx index 704a8d82e..66c768fcf 100644 --- a/web/src/components/netflow-traffic.tsx +++ b/web/src/components/netflow-traffic.tsx @@ -32,7 +32,7 @@ import { useTranslation } from 'react-i18next'; import { useTheme } from '../utils/theme-hook'; import { Record } from '../api/ipfix'; import { GenericMetric, Stats, TopologyMetrics } from '../api/loki'; -import { getGenericMetrics, getFlows, getTopologyMetrics } from '../api/routes'; +import { getGenericMetrics } from '../api/routes'; import { DisabledFilters, Filter, @@ -45,13 +45,13 @@ import { } from '../model/filters'; import { FlowQuery, - groupFilters, Match, MetricFunction, FlowScope, MetricType, PacketLoss, - RecordType + RecordType, + filtersToString } from '../model/flow-query'; import { MetricScopeOptions } from '../model/metrics'; import { parseQuickFilters } from '../model/quick-filters'; @@ -142,6 +142,8 @@ import { exportToPng } from '../utils/export'; import { navigate } from './dynamic-loader/dynamic-loader'; import { LinksOverflow } from './overflow/links-overflow'; import { mergeFlowReporters } from '../utils/flows'; +import { getFetchFunctions as getBackAndForthFetch } from '../utils/back-and-forth'; +import { mergeStats } from '../utils/metrics'; export type ViewId = 'overview' | 'table' | 'topology'; @@ -339,9 +341,8 @@ export const NetflowTraffic: React.FC<{ const buildFlowQuery = React.useCallback((): FlowQuery => { const enabledFilters = getEnabledFilters(forcedFilters || filters); - const groupedFilters = groupFilters(enabledFilters, match === 'any'); const query: FlowQuery = { - filters: groupedFilters, + filters: filtersToString(enabledFilters.list, match === 'any'), limit: LIMIT_VALUES.includes(limit) ? limit : LIMIT_VALUES[0], recordType: recordType, dedup: !showDuplicates, @@ -387,6 +388,13 @@ export const NetflowTraffic: React.FC<{ topologyOptions.groupTypes ]); + const getFetchFunctions = React.useCallback(() => { + // check back-and-forth + const enabledFilters = getEnabledFilters(forcedFilters || filters); + const matchAny = match === 'any'; + return getBackAndForthFetch(enabledFilters, matchAny); + }, [forcedFilters, filters, match]); + const manageWarnings = React.useCallback( (query: Promise) => { Promise.race([query, new Promise((resolve, reject) => setTimeout(reject, 2000, 'slow'))]).then( @@ -403,20 +411,12 @@ export const NetflowTraffic: React.FC<{ [] ); - const mergeStats = (prev: Stats | undefined, current: Stats): Stats => { - if (!prev) { - return current; - } - return { - ...prev, - limitReached: prev.limitReached || current.limitReached, - numQueries: prev.numQueries + current.numQueries - }; - }; - const fetchTable = React.useCallback( (fq: FlowQuery, droppedType: MetricType | undefined) => { setMetrics([]); + + const { getFlows, getTopologyMetrics } = getFetchFunctions(); + // table query is based on histogram range if available const tableQuery = { ...fq }; if (histogramRange) { @@ -452,12 +452,15 @@ export const NetflowTraffic: React.FC<{ } return Promise.all(promises); }, - [histogramRange, range, showHistogram, showDuplicates] + [histogramRange, range, showHistogram, showDuplicates, getFetchFunctions] ); const fetchOverview = React.useCallback( (fq: FlowQuery, droppedType: MetricType | undefined) => { setFlows([]); + + const { getTopologyMetrics } = getFetchFunctions(); + const promises: Promise[] = [ //get bytes or packets getTopologyMetrics(fq, range).then(res => { @@ -555,12 +558,15 @@ export const NetflowTraffic: React.FC<{ } return Promise.all(promises); }, - [range, config.features] + [range, config.features, getFetchFunctions] ); const fetchTopology = React.useCallback( (fq: FlowQuery, droppedType: MetricType | undefined) => { setFlows([]); + + const { getTopologyMetrics } = getFetchFunctions(); + const promises: Promise[] = [ //get bytes or packets getTopologyMetrics(fq, range).then(res => { @@ -581,7 +587,7 @@ export const NetflowTraffic: React.FC<{ } return Promise.all(promises); }, - [range] + [range, getFetchFunctions] ); const tick = React.useCallback(() => { diff --git a/web/src/components/query-summary/__tests__/metrics-query-summary.spec.tsx b/web/src/components/query-summary/__tests__/metrics-query-summary.spec.tsx index 9aa85ebe0..0a0d2e6a2 100644 --- a/web/src/components/query-summary/__tests__/metrics-query-summary.spec.tsx +++ b/web/src/components/query-summary/__tests__/metrics-query-summary.spec.tsx @@ -28,9 +28,9 @@ describe('', () => { it('should show summary', async () => { const wrapper = mount(); - expect(wrapper.find('#bytesCount').last().text()).toBe('7 MB'); + expect(wrapper.find('#bytesCount').last().text()).toBe('6.8 MB'); expect(wrapper.find('#packetsCount')).toHaveLength(0); - expect(wrapper.find('#bpsCount').last().text()).toBe('23.2 kBps'); + expect(wrapper.find('#bpsCount').last().text()).toBe('22.79 kBps'); expect(wrapper.find('#lastRefresh').last().text()).toBe(now.toLocaleTimeString()); }); diff --git a/web/src/model/__tests__/flow-query.spec.ts b/web/src/model/__tests__/flow-query.spec.ts deleted file mode 100644 index 478c76452..000000000 --- a/web/src/model/__tests__/flow-query.spec.ts +++ /dev/null @@ -1,280 +0,0 @@ -import { findFilter } from '../../utils/filter-definitions'; -import { Filter, FilterId, FilterValue } from '../filters'; -import { groupFilters } from '../flow-query'; - -const filter = (id: FilterId, values: FilterValue[], not?: boolean): Filter => { - return { - def: findFilter((k: string) => k, id)!, - values: values, - not: not - }; -}; - -describe('groupFiltersMatchAll', () => { - it('should encode', () => { - const grouped = groupFilters( - { list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], backAndForth: false }, - false - ); - expect(grouped).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); - }); - - it('should generate AND groups', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], - backAndForth: false - }, - false - ) - ); - expect(grouped).toEqual('SrcK8S_Name=test1,test2&SrcK8S_Namespace=ns'); - }); - - it('should generate AND groups, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], - backAndForth: true - }, - false - ) - ); - expect(grouped).toEqual('SrcK8S_Name=test1,test2&DstPort=443|DstK8S_Name=test1,test2&SrcPort=443'); - }); - - it('should generate AND groups, back and forth, accounting for overlap', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_namespace', [{ v: 'test1' }, { v: 'test2' }]), filter('src_port', [{ v: '443' }])], - backAndForth: true - }, - false - ) - ); - expect(grouped).toEqual( - 'SrcK8S_Namespace=test1,test2&SrcPort=443|DstK8S_Namespace=test1,test2&DstPort=443&SrcK8S_Namespace!=test1,test2' - ); - }); - - it('should not swap and correct overlap on fully symetric parts', () => { - // For explanations see comments in function "determineOverlap" - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_namespace', [{ v: 'infra*' }], true), filter('dst_namespace', [{ v: 'infra*' }], true)], - backAndForth: true - }, - false - ) - ); - expect(grouped).toEqual('SrcK8S_Namespace!=infra*&DstK8S_Namespace!=infra*'); - }); - - it('should not correct overlap on partial symetric parts', () => { - // For explanations see comments in function "determineOverlap" - const grouped = decodeURIComponent( - groupFilters( - { - list: [ - filter('src_namespace', [{ v: 'infra*' }], true), - filter('src_port', [{ v: '443' }]), - filter('dst_namespace', [{ v: 'infra*' }], true) - ], - backAndForth: true - }, - false - ) - ); - expect(grouped).toEqual( - 'SrcK8S_Namespace!=infra*&SrcPort=443&DstK8S_Namespace!=infra*|DstK8S_Namespace!=infra*&DstPort=443&SrcK8S_Namespace!=infra*' - ); - }); - - it('should generate AND groups, back and forth, mixed with non-Src/Dst', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [ - filter('src_name', [{ v: 'test' }]), - filter('dst_port', [{ v: '443' }]), - filter('src_kind', [{ v: 'Pod' }]), - filter('protocol', [{ v: '6' }]) - ], - backAndForth: true - }, - false - ) - ); - expect(grouped).toEqual( - 'SrcK8S_Name=test&DstPort=443&SrcK8S_Type=Pod&Proto=6' + '|DstK8S_Name=test&SrcPort=443&DstK8S_Type=Pod&Proto=6' - ); - }); - - it('should generate simple Src K8S resource', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: false }, false) - ); - expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"'); - }); - - it('should generate K8S resource, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: true }, false) - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + - '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' - ); - }); - - it('should generate Node Src/Dst K8S resource, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'Node.test' }])], backAndForth: true }, false) - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Node"&SrcK8S_Namespace=""&SrcK8S_Name="test"' + - '|DstK8S_Type="Node"&DstK8S_Namespace=""&DstK8S_Name="test"' - ); - }); - - it('should generate Owner Src/Dst K8S resource, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], backAndForth: true }, false) - ); - expect(grouped).toEqual( - 'SrcK8S_OwnerType="DaemonSet"&SrcK8S_Namespace="ns"&SrcK8S_OwnerName="test"' + - '|DstK8S_OwnerType="DaemonSet"&DstK8S_Namespace="ns"&DstK8S_OwnerName="test"' - ); - }); - - it('should generate Src/Dst K8S resource ANDed with Dst Name, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], - backAndForth: true - }, - false - ) - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"&DstK8S_Name=peer' + - '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"&SrcK8S_Name=peer' - ); - }); -}); - -describe('groupFiltersMatchAny', () => { - it('should encode', () => { - const grouped = groupFilters( - { list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], backAndForth: false }, - true - ); - expect(grouped).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); - }); - - it('should generate OR groups', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], - backAndForth: false - }, - true - ) - ); - expect(grouped).toEqual('SrcK8S_Name=test1,test2|SrcK8S_Namespace=ns'); - }); - - it('should generate OR groups, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], - backAndForth: true - }, - true - ) - ); - expect(grouped).toEqual('SrcK8S_Name=test1,test2|DstPort=443|DstK8S_Name=test1,test2|SrcPort=443'); - }); - - it('should generate flat OR groups, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [ - filter('src_name', [{ v: 'test' }]), - filter('src_port', [{ v: '443' }]), - filter('src_kind', [{ v: 'Pod' }]), - filter('protocol', [{ v: '6' }]) - ], - backAndForth: true - }, - true - ) - ); - expect(grouped).toEqual( - 'SrcK8S_Name=test|SrcPort=443|SrcK8S_Type=Pod|Proto=6|DstK8S_Name=test|DstPort=443|DstK8S_Type=Pod' - ); - }); - - it('should generate simple Src K8S resource', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: false }, true) - ); - expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"'); - }); - - it('should generate K8S resource, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: true }, true) - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + - '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' - ); - }); - - it('should generate Node K8S resource, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'Node.test' }])], backAndForth: true }, true) - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Node"&SrcK8S_Namespace=""&SrcK8S_Name="test"' + - '|DstK8S_Type="Node"&DstK8S_Namespace=""&DstK8S_Name="test"' - ); - }); - - it('should generate Owner K8S resource, back and forth', () => { - const grouped = decodeURIComponent( - groupFilters({ list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], backAndForth: true }, true) - ); - expect(grouped).toEqual( - 'SrcK8S_OwnerType="DaemonSet"&SrcK8S_Namespace="ns"&SrcK8S_OwnerName="test"' + - '|DstK8S_OwnerType="DaemonSet"&DstK8S_Namespace="ns"&DstK8S_OwnerName="test"' - ); - }); - - it('should generate K8S resource, back and forth, ORed with Name', () => { - const grouped = decodeURIComponent( - groupFilters( - { - list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], - backAndForth: true - }, - true - ) - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + - '|DstK8S_Name=peer' + - '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' + - '|SrcK8S_Name=peer' - ); - }); -}); diff --git a/web/src/model/filters.ts b/web/src/model/filters.ts index 0b4f7e002..02aa6f418 100644 --- a/web/src/model/filters.ts +++ b/web/src/model/filters.ts @@ -56,9 +56,6 @@ export interface FilterDefinition { docUrl?: string; placeholder?: string; encoder: FiltersEncoder; - // overlap tells if the type of entity referred to with this filter may have overlapping (duplicated) - // flows when querying for returned traffic (back and forth) - they result in slightly more complicated queries. - overlap: boolean; } export interface FilterValue { diff --git a/web/src/model/flow-query.ts b/web/src/model/flow-query.ts index 09b8acc50..057964bc5 100644 --- a/web/src/model/flow-query.ts +++ b/web/src/model/flow-query.ts @@ -1,5 +1,4 @@ -import { Filter, Filters, filterKeyEqual } from './filters'; -import { swapFilters } from '../components/filters/filters-helper'; +import { Filter } from './filters'; export type RecordType = 'allConnections' | 'newConnection' | 'heartbeat' | 'endConnection' | 'flowLog'; export type Match = 'all' | 'any'; @@ -36,92 +35,13 @@ export interface FlowQuery { step?: string; } -export const groupFilters = (filters: Filters, matchAny: boolean): string => { - let result = filtersToString(filters.list, matchAny); - if (filters.backAndForth) { - const { swapped, overlaps } = swap(filters.list, matchAny); - if (swapped.length > 0) { - result = `${result}|${filtersToString(swapped, matchAny)}`; - if (overlaps.length > 0) { - result = `${result}&${filtersToString(overlaps, matchAny)}`; - } - } - } - return encodeURIComponent(result); -}; - -const filtersToString = (filters: Filter[], matchAny: boolean): string => { +export const filtersToString = (filters: Filter[], matchAny: boolean): string => { const matches: string[] = []; filters.forEach(f => { const str = f.def.encoder(f.values, matchAny, f.not || false, f.moreThan || false); matches.push(str); }); - return matches.join(matchAny ? '|' : '&'); -}; - -const swap = (filters: Filter[], matchAny: boolean): { swapped: Filter[]; overlaps: Filter[] } => { - // include swapped traffic - const swapped = swapFilters((k: string) => k, filters); - if (matchAny) { - return { - // In match-any mode, remove non-swappable filters as they would result in duplicates - swapped: swapped.filter(f => f.def.id.startsWith('src_') || f.def.id.startsWith('dst_')), - overlaps: [] // overlap not handled in match-any - }; - } - // match-all mode - const { overlaps, cancelSwap } = determineOverlap(filters, swapped); - if (cancelSwap) { - return { swapped: [], overlaps: [] }; - } - return { - swapped, - overlaps - }; -}; - -const determineOverlap = (orig: Filter[], swapped: Filter[]): { overlaps: Filter[]; cancelSwap: boolean } => { - // With "back and forth", the input query is "doubled" with an analoguous query that captures the return traffic - // Overlap detection consists in excluding from that added query the overlapping part, by adding the opposite of the 1st query to the second - // E.g. src=A => src=A OR (dst=A AND src!=A) - // | | | |-> excluding overlaping part - // | | |-> flipped part (return traffic) - // | |-> resulting query - // |-> initial query - // - // When the input query is fully symetric (same filters as src and dst), this is a special case: we don't want this: - // E.g. src=A AND dst=A => (src=A AND dst=A) OR (dst=A AND src=A AND src!=A AND dst !=A) - // |-> this is void => omit that part, ie. cancel swapping - // - // When the input query is only partially symetric, we need to keep swapping but remove symetric parts from the overlap: - // E.g. src=A AND dst=A AND dstnode=N => (src=A AND dst=A AND dstnode=N) OR (dst=A AND src=A AND srcnode=N AND dstnode!=N) - // |-> keep swapping, but do not exclude overlap - // ie. do not write "... OR (dst=A AND src=A AND srcnode=N AND src!=A AND dst!=A AND dstnode!=N)" - // as that would result in an always empty set. - let cancelSwap = true; - const overlaps: Filter[] = []; - orig.forEach(o => { - if (o.def.overlap) { - const valuesFromSwapped = swapped.find(s => filterKeyEqual(o, s))?.values.map(v => v.v); - const overlap: Filter = valuesFromSwapped - ? { - def: o.def, - not: o.not !== true, - moreThan: o.moreThan !== true, - // only include non-symetric values - values: o.values.filter(ov => !valuesFromSwapped.includes(ov.v)) - } - : { ...o, not: o.not !== true, moreThan: o.moreThan !== true }; - if (overlap.values.length > 0) { - // if there's some overlap here, it's because we found at least one non-symetric value - cancelSwap = false; - overlaps.push(overlap); - } - } else { - cancelSwap = false; - } - }); - return { overlaps, cancelSwap }; + return encodeURIComponent(matches.join(matchAny ? '|' : '&')); }; export const filterByHashId = (hashId: string): string => { diff --git a/web/src/utils/__tests__/back-and-forth.spec.ts b/web/src/utils/__tests__/back-and-forth.spec.ts new file mode 100644 index 000000000..4c80b35c2 --- /dev/null +++ b/web/src/utils/__tests__/back-and-forth.spec.ts @@ -0,0 +1,482 @@ +import { findFilter } from '../filter-definitions'; +import { Filter, FilterId, FilterValue, Filters } from '../../model/filters'; +import { getFlows, getTopologyMetrics } from '../../api/routes'; +import { getFetchFunctions, mergeTopologyMetricsBNF } from '../back-and-forth'; +import { filtersToString } from '../../model/flow-query'; +import { RawTopologyMetrics, TopologyMetricsResult } from '../../api/loki'; +import { parseTopologyMetrics } from '../metrics'; + +jest.mock('../../api/routes', () => ({ + getFlows: jest.fn(() => Promise.resolve({ records: [] })), + getTopologyMetrics: jest.fn(() => Promise.resolve({ metrics: [] })) +})); +const getFlowsMock = getFlows as jest.Mock; +const getTopologyMock = getTopologyMetrics as jest.Mock; + +const filter = (id: FilterId, values: FilterValue[], not?: boolean): Filter => { + return { + def: findFilter((k: string) => k, id)!, + values: values, + not: not + }; +}; + +const getEncodedFilter = (filters: Filters, matchAny: boolean) => { + getFetchFunctions(filters, matchAny).getFlows({ + filters: filtersToString(filters.list, matchAny), + recordType: 'flowLog', + limit: 5, + packetLoss: 'all' + }); + expect(getFlowsMock).toHaveBeenCalledTimes(1); + return getFlowsMock.mock.calls[0][0].filters; +}; + +const getDecodedFilter = (filters: Filters, matchAny: boolean) => { + return decodeURIComponent(getEncodedFilter(filters, matchAny)); +}; + +describe('Match all, flows', () => { + beforeEach(() => { + getFlowsMock.mockClear(); + }); + + it('should encode', () => { + const filters = getEncodedFilter( + { list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], backAndForth: false }, + false + ); + expect(filters).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); + }); + + it('should generate AND groups', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], + backAndForth: false + }, + false + ); + expect(grouped).toEqual('SrcK8S_Name=test1,test2&SrcK8S_Namespace=ns'); + }); + + it('should generate AND groups, back and forth', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], + backAndForth: true + }, + false + ); + expect(grouped).toEqual('SrcK8S_Name=test1,test2&DstPort=443|DstK8S_Name=test1,test2&SrcPort=443'); + }); + + it('should filter for namespace to owner back and forth', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_namespace', [{ v: 'ns' }]), filter('dst_owner_name', [{ v: 'test' }])], + backAndForth: true + }, + false + ); + expect(grouped).toEqual('SrcK8S_Namespace=ns&DstK8S_OwnerName=test|DstK8S_Namespace=ns&SrcK8S_OwnerName=test'); + }); + + it('should generate AND groups, back and forth, mixed with non-Src/Dst', () => { + const grouped = getDecodedFilter( + { + list: [ + filter('src_name', [{ v: 'test' }]), + filter('dst_port', [{ v: '443' }]), + filter('src_kind', [{ v: 'Pod' }]), + filter('protocol', [{ v: '6' }]) + ], + backAndForth: true + }, + false + ); + expect(grouped).toEqual( + 'SrcK8S_Name=test&DstPort=443&SrcK8S_Type=Pod&Proto=6' + '|DstK8S_Name=test&SrcPort=443&DstK8S_Type=Pod&Proto=6' + ); + }); + + it('should generate simple Src K8S resource', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: false }, + false + ); + expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"'); + }); + + it('should generate K8S resource, back and forth', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: true }, + false + ); + expect(grouped).toEqual( + 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + + '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' + ); + }); + + it('should generate Node Src/Dst K8S resource, back and forth', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'Node.test' }])], backAndForth: true }, + false + ); + expect(grouped).toEqual( + 'SrcK8S_Type="Node"&SrcK8S_Namespace=""&SrcK8S_Name="test"' + + '|DstK8S_Type="Node"&DstK8S_Namespace=""&DstK8S_Name="test"' + ); + }); + + it('should generate Owner Src/Dst K8S resource, back and forth', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], backAndForth: true }, + false + ); + expect(grouped).toEqual( + 'SrcK8S_OwnerType="DaemonSet"&SrcK8S_Namespace="ns"&SrcK8S_OwnerName="test"' + + '|DstK8S_OwnerType="DaemonSet"&DstK8S_Namespace="ns"&DstK8S_OwnerName="test"' + ); + }); + + it('should generate Src/Dst K8S resource ANDed with Dst Name, back and forth', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], + backAndForth: true + }, + false + ); + expect(grouped).toEqual( + 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"&DstK8S_Name=peer' + + '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"&SrcK8S_Name=peer' + ); + }); +}); + +describe('Match any, flows', () => { + beforeEach(() => { + getFlowsMock.mockClear(); + }); + + it('should encode', () => { + const grouped = getEncodedFilter( + { list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], backAndForth: false }, + true + ); + expect(grouped).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); + }); + + it('should generate OR groups', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], + backAndForth: false + }, + true + ); + expect(grouped).toEqual('SrcK8S_Name=test1,test2|SrcK8S_Namespace=ns'); + }); + + it('should generate OR groups, back and forth', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], + backAndForth: true + }, + true + ); + expect(grouped).toEqual('SrcK8S_Name=test1,test2|DstPort=443|DstK8S_Name=test1,test2|SrcPort=443'); + }); + + it('should generate flat OR groups, back and forth', () => { + const grouped = getDecodedFilter( + { + list: [ + filter('src_name', [{ v: 'test' }]), + filter('src_port', [{ v: '443' }]), + filter('src_kind', [{ v: 'Pod' }]), + filter('protocol', [{ v: '6' }]) + ], + backAndForth: true + }, + true + ); + expect(grouped).toEqual( + 'SrcK8S_Name=test|SrcPort=443|SrcK8S_Type=Pod|Proto=6|DstK8S_Name=test|DstPort=443|DstK8S_Type=Pod' + ); + }); + + it('should generate simple Src K8S resource', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: false }, + true + ); + expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"'); + }); + + it('should generate K8S resource, back and forth', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], backAndForth: true }, + true + ); + expect(grouped).toEqual( + 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + + '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' + ); + }); + + it('should generate Node K8S resource, back and forth', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'Node.test' }])], backAndForth: true }, + true + ); + expect(grouped).toEqual( + 'SrcK8S_Type="Node"&SrcK8S_Namespace=""&SrcK8S_Name="test"' + + '|DstK8S_Type="Node"&DstK8S_Namespace=""&DstK8S_Name="test"' + ); + }); + + it('should generate Owner K8S resource, back and forth', () => { + const grouped = getDecodedFilter( + { list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], backAndForth: true }, + true + ); + expect(grouped).toEqual( + 'SrcK8S_OwnerType="DaemonSet"&SrcK8S_Namespace="ns"&SrcK8S_OwnerName="test"' + + '|DstK8S_OwnerType="DaemonSet"&DstK8S_Namespace="ns"&DstK8S_OwnerName="test"' + ); + }); + + it('should generate K8S resource, back and forth, ORed with Name', () => { + const grouped = getDecodedFilter( + { + list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], + backAndForth: true + }, + true + ); + expect(grouped).toEqual( + 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + + '|DstK8S_Name=peer' + + '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' + + '|SrcK8S_Name=peer' + ); + }); +}); + +const getTopoForFilter = (filters: Filters, matchAny: boolean) => { + getFetchFunctions(filters, matchAny).getTopologyMetrics( + { + filters: filtersToString(filters.list, matchAny), + recordType: 'flowLog', + limit: 5, + packetLoss: 'all' + }, + 300 + ); +}; + +describe('Match all, topology', () => { + beforeEach(() => { + getTopologyMock.mockClear(); + }); + + it('should encode', () => { + getTopoForFilter({ list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], backAndForth: false }, false); + expect(getTopologyMock).toHaveBeenCalledTimes(1); + expect(getTopologyMock.mock.calls[0][0].filters).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); + }); + + it('should generate AND groups', () => { + getTopoForFilter( + { + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], + backAndForth: false + }, + false + ); + expect(getTopologyMock).toHaveBeenCalledTimes(1); + expect(decodeURIComponent(getTopologyMock.mock.calls[0][0].filters)).toEqual( + 'SrcK8S_Name=test1,test2&SrcK8S_Namespace=ns' + ); + }); + + it('should generate AND groups, back and forth', () => { + getTopoForFilter( + { + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], + backAndForth: true + }, + false + ); + expect(getTopologyMock).toHaveBeenCalledTimes(3); + expect(decodeURIComponent(getTopologyMock.mock.calls[0][0].filters)).toEqual('SrcK8S_Name=test1,test2&DstPort=443'); + expect(decodeURIComponent(getTopologyMock.mock.calls[1][0].filters)).toEqual('DstK8S_Name=test1,test2&SrcPort=443'); + expect(decodeURIComponent(getTopologyMock.mock.calls[2][0].filters)).toEqual( + 'SrcK8S_Name=test1,test2&DstPort=443&DstK8S_Name=test1,test2&SrcPort=443' + ); + }); +}); + +describe('Merge topology BNF', () => { + const range = { from: 0, to: 300 }; + const genNsMetric = ( + srcns: string | undefined, + dstns: string | undefined, + value1stHalf: number | undefined, + value2ndHalf: number | undefined + ): RawTopologyMetrics => { + const m: RawTopologyMetrics = { + metric: { SrcK8S_Namespace: srcns, DstK8S_Namespace: dstns }, + values: [] + }; + if (value1stHalf !== undefined) { + for (let i = 0; i < 10; i++) { + m.values.push([i * 15, value1stHalf]); + } + } + if (value2ndHalf !== undefined) { + for (let i = 10; i < 20; i++) { + m.values.push([i * 15, value2ndHalf]); + } + } + return m; + }; + + it('should merge without overlap', () => { + const rsOrig: TopologyMetricsResult = { + metrics: parseTopologyMetrics( + [ + genNsMetric('foo', 'bar', 10, 20), + genNsMetric('foo', 'foo', 10, 5), + genNsMetric('foo', undefined, 5, undefined) + ], + range, + 'bytes', + 'namespace', + 0, + true + ), + stats: { limitReached: true, numQueries: 2 } + }; + const rsSwap: TopologyMetricsResult = { + metrics: parseTopologyMetrics( + [genNsMetric('bar', 'foo', 1, 1), genNsMetric('foo', 'foo', 5, 5)], + range, + 'bytes', + 'namespace', + 0, + true + ), + stats: { limitReached: false, numQueries: 1 } + }; + + const merged = mergeTopologyMetricsBNF(range, rsOrig, rsSwap); + expect(merged.metrics).toHaveLength(4); + expect(merged.metrics[0].source.namespace).toEqual('foo'); + expect(merged.metrics[0].destination.namespace).toEqual('bar'); + expect(merged.metrics[0].stats).toEqual({ + avg: 15, + max: 20, + total: 4275, + latest: 20 + }); + expect(merged.metrics[1].source.namespace).toEqual('foo'); + expect(merged.metrics[1].destination.namespace).toEqual('foo'); + expect(merged.metrics[1].stats).toEqual({ + avg: 12.5, + max: 15, + total: 3562, + latest: 10 + }); + expect(merged.metrics[2].source.namespace).toEqual('foo'); + expect(merged.metrics[2].destination.namespace).toBeUndefined(); + expect(merged.metrics[2].stats).toEqual({ + avg: 2.63, + max: 5, + total: 710, + latest: 0 + }); + expect(merged.metrics[3].source.namespace).toEqual('bar'); + expect(merged.metrics[3].destination.namespace).toEqual('foo'); + expect(merged.metrics[3].stats).toEqual({ + avg: 1, + max: 1, + total: 285, + latest: 1 + }); + expect(merged.stats).toEqual({ limitReached: true, numQueries: 3 }); + }); + + it('should merge with overlap', () => { + const rsOrig: TopologyMetricsResult = { + metrics: parseTopologyMetrics( + [ + genNsMetric('foo', 'bar', 10, 20), + genNsMetric('foo', 'foo', 10, 5), + genNsMetric('foo', undefined, 5, undefined) + ], + range, + 'bytes', + 'namespace', + 0, + true + ), + stats: { limitReached: true, numQueries: 2 } + }; + const rsSwap: TopologyMetricsResult = { + metrics: parseTopologyMetrics( + [genNsMetric('bar', 'foo', 1, 1), genNsMetric('foo', 'foo', 5, 5)], + range, + 'bytes', + 'namespace', + 0, + true + ), + stats: { limitReached: false, numQueries: 1 } + }; + const rsOverlap: TopologyMetricsResult = { + metrics: parseTopologyMetrics([genNsMetric('foo', 'foo', 3, 3)], range, 'bytes', 'namespace', 0, true), + stats: { limitReached: false, numQueries: 1 } + }; + + const merged = mergeTopologyMetricsBNF(range, rsOrig, rsSwap, rsOverlap); + expect(merged.metrics).toHaveLength(4); + expect(merged.metrics[0].source.namespace).toEqual('foo'); + expect(merged.metrics[0].destination.namespace).toEqual('bar'); + expect(merged.metrics[0].stats).toEqual({ + avg: 15, + max: 20, + total: 4275, + latest: 20 + }); + expect(merged.metrics[1].source.namespace).toEqual('foo'); + expect(merged.metrics[1].destination.namespace).toEqual('foo'); + expect(merged.metrics[1].stats).toEqual({ + avg: 9.5, + max: 12, + total: 2707, + latest: 7 + }); + expect(merged.metrics[2].source.namespace).toEqual('foo'); + expect(merged.metrics[2].destination.namespace).toBeUndefined(); + expect(merged.metrics[2].stats).toEqual({ + avg: 2.63, + max: 5, + total: 710, + latest: 0 + }); + expect(merged.metrics[3].source.namespace).toEqual('bar'); + expect(merged.metrics[3].destination.namespace).toEqual('foo'); + expect(merged.metrics[3].stats).toEqual({ + avg: 1, + max: 1, + total: 285, + latest: 1 + }); + expect(merged.stats).toEqual({ limitReached: true, numQueries: 4 }); + }); +}); diff --git a/web/src/utils/back-and-forth.ts b/web/src/utils/back-and-forth.ts new file mode 100644 index 000000000..117edbb1b --- /dev/null +++ b/web/src/utils/back-and-forth.ts @@ -0,0 +1,99 @@ +import { RecordsResult, TopologyMetricsResult } from '../api/loki'; +import { getFlows, getTopologyMetrics } from '../api/routes'; +import { swapFilters } from '../components/filters/filters-helper'; +import { Filter, Filters } from '../model/filters'; +import { FlowQuery, filtersToString } from '../model/flow-query'; +import { TimeRange, computeStepInterval } from './datetime'; +import { mergeStats, substractMetrics, sumMetrics } from './metrics'; + +export const getFetchFunctions = (filters: Filters, matchAny: boolean) => { + // check back-and-forth + if (filters.backAndForth) { + const swapped = swap(filters.list, matchAny); + if (swapped.length > 0) { + return { + getFlows: (q: FlowQuery) => { + return getFlowsBNF(q, filters.list, swapped, matchAny); + }, + getTopologyMetrics: (q: FlowQuery, range: number | TimeRange) => { + return getTopologyMetricsBNF(q, range, filters.list, swapped, matchAny); + } + }; + } + } + return { + getFlows: getFlows, + getTopologyMetrics: getTopologyMetrics + }; +}; + +const encodedPipe = encodeURIComponent('|'); +const getFlowsBNF = ( + initialQuery: FlowQuery, + orig: Filter[], + swapped: Filter[], + matchAny: boolean +): Promise => { + // Combine original filters and swapped. Note that we leave any potential overlapping flows: they can be deduped with "showDuplicates: false". + const newFilters = filtersToString(orig, matchAny) + encodedPipe + filtersToString(swapped, matchAny); + return getFlows({ ...initialQuery, filters: newFilters }); +}; + +const getTopologyMetricsBNF = ( + initialQuery: FlowQuery, + range: number | TimeRange, + orig: Filter[], + swapped: Filter[], + matchAny: boolean +): Promise => { + // When bnf is on, this replaces the usual getTopologyMetrics with a function with same arguments that runs 3 queries and merge their results + // in order to get the ORIGINAL + SWAPPED - OVERLAP + // OVERLAP being ORIGINAL AND SWAPPED. + // E.g: if ORIGINAL is "SrcNs=foo", SWAPPED is "DstNs=foo" and OVERLAP is "SrcNs=foo AND DstNs=foo" + const overlapFilters = matchAny ? undefined : [...orig, ...swapped]; + const promOrig = getTopologyMetrics(initialQuery, range); + const promSwapped = getTopologyMetrics({ ...initialQuery, filters: filtersToString(swapped, matchAny) }, range); + const promOverlap = overlapFilters + ? getTopologyMetrics( + { + ...initialQuery, + filters: filtersToString(overlapFilters, matchAny) + }, + range + ) + : Promise.resolve(undefined); + return Promise.all([promOrig, promSwapped, promOverlap]).then(([rsOrig, rsSwapped, rsOverlap]) => + mergeTopologyMetricsBNF(range, rsOrig, rsSwapped, rsOverlap) + ); +}; + +// exported for testing +export const mergeTopologyMetricsBNF = ( + range: number | TimeRange, + rsOrig: TopologyMetricsResult, + rsSwapped: TopologyMetricsResult, + rsOverlap?: TopologyMetricsResult +): TopologyMetricsResult => { + const { stepSeconds } = computeStepInterval(range); + // Sum ORIGINAL + SWAPPED + const metrics = sumMetrics(rsOrig.metrics, rsSwapped.metrics, stepSeconds); + const stats = mergeStats(rsOrig.stats, rsSwapped.stats); + if (rsOverlap) { + // Substract OVERLAP + return { + metrics: substractMetrics(metrics, rsOverlap.metrics, stepSeconds), + stats: mergeStats(stats, rsOverlap.stats) + }; + } + return { metrics, stats }; +}; + +const swap = (filters: Filter[], matchAny: boolean): Filter[] => { + // include swapped traffic + const swapped = swapFilters((k: string) => k, filters); + if (matchAny) { + // In match-any mode, remove non-swappable filters as they would result in duplicates + return swapped.filter(f => f.def.id.startsWith('src_') || f.def.id.startsWith('dst_')); + } + return swapped; +}; diff --git a/web/src/utils/filter-definitions.ts b/web/src/utils/filter-definitions.ts index a825bfa54..ec87125aa 100644 --- a/web/src/utils/filter-definitions.ts +++ b/web/src/utils/filter-definitions.ts @@ -191,8 +191,7 @@ export const getFilterDefinitions = ( validate: k8sNameValidation, hint: k8sNameHint, examples: k8sNameExamples, - encoder: simpleFiltersEncoder('SrcK8S_Namespace'), - overlap: true + encoder: simpleFiltersEncoder('SrcK8S_Namespace') }, simpleFiltersEncoder('DstK8S_Namespace') ), @@ -206,8 +205,7 @@ export const getFilterDefinitions = ( validate: k8sNameValidation, hint: k8sNameHint, examples: k8sNameExamples, - encoder: simpleFiltersEncoder('SrcK8S_Name'), - overlap: false + encoder: simpleFiltersEncoder('SrcK8S_Name') }, simpleFiltersEncoder('DstK8S_Name') ), @@ -220,8 +218,7 @@ export const getFilterDefinitions = ( category: FilterCategory.Source, getOptions: cap10(getKindOptions), validate: rejectEmptyValue, - encoder: kindFiltersEncoder('SrcK8S_Type', 'SrcK8S_OwnerType'), - overlap: false + encoder: kindFiltersEncoder('SrcK8S_Type', 'SrcK8S_OwnerType') }, kindFiltersEncoder('DstK8S_Type', 'DstK8S_OwnerType') ), @@ -235,8 +232,7 @@ export const getFilterDefinitions = ( validate: k8sNameValidation, hint: k8sNameHint, examples: k8sNameExamples, - encoder: simpleFiltersEncoder('SrcK8S_OwnerName'), - overlap: true + encoder: simpleFiltersEncoder('SrcK8S_OwnerName') }, simpleFiltersEncoder('DstK8S_OwnerName') ), @@ -295,8 +291,7 @@ export const getFilterDefinitions = ( 'SrcK8S_Namespace', 'SrcK8S_Name', 'SrcK8S_OwnerName' - ), - overlap: false + ) }, k8sResourceFiltersEncoder( 'DstK8S_Type', @@ -321,8 +316,7 @@ export const getFilterDefinitions = ( }, hint: ipHint, examples: ipExamples, - encoder: simpleFiltersEncoder('SrcAddr'), - overlap: false + encoder: simpleFiltersEncoder('SrcAddr') }, simpleFiltersEncoder('DstAddr') ), @@ -348,8 +342,7 @@ export const getFilterDefinitions = ( - ${t('A port number like 80, 21')} - ${t('A IANA name like HTTP, FTP')}`, docUrl: 'https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml', - encoder: simpleFiltersEncoder('SrcPort'), - overlap: false + encoder: simpleFiltersEncoder('SrcPort') }, simpleFiltersEncoder('DstPort') ), @@ -367,8 +360,7 @@ export const getFilterDefinitions = ( return /^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})/.test(value) ? valid(value) : invalid(invalidMACMessage); }, hint: t('Specify a single MAC address.'), - encoder: simpleFiltersEncoder('SrcMac'), - overlap: false + encoder: simpleFiltersEncoder('SrcMac') }, simpleFiltersEncoder('DstMac') ), @@ -387,8 +379,7 @@ export const getFilterDefinitions = ( }, hint: ipHint, examples: ipExamples, - encoder: simpleFiltersEncoder('SrcK8S_HostIP'), - overlap: true + encoder: simpleFiltersEncoder('SrcK8S_HostIP') }, simpleFiltersEncoder('DstK8S_HostIP') ), @@ -402,8 +393,7 @@ export const getFilterDefinitions = ( validate: k8sNameValidation, hint: k8sNameHint, examples: k8sNameExamples, - encoder: simpleFiltersEncoder('SrcK8S_HostName'), - overlap: true + encoder: simpleFiltersEncoder('SrcK8S_HostName') }, simpleFiltersEncoder('DstK8S_HostName') ), @@ -434,8 +424,7 @@ export const getFilterDefinitions = ( - ${t('A IANA name like TCP, UDP')} - ${t('Empty double quotes "" for undefined protocol')}`, docUrl: 'https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml', - encoder: simpleFiltersEncoder('Proto'), - overlap: false + encoder: simpleFiltersEncoder('Proto') }, { id: 'direction', @@ -455,8 +444,7 @@ export const getFilterDefinitions = ( return invalid(t('Unknown direction')); }, hint: t('Specify the direction of the Flow observed at the Node observation point.'), - encoder: simpleFiltersEncoder('FlowDirection'), - overlap: false + encoder: simpleFiltersEncoder('FlowDirection') }, { id: 'interface', @@ -466,8 +454,7 @@ export const getFilterDefinitions = ( getOptions: noOption, validate: rejectEmptyValue, hint: t('Specify a network interface.'), - encoder: simpleFiltersEncoder('Interface'), - overlap: false + encoder: simpleFiltersEncoder('Interface') }, { id: 'id', @@ -477,8 +464,7 @@ export const getFilterDefinitions = ( getOptions: noOption, validate: rejectEmptyValue, hint: t('Specify a single conversation hash Id.'), - encoder: simpleFiltersEncoder('_HashId'), - overlap: false + encoder: simpleFiltersEncoder('_HashId') }, { id: 'pkt_drop_state', @@ -492,8 +478,7 @@ export const getFilterDefinitions = ( - ${t('A _LINUX_TCP_STATES_H number like 1, 2, 3')} - ${t('A _LINUX_TCP_STATES_H TCP name like ESTABLISHED, SYN_SENT, SYN_RECV')}`, docUrl: 'https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h', - encoder: simpleFiltersEncoder('PktDropLatestState'), - overlap: false + encoder: simpleFiltersEncoder('PktDropLatestState') }, { id: 'pkt_drop_cause', @@ -507,8 +492,7 @@ export const getFilterDefinitions = ( - ${t('A _LINUX_DROPREASON_CORE_H number like 2, 3, 4')} - ${t('A _LINUX_DROPREASON_CORE_H SKB_DROP_REASON name like NOT_SPECIFIED, NO_SOCKET, PKT_TOO_SMALL')}`, docUrl: 'https://github.com/torvalds/linux/blob/master/include/net/dropreason-core.h', - encoder: simpleFiltersEncoder('PktDropLatestDropCause'), - overlap: false + encoder: simpleFiltersEncoder('PktDropLatestDropCause') }, { id: 'dns_id', @@ -518,8 +502,7 @@ export const getFilterDefinitions = ( getOptions: noOption, validate: rejectEmptyValue, hint: t('Specify a single DNS Id.'), - encoder: simpleFiltersEncoder('DnsId'), - overlap: false + encoder: simpleFiltersEncoder('DnsId') }, { id: 'dns_latency', @@ -529,8 +512,7 @@ export const getFilterDefinitions = ( getOptions: noOption, validate: rejectEmptyValue, hint: t('Specify a DNS Latency in miliseconds.'), - encoder: simpleFiltersEncoder('DnsLatencyMs'), - overlap: false + encoder: simpleFiltersEncoder('DnsLatencyMs') }, { id: 'dns_flag_response_code', @@ -544,8 +526,7 @@ export const getFilterDefinitions = ( - ${t('A IANA RCODE number like 0, 3, 9')} - ${t('A IANA RCODE name like NoError, NXDomain, NotAuth')}`, docUrl: 'https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6', - encoder: simpleFiltersEncoder('DnsFlagsResponseCode'), - overlap: false + encoder: simpleFiltersEncoder('DnsFlagsResponseCode') }, { id: 'time_flow_rtt', @@ -555,8 +536,7 @@ export const getFilterDefinitions = ( getOptions: noOption, validate: rejectEmptyValue, hint: t('Specify a Flow Round Trip Time in nanoseconds.'), - encoder: simpleFiltersEncoder('TimeFlowRttNs'), - overlap: false + encoder: simpleFiltersEncoder('TimeFlowRttNs') } ]; } diff --git a/web/src/utils/metrics.ts b/web/src/utils/metrics.ts index 60bd829aa..f0b058fa6 100644 --- a/web/src/utils/metrics.ts +++ b/web/src/utils/metrics.ts @@ -5,7 +5,8 @@ import { TopologyMetricPeer, TopologyMetrics, NameAndType, - GenericMetric + GenericMetric, + Stats } from '../api/loki'; import { MetricFunction, MetricType, FlowScope, GenericAggregation } from '../model/flow-query'; import { roundTwoDigits } from './count'; @@ -231,6 +232,15 @@ export const calibrateRange = ( } } + // Extend normalization interval to latest timestamp if bigger than computed endWithTolerance + const allLasts = raw.filter(dp => dp.length > 0).map(dp => dp[dp.length - 1][0]); + if (allLasts.length > 0) { + const lastTimestamp = Math.max(...allLasts); + if (lastTimestamp > endWithTolerance) { + endWithTolerance = lastTimestamp; + } + } + // End time needs to be overridden to avoid huge range since mock is outdated compared to current date if (isMock) { endWithTolerance = Math.max(...raw.filter(dp => dp.length > 0).map(dp => dp[dp.length - 1][0])); @@ -262,9 +272,8 @@ export const normalizeMetrics = ( }); // Normalize by filling missing datapoints with zeros - const tolerance = step / 2; for (let current = start; current < end; current += step) { - if (!normalized.some(rv => rv[0] > current - tolerance && rv[0] < current + tolerance)) { + if (!getValueCloseTo(normalized, current, step)) { normalized.push([current, 0]); } } @@ -272,6 +281,12 @@ export const normalizeMetrics = ( return normalized.sort((a, b) => a[0] - b[0]); }; +const getValueCloseTo = (values: [number, number][], timestamp: number, step: number): number | undefined => { + const tolerance = step / 2; + const datapoint = values.find(dp => dp[0] > timestamp - tolerance && dp[0] < timestamp + tolerance); + return datapoint ? datapoint[1] : undefined; +}; + /** * computeStats computes avg, max and total. Input metric is always the bytes rate (Bps). */ @@ -341,3 +356,72 @@ export const matchPeer = (data: NodeData, peer: TopologyMetricPeer): boolean => }; export const isUnknownPeer = (peer: TopologyMetricPeer): boolean => peer.id === idUnknown; + +const combineValues = ( + values1: [number, number][], + values2: [number, number][], + step: number, + op: (a: number, b: number) => number +): [number, number][] => { + return values1.map(dp1 => { + const t = dp1[0]; + const v1 = dp1[1]; + const v2 = getValueCloseTo(values2, t, step); + if (v2 === undefined) { + // shouldn't happen in theory since metrics are normalized, except on end timerange boundary + return [t, op(v1, 0)]; + } + return [t, op(v1, v2)]; + }); +}; + +const combineMetrics = ( + metrics1: TopologyMetrics[], + metrics2: TopologyMetrics[], + step: number, + op: (a: number, b: number) => number, + ignoreAbsentMetric?: boolean +): TopologyMetrics[] => { + const cache: Map = new Map(); + const keyFunc = (m: TopologyMetrics) => `${m.source.id}@${m.destination.id}`; + metrics1.forEach(m => { + cache.set(keyFunc(m), m); + }); + metrics2.forEach(m => { + const inCache = cache.get(keyFunc(m)); + if (inCache) { + inCache.values = combineValues(inCache.values, m.values, step, op); + inCache.stats = computeStats(inCache.values); + } else if (!ignoreAbsentMetric) { + cache.set(keyFunc(m), m); + } + }); + return Array.from(cache.values()); +}; + +export const sumMetrics = ( + metrics1: TopologyMetrics[], + metrics2: TopologyMetrics[], + step: number +): TopologyMetrics[] => { + return combineMetrics(metrics1, metrics2, step, (a, b) => a + b); +}; + +export const substractMetrics = ( + metrics1: TopologyMetrics[], + metrics2: TopologyMetrics[], + step: number +): TopologyMetrics[] => { + return combineMetrics(metrics1, metrics2, step, (a, b) => a - b, true); +}; + +export const mergeStats = (prev: Stats | undefined, current: Stats): Stats => { + if (!prev) { + return current; + } + return { + ...prev, + limitReached: prev.limitReached || current.limitReached, + numQueries: prev.numQueries + current.numQueries + }; +};