NETOBSERV-1274 index duplicate (improve query perfs) (#380)

* NETOBSERV-1274 index duplicate (improve query perfs) * Move Type from field to label
netobserv · Sep 27, 2023 · 2804fd2 · 2804fd2
1 parent 82c4ac9
commit 2804fd2
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 32 deletions.
diff --git a/pkg/loki/flow_query.go b/pkg/loki/flow_query.go
@@ -56,13 +56,17 @@ func NewFlowQueryBuilder(cfg *Config, start, end, limit string, dedup bool,
 
 	lineFilters := []lineFilter{}
 	if dedup {
-		lineFilters = append(lineFilters, lineFilter{
-			key: fields.Duplicate,
-			values: []lineMatch{{
-				valueType: typeBool,
-				value:     "false",
-			}},
-		})
+		if cfg.IsLabel(fields.Duplicate) {
+			labelFilters = append(labelFilters, stringEqualLabelFilter(fields.Duplicate, "false"))
+		} else {
+			lineFilters = append(lineFilters, lineFilter{
+				key: fields.Duplicate,
+				values: []lineMatch{{
+					valueType: typeBool,
+					value:     "false",
+				}},
+			})
+		}
 	}
 
 	if packetLoss == constants.PacketLossDropped {

diff --git a/web/src/api/ipfix.ts b/web/src/api/ipfix.ts
@@ -20,6 +20,10 @@ export interface Labels {
   SrcK8S_OwnerName?: string;
   /** Destination owner, such as Deployment, StatefulSet, etc. */
   DstK8S_OwnerName?: string;
+  /** Kind of the source matched Kubernetes object, such as Pod, Service, etc. */
+  SrcK8S_Type?: string;
+  /** Kind of the destination matched Kubernetes object, such as Pod name, Service name, etc. */
+  DstK8S_Type?: string;
   /** Flow direction from the node observation point */
   FlowDirection: FlowDirection;
   /** Type of record: 'flowLog' for regular flow logs, or 'allConnections',
@@ -56,10 +60,6 @@ export interface Fields {
   SrcK8S_Name?: string;
   /** Name of the destination matched Kubernetes object, such as Pod name, Service name, etc. */
   DstK8S_Name?: string;
-  /** Kind of the source matched Kubernetes object, such as Pod, Service, etc. */
-  SrcK8S_Type?: string;
-  /** Kind of the destination matched Kubernetes object, such as Pod name, Service name, etc. */
-  DstK8S_Type?: string;
   /** Source port */
   SrcPort?: number;
   /** Destination port */

diff --git a/web/src/components/netflow-record/record-field.tsx b/web/src/components/netflow-record/record-field.tsx
@@ -235,13 +235,13 @@ export const RecordField: React.FC<{
         );
       case ColumnsId.name:
         return doubleContainer(
-          kubeObjContent(flow.fields.SrcK8S_Name, flow.fields.SrcK8S_Type, flow.labels.SrcK8S_Namespace),
-          kubeObjContent(flow.fields.DstK8S_Name, flow.fields.DstK8S_Type, flow.labels.DstK8S_Namespace)
+          kubeObjContent(flow.fields.SrcK8S_Name, flow.labels.SrcK8S_Type, flow.labels.SrcK8S_Namespace),
+          kubeObjContent(flow.fields.DstK8S_Name, flow.labels.DstK8S_Type, flow.labels.DstK8S_Namespace)
         );
       case ColumnsId.srcname:
-        return singleContainer(kubeObjContent(value as string, flow.fields.SrcK8S_Type, flow.labels.SrcK8S_Namespace));
+        return singleContainer(kubeObjContent(value as string, flow.labels.SrcK8S_Type, flow.labels.SrcK8S_Namespace));
       case ColumnsId.dstname:
-        return singleContainer(kubeObjContent(value as string, flow.fields.DstK8S_Type, flow.labels.DstK8S_Namespace));
+        return singleContainer(kubeObjContent(value as string, flow.labels.DstK8S_Type, flow.labels.DstK8S_Namespace));
       case ColumnsId.owner:
         return doubleContainer(
           kubeObjContent(flow.labels.SrcK8S_OwnerName, flow.fields.SrcK8S_OwnerType, flow.labels.SrcK8S_Namespace),
@@ -269,14 +269,14 @@ export const RecordField: React.FC<{
           explicitKubeObjContent(
             flow.fields.SrcAddr,
             flow.fields.SrcPort || NaN,
-            flow.fields.SrcK8S_Type,
+            flow.labels.SrcK8S_Type,
             flow.labels.SrcK8S_Namespace,
             flow.fields.SrcK8S_Name
           ),
           explicitKubeObjContent(
             flow.fields.DstAddr,
             flow.fields.DstPort || NaN,
-            flow.fields.DstK8S_Type,
+            flow.labels.DstK8S_Type,
             flow.labels.DstK8S_Namespace,
             flow.fields.DstK8S_Name
           )
@@ -286,7 +286,7 @@ export const RecordField: React.FC<{
           explicitKubeObjContent(
             flow.fields.SrcAddr,
             flow.fields.SrcPort || NaN,
-            flow.fields.SrcK8S_Type,
+            flow.labels.SrcK8S_Type,
             flow.labels.SrcK8S_Namespace,
             flow.fields.SrcK8S_Name
           )
@@ -296,7 +296,7 @@ export const RecordField: React.FC<{
           explicitKubeObjContent(
             flow.fields.DstAddr,
             flow.fields.DstPort || NaN,
-            flow.fields.DstK8S_Type,
+            flow.labels.DstK8S_Type,
             flow.labels.DstK8S_Namespace,
             flow.fields.DstK8S_Name
           )

diff --git a/web/src/components/netflow-record/record-panel.tsx b/web/src/components/netflow-record/record-panel.tsx
@@ -231,15 +231,15 @@ export const RecordPanel: React.FC<RecordDrawerProps> = ({
     columns.filter(
       c =>
         //remove empty / duplicates columns for Node
-        (record?.fields.SrcK8S_Type !== 'Node' ||
+        (record?.labels.SrcK8S_Type !== 'Node' ||
           ![
             ColumnsId.srcnamespace,
             ColumnsId.srcowner,
             ColumnsId.srcownertype,
             ColumnsId.srchostaddr,
             ColumnsId.srchostname
           ].includes(c.id)) &&
-        (record?.fields.DstK8S_Type !== 'Node' ||
+        (record?.labels.DstK8S_Type !== 'Node' ||
           ![
             ColumnsId.dstnamespace,
             ColumnsId.dstowner,

diff --git a/web/src/components/query-summary/summary-panel.tsx b/web/src/components/query-summary/summary-panel.tsx
@@ -165,13 +165,13 @@ export const SummaryPanelContent: React.FC<{
 
     if (flows && flows.length) {
       //list all types
-      const types = Array.from(new Set(flows.flatMap(f => [f.fields.SrcK8S_Type, f.fields.DstK8S_Type])));
+      const types = Array.from(new Set(flows.flatMap(f => [f.labels.SrcK8S_Type, f.labels.DstK8S_Type])));
       types
         .filter((t: string | undefined) => t !== undefined)
         .forEach((type: string) => {
           const tc: TypeCardinality = { type, objects: [] };
 
-          const typeFilteredFlows = flows.filter(f => [f.fields.SrcK8S_Type, f.fields.DstK8S_Type].includes(type));
+          const typeFilteredFlows = flows.filter(f => [f.labels.SrcK8S_Type, f.labels.DstK8S_Type].includes(type));
           //list all namespaces of this type
           const typeNamespaces = new Set(
             typeFilteredFlows.flatMap(f => [f.labels.SrcK8S_Namespace, f.labels.DstK8S_Namespace])
@@ -189,14 +189,14 @@ export const SummaryPanelContent: React.FC<{
             //add all names of this namespace of type
             namespaceFilteredFlows.forEach(record => {
               const srcName =
-                record.fields.SrcK8S_Type === type && record.labels.SrcK8S_Namespace === namespace
+                record.labels.SrcK8S_Type === type && record.labels.SrcK8S_Namespace === namespace
                   ? record.fields.SrcK8S_Name
                   : undefined;
               if (srcName && !nsObject.names.includes(srcName)) {
                 nsObject.names.push(srcName);
               }
               const dstName =
-                record.fields.DstK8S_Type === type && record.labels.DstK8S_Namespace === namespace
+                record.labels.DstK8S_Type === type && record.labels.DstK8S_Namespace === namespace
                   ? record.fields.DstK8S_Name
                   : undefined;
               if (dstName && !nsObject.names.includes(dstName)) {

diff --git a/web/src/utils/columns.ts b/web/src/utils/columns.ts
@@ -207,7 +207,7 @@ export const getCommonColumns = (t: TFunction, withConcatenatedFields = true): C
       id: ColumnsId.type,
       name: t('Kinds'),
       isSelected: false,
-      value: f => getSrcOrDstValue(f.fields.SrcK8S_Type, f.fields.DstK8S_Type),
+      value: f => getSrcOrDstValue(f.labels.SrcK8S_Type, f.labels.DstK8S_Type),
       sort: (a, b, col) => compareStrings((col.value(a) as string[]).join(''), (col.value(b) as string[]).join('')),
       width: 10
     },
@@ -292,14 +292,14 @@ export const getCommonColumns = (t: TFunction, withConcatenatedFields = true): C
           ...getConcatenatedValue(
             f.fields.SrcAddr,
             f.fields.SrcPort || NaN,
-            f.fields.SrcK8S_Type,
+            f.labels.SrcK8S_Type,
             f.labels.SrcK8S_Namespace,
             f.fields.SrcK8S_Name
           ),
           ...getConcatenatedValue(
             f.fields.DstAddr,
             f.fields.DstPort || NaN,
-            f.fields.DstK8S_Type,
+            f.labels.DstK8S_Type,
             f.labels.DstK8S_Namespace,
             f.fields.DstK8S_Name
           )
@@ -374,7 +374,7 @@ export const getSrcColumns = (t: TFunction): Column[] => {
       fieldName: 'SrcK8S_Type',
       quickFilter: 'src_kind',
       isSelected: false,
-      value: f => f.fields.SrcK8S_Type || '',
+      value: f => f.labels.SrcK8S_Type || '',
       sort: (a, b, col) => compareStrings(col.value(a) as string, col.value(b) as string),
       width: 10
     },
@@ -514,7 +514,7 @@ export const getDstColumns = (t: TFunction): Column[] => {
       fieldName: 'DstK8S_Type',
       quickFilter: 'dst_kind',
       isSelected: false,
-      value: f => f.fields.DstK8S_Type || '',
+      value: f => f.labels.DstK8S_Type || '',
       sort: (a, b, col) => compareStrings(col.value(a) as string, col.value(b) as string),
       width: 10
     },
@@ -640,7 +640,7 @@ export const getSrcDstColumns = (t: TFunction, withConcatenatedFields = true): C
           getConcatenatedValue(
             f.fields.SrcAddr,
             f.fields.SrcPort || NaN,
-            f.fields.SrcK8S_Type,
+            f.labels.SrcK8S_Type,
             f.labels.SrcK8S_Namespace,
             f.fields.SrcK8S_Name
           ),
@@ -682,7 +682,7 @@ export const getSrcDstColumns = (t: TFunction, withConcatenatedFields = true): C
           getConcatenatedValue(
             f.fields.DstAddr,
             f.fields.DstPort || NaN,
-            f.fields.DstK8S_Type,
+            f.labels.DstK8S_Type,
             f.labels.DstK8S_Namespace,
             f.fields.DstK8S_Name
           ),