NETOBSERV-1610: document cardinality warning per field (#623)

* NETOBSERV-1610: document cardinality warning per field * Update docs/flows-format.adoc Co-authored-by: Sara Thomas <[email protected]> --------- Co-authored-by: Sara Thomas <[email protected]>
netobserv · Apr 19, 2024 · ba8726e · ba8726e
1 parent b26356d
commit ba8726e
Show file tree

Hide file tree

Showing 6 changed files with 172 additions and 17 deletions.
diff --git a/controllers/consoleplugin/config/config.go b/controllers/consoleplugin/config/config.go
@@ -64,12 +64,21 @@ type FilterConfig struct {
 	Placeholder            string `yaml:"placeholder,omitempty" json:"placeholder,omitempty"`
 }
 
+type CardinalityWarn string
+
+const (
+	CardinalityWarnAvoid   CardinalityWarn = "avoid"
+	CardinalityWarnCareful CardinalityWarn = "careful"
+	CardinalityWarnFine    CardinalityWarn = "fine"
+)
+
 type FieldConfig struct {
-	Name        string `yaml:"name" json:"name"`
-	Type        string `yaml:"type" json:"type"`
-	Description string `yaml:"description" json:"description"`
-	LokiLabel   bool   `yaml:"lokiLabel,omitempty" json:"lokiLabel,omitempty"`
-	Filter      string `yaml:"filter,omitempty" json:"filter,omitempty"`
+	Name            string          `yaml:"name" json:"name"`
+	Type            string          `yaml:"type" json:"type"`
+	Description     string          `yaml:"description" json:"description"`
+	LokiLabel       bool            `yaml:"lokiLabel,omitempty" json:"lokiLabel,omitempty"`
+	Filter          string          `yaml:"filter,omitempty" json:"filter,omitempty"`
+	CardinalityWarn CardinalityWarn `yaml:"cardinalityWarn,omitempty" json:"cardinalityWarn,omitempty"`
 }
 
 type Deduper struct {

diff --git a/controllers/consoleplugin/config/static-frontend-config.yaml b/controllers/consoleplugin/config/static-frontend-config.yaml
@@ -862,115 +862,149 @@ filters:
     hint: Specify a TCP handshake Round Trip Time in nanoseconds.
 
 # Fields definition, used to generate documentation
+# The "cardinalityWarn" property relates to how the field is suitable for usage as a metric label wrt cardinality; it may have 3 values: fine, careful, avoid
 fields:
   - name: TimeFlowStartMs
     type: number
     description: Start timestamp of this flow, in milliseconds
+    cardinalityWarn: avoid
   - name: TimeFlowEndMs
     type: number
     description: End timestamp of this flow, in milliseconds
+    cardinalityWarn: avoid
   - name: TimeReceived
     type: number
     description: Timestamp when this flow was received and processed by the flow collector, in seconds
+    cardinalityWarn: avoid
   - name: SrcK8S_Name
     type: string
     description: Name of the source Kubernetes object, such as Pod name, Service name or Node name.
+    cardinalityWarn: careful
   - name: SrcK8S_Type
     type: string
     description: Kind of the source Kubernetes object, such as Pod, Service or Node.
     lokiLabel: true
+    cardinalityWarn: fine
   - name: SrcK8S_OwnerName
     type: string
     description: Name of the source owner, such as Deployment name, StatefulSet name, etc.
     lokiLabel: true
+    cardinalityWarn: fine
   - name: SrcK8S_OwnerType
     type: string
     description: Kind of the source owner, such as Deployment, StatefulSet, etc.
+    cardinalityWarn: fine
   - name: SrcK8S_Namespace
     type: string
     description: Source namespace
     lokiLabel: true
+    cardinalityWarn: fine
   - name: SrcAddr
     type: string
     description: Source IP address (ipv4 or ipv6)
+    cardinalityWarn: avoid
   - name: SrcPort
     type: number
     description: Source port
+    cardinalityWarn: careful
   - name: SrcMac
     type: string
     description: Source MAC address
+    cardinalityWarn: avoid
   - name: SrcK8S_HostIP
     type: string
     description: Source node IP
+    cardinalityWarn: fine
   - name: SrcK8S_HostName
     type: string
     description: Source node name
+    cardinalityWarn: fine
   - name: SrcK8S_Zone
     type: string
     description: Source availability zone
     lokiLabel: true
+    cardinalityWarn: fine
   - name: SrcSubnetLabel
     type: string
     description: Source subnet label
+    cardinalityWarn: fine
   - name: DstK8S_Name
     type: string
     description: Name of the destination Kubernetes object, such as Pod name, Service name or Node name.
+    cardinalityWarn: careful
   - name: DstK8S_Type
     type: string
     description: Kind of the destination Kubernetes object, such as Pod, Service or Node.
     lokiLabel: true
+    cardinalityWarn: fine
   - name: DstK8S_OwnerName
     type: string
     description: Name of the destination owner, such as Deployment name, StatefulSet name, etc.
     lokiLabel: true
+    cardinalityWarn: fine
   - name: DstK8S_OwnerType
     type: string
     description: Kind of the destination owner, such as Deployment, StatefulSet, etc.
+    cardinalityWarn: fine
   - name: DstK8S_Namespace
     type: string
     description: Destination namespace
     lokiLabel: true
+    cardinalityWarn: fine
   - name: DstAddr
     type: string
     description: Destination IP address (ipv4 or ipv6)
+    cardinalityWarn: avoid
   - name: DstPort
     type: number
     description: Destination port
+    cardinalityWarn: careful
   - name: DstMac
     type: string
     description: Destination MAC address
+    cardinalityWarn: avoid
   - name: DstK8S_HostIP
     type: string
     description: Destination node IP
+    cardinalityWarn: fine
   - name: DstK8S_HostName
     type: string
     description: Destination node name
+    cardinalityWarn: fine
   - name: DstK8S_Zone
     type: string
     description: Destination availability zone
     lokiLabel: true
+    cardinalityWarn: fine
   - name: DstSubnetLabel
     type: string
     description: Destination subnet label
+    cardinalityWarn: fine
   - name: K8S_FlowLayer
     type: string
     description: "Flow layer: 'app' or 'infra'"
+    cardinalityWarn: fine
   - name: Proto
     type: number
     description: L4 protocol
+    cardinalityWarn: fine
   - name: Dscp
     type: number
     description: Differentiated Services Code Point (DSCP) value
+    cardinalityWarn: fine
   - name: IcmpType
     type: number
     description: ICMP type
+    cardinalityWarn: fine
   - name: IcmpCode
     type: number
     description: ICMP code
+    cardinalityWarn: fine
   - name: Duplicate
     type: boolean
     description: Indicates if this flow was also captured from another interface on the same host
     lokiLabel: true
+    cardinalityWarn: fine
   - name: FlowDirection
     type: number
     description: |
@@ -979,71 +1013,91 @@ fields:
       - 1: Egress (outgoing traffic, from the node observation point) +
       - 2: Inner (with the same source and destination node)
     lokiLabel: true
+    cardinalityWarn: fine
   - name: IfDirections
     type: number
     description: |
       Flow directions from the network interface observation point. Can be one of: +
       - 0: Ingress (interface incoming traffic) +
       - 1: Egress (interface outgoing traffic)
+    cardinalityWarn: fine
   - name: Interfaces
     type: string
     description: Network interfaces
+    cardinalityWarn: careful
   - name: Flags
     type: number
     description: |
       Logical OR combination of unique TCP flags comprised in the flow, as per RFC-9293, with additional custom flags to represent the following per-packet combinations: +
       - SYN+ACK (0x100) +
       - FIN+ACK (0x200) +
       - RST+ACK (0x400)
+    cardinalityWarn: fine
   - name: Bytes
     type: number
     description: Number of bytes
+    cardinalityWarn: avoid
   - name: Packets
     type: number
     description: Number of packets
+    cardinalityWarn: avoid
   - name: PktDropBytes
     type: number
     description: Number of bytes dropped by the kernel
+    cardinalityWarn: avoid
   - name: PktDropPackets
     type: number
     description: Number of packets dropped by the kernel
+    cardinalityWarn: avoid
   - name: PktDropLatestState
     type: string
     description: TCP state on last dropped packet
     filter: pkt_drop_state # couldn't guess from config
+    cardinalityWarn: fine
   - name: PktDropLatestDropCause
     type: string
     description: Latest drop cause
     filter: pkt_drop_cause # couldn't guess from config
+    cardinalityWarn: fine
   - name: PktDropLatestFlags
     type: number
     description: TCP flags on last dropped packet
+    cardinalityWarn: fine
   - name: DnsId
     type: number
     description: DNS record id
+    cardinalityWarn: avoid
   - name: DnsLatencyMs
     type: number
     description: Time between a DNS request and response, in milliseconds
+    cardinalityWarn: avoid
   - name: DnsFlags
     type: number
     description: DNS flags for DNS record
+    cardinalityWarn: fine
   - name: DnsFlagsResponseCode
     type: string
     description: Parsed DNS header RCODEs name
+    cardinalityWarn: fine
   - name: DnsErrno
     type: number
     description: Error number returned from DNS tracker ebpf hook function
+    cardinalityWarn: fine
   - name: TimeFlowRttNs
     type: number
     description: TCP Smoothed Round Trip Time (SRTT), in nanoseconds
+    cardinalityWarn: avoid
   - name: K8S_ClusterName
     type: string
     description: Cluster name or identifier
     lokiLabel: true
+    cardinalityWarn: fine
   - name: _RecordType
     type: string
     description: "Type of record: 'flowLog' for regular flow logs, or 'newConnection', 'heartbeat', 'endConnection' for conversation tracking"
     lokiLabel: true
+    cardinalityWarn: fine
   - name: _HashId
     type: string
     description: In conversation tracking, the conversation identifier
+    cardinalityWarn: avoid
diff --git a/controllers/consoleplugin/consoleplugin_test.go b/controllers/consoleplugin/consoleplugin_test.go
@@ -2,6 +2,7 @@ package consoleplugin
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 
 	promConfig "github.com/prometheus/common/config"
@@ -534,3 +535,25 @@ func TestNoMissingFields(t *testing.T) {
 	}
 	assert.Empty(t, missing, "Missing fields should be added in static config file, under 'fields'")
 }
+
+func TestFieldsCardinalityWarns(t *testing.T) {
+	var cfg config.FrontendConfig
+	err := yaml.Unmarshal(staticFrontendConfig, &cfg)
+	assert.NoError(t, err)
+
+	allowed := []config.CardinalityWarn{config.CardinalityWarnAvoid, config.CardinalityWarnCareful, config.CardinalityWarnFine}
+	mapCardinality := map[string]config.CardinalityWarn{}
+	for _, field := range cfg.Fields {
+		assert.Containsf(t, allowed, field.CardinalityWarn, "Field %s: cardinalityWarn '%s' is invalid", field.Name, field.CardinalityWarn)
+		mapCardinality[field.Name] = field.CardinalityWarn
+	}
+
+	for name, card := range mapCardinality {
+		if strings.HasPrefix(name, "Src") {
+			base := strings.TrimPrefix(name, "Src")
+			dst, ok := mapCardinality["Dst"+base]
+			assert.True(t, ok)
+			assert.Equalf(t, card, dst, "Cardinality for %s and %s differs", name, "Dst"+base)
+		}
+	}
+}