NETOBSERV-1326: NETOBSERV-1231: Drops & RTT metrics

api/v1beta1/flowcollector_webhook_test.go

@@ -118,14 +118,14 @@ func TestBeta1ConversionRoundtrip_Metrics(t *testing.T) {
 
 	assert.Equal([]v1beta2.FLPAlert{v1beta2.AlertLokiError}, converted.Spec.Processor.Metrics.DisableAlerts)
 	assert.NotNil(converted.Spec.Processor.Metrics.IncludeList)
-	assert.Equal([]string{"namespace_egress_packets_total", "namespace_flows_total"}, *converted.Spec.Processor.Metrics.IncludeList)
+	assert.Equal([]string{"namespace_egress_packets_total", "namespace_flows_total", "namespace_rtt_seconds", "namespace_drop_packets_total"}, *converted.Spec.Processor.Metrics.IncludeList)
 
 	// Other way
 	var back FlowCollector
 	err = back.ConvertFrom(&converted)
 	assert.NoError(err)
 	// Here, includeList is preserved; it takes precedence over ignoreTags
-	assert.Equal([]string{"namespace_egress_packets_total", "namespace_flows_total"}, *back.Spec.Processor.Metrics.IncludeList)
+	assert.Equal([]string{"namespace_egress_packets_total", "namespace_flows_total", "namespace_rtt_seconds", "namespace_drop_packets_total"}, *back.Spec.Processor.Metrics.IncludeList)
 	assert.Equal(initial.Spec.Processor.Metrics.DisableAlerts, back.Spec.Processor.Metrics.DisableAlerts)
 	assert.Equal(initial.Spec.Processor.Metrics.Server, back.Spec.Processor.Metrics.Server)
 }

api/v1beta2/flowcollector_types.go

@@ -351,6 +351,11 @@ type FLPMetrics struct {
 	// `namespace_ingress_packets_total`, `namespace_flows_total`, `node_egress_bytes_total`, `node_egress_packets_total`,
 	// `node_ingress_bytes_total`, `node_ingress_packets_total`, `node_flows_total`, `workload_egress_bytes_total`,
 	// `workload_egress_packets_total`, `workload_ingress_bytes_total`, `workload_ingress_packets_total`, `workload_flows_total`.
+	// When the `PacketDrop` feature is enabled in `spec.agent.ebpf.features`, these additional metrics are available:
+	// `namespace_drop_bytes_total`, `namespace_drop_packets_total`, `node_drop_bytes_total`, `node_drop_packets_total`,
+	// `workload_drop_bytes_total`, `workload_drop_packets_total`.
+	// When the `FlowRTT` feature is enabled in `spec.agent.ebpf.features`, these additional metrics are available:
+	// `namespace_rtt_seconds`, `node_rtt_seconds`, `workload_rtt_seconds`.
 	// +optional
 	IncludeList *[]string `json:"includeList,omitempty"`
 

bundle/manifests/flows.netobserv.io_flowcollectors.yaml

@@ -7714,7 +7714,14 @@ spec:
                           `node_ingress_bytes_total`, `node_ingress_packets_total`,
                           `node_flows_total`, `workload_egress_bytes_total`, `workload_egress_packets_total`,
                           `workload_ingress_bytes_total`, `workload_ingress_packets_total`,
-                          `workload_flows_total`.'
+                          `workload_flows_total`. When the `PacketDrop` feature is
+                          enabled in `spec.agent.ebpf.features`, these additional
+                          metrics are available: `namespace_drop_bytes_total`, `namespace_drop_packets_total`,
+                          `node_drop_bytes_total`, `node_drop_packets_total`, `workload_drop_bytes_total`,
+                          `workload_drop_packets_total`. When the `FlowRTT` feature
+                          is enabled in `spec.agent.ebpf.features`, these additional
+                          metrics are available: `namespace_rtt_seconds`, `node_rtt_seconds`,
+                          `workload_rtt_seconds`.'
                         items:
                           type: string
                         type: array

config/crd/bases/flows.netobserv.io_flowcollectors.yaml

@@ -7700,7 +7700,14 @@ spec:
                           `node_ingress_bytes_total`, `node_ingress_packets_total`,
                           `node_flows_total`, `workload_egress_bytes_total`, `workload_egress_packets_total`,
                           `workload_ingress_bytes_total`, `workload_ingress_packets_total`,
-                          `workload_flows_total`.'
+                          `workload_flows_total`. When the `PacketDrop` feature is
+                          enabled in `spec.agent.ebpf.features`, these additional
+                          metrics are available: `namespace_drop_bytes_total`, `namespace_drop_packets_total`,
+                          `node_drop_bytes_total`, `node_drop_packets_total`, `workload_drop_bytes_total`,
+                          `workload_drop_packets_total`. When the `FlowRTT` feature
+                          is enabled in `spec.agent.ebpf.features`, these additional
+                          metrics are available: `namespace_rtt_seconds`, `node_rtt_seconds`,
+                          `workload_rtt_seconds`.'
                         items:
                           type: string
                         type: array

controllers/flowlogspipeline/flp_test.go

@@ -915,10 +915,13 @@ func TestMergeMetricsConfiguration_Default(t *testing.T) {
 	jsonStages, _ := json.Marshal(stages)
 	assert.Equal(`[{"name":"ipfix"},{"name":"extract_conntrack","follows":"ipfix"},{"name":"enrich","follows":"extract_conntrack"},{"name":"loki","follows":"enrich"},{"name":"stdout","follows":"enrich"},{"name":"prometheus","follows":"enrich"}]`, string(jsonStages))
 	names := getSortedMetricsNames(parameters[5].Encode.Prom.Metrics)
-	assert.Len(names, 3)
-	assert.Equal("namespace_flows_total", names[0])
-	assert.Equal("node_ingress_bytes_total", names[1])
-	assert.Equal("workload_ingress_bytes_total", names[2])
+	assert.Equal([]string{
+		"namespace_drop_packets_total",
+		"namespace_flows_total",
+		"namespace_rtt_seconds",
+		"node_ingress_bytes_total",
+		"workload_ingress_bytes_total",
+	}, names)
 	assert.Equal("netobserv_", parameters[5].Encode.Prom.Prefix)
 }
 

docs/FlowCollector.md

@@ -13717,7 +13717,7 @@ target specifies the target value for the given metric
         <td><b>includeList</b></td>
         <td>[]string</td>
         <td>
-          `includeList` is a list of metric names to specify which metrics to generate. The names correspond to the name in Prometheus, without the prefix. For example, `namespace_egress_packets_total` will show up as `netobserv_namespace_egress_packets_total` in Prometheus. Available names are: `namespace_egress_bytes_total`, `namespace_egress_packets_total`, `namespace_ingress_bytes_total`, `namespace_ingress_packets_total`, `namespace_flows_total`, `node_egress_bytes_total`, `node_egress_packets_total`, `node_ingress_bytes_total`, `node_ingress_packets_total`, `node_flows_total`, `workload_egress_bytes_total`, `workload_egress_packets_total`, `workload_ingress_bytes_total`, `workload_ingress_packets_total`, `workload_flows_total`.<br/>
+          `includeList` is a list of metric names to specify which metrics to generate. The names correspond to the name in Prometheus, without the prefix. For example, `namespace_egress_packets_total` will show up as `netobserv_namespace_egress_packets_total` in Prometheus. Available names are: `namespace_egress_bytes_total`, `namespace_egress_packets_total`, `namespace_ingress_bytes_total`, `namespace_ingress_packets_total`, `namespace_flows_total`, `node_egress_bytes_total`, `node_egress_packets_total`, `node_ingress_bytes_total`, `node_ingress_packets_total`, `node_flows_total`, `workload_egress_bytes_total`, `workload_egress_packets_total`, `workload_ingress_bytes_total`, `workload_ingress_packets_total`, `workload_flows_total`. When the `PacketDrop` feature is enabled in `spec.agent.ebpf.features`, these additional metrics are available: `namespace_drop_bytes_total`, `namespace_drop_packets_total`, `node_drop_bytes_total`, `node_drop_packets_total`, `workload_drop_bytes_total`, `workload_drop_packets_total`. When the `FlowRTT` feature is enabled in `spec.agent.ebpf.features`, these additional metrics are available: `namespace_rtt_seconds`, `node_rtt_seconds`, `workload_rtt_seconds`.<br/>
         </td>
         <td>false</td>
       </tr><tr>

hack/cloned.flows.netobserv.io_flowcollectors.yaml

@@ -5320,7 +5320,7 @@ spec:
                             type: string
                           type: array
                         includeList:
-                          description: '`includeList` is a list of metric names to specify which metrics to generate. The names correspond to the name in Prometheus, without the prefix. For example, `namespace_egress_packets_total` will show up as `netobserv_namespace_egress_packets_total` in Prometheus. Available names are: `namespace_egress_bytes_total`, `namespace_egress_packets_total`, `namespace_ingress_bytes_total`, `namespace_ingress_packets_total`, `namespace_flows_total`, `node_egress_bytes_total`, `node_egress_packets_total`, `node_ingress_bytes_total`, `node_ingress_packets_total`, `node_flows_total`, `workload_egress_bytes_total`, `workload_egress_packets_total`, `workload_ingress_bytes_total`, `workload_ingress_packets_total`, `workload_flows_total`.'
+                          description: '`includeList` is a list of metric names to specify which metrics to generate. The names correspond to the name in Prometheus, without the prefix. For example, `namespace_egress_packets_total` will show up as `netobserv_namespace_egress_packets_total` in Prometheus. Available names are: `namespace_egress_bytes_total`, `namespace_egress_packets_total`, `namespace_ingress_bytes_total`, `namespace_ingress_packets_total`, `namespace_flows_total`, `node_egress_bytes_total`, `node_egress_packets_total`, `node_ingress_bytes_total`, `node_ingress_packets_total`, `node_flows_total`, `workload_egress_bytes_total`, `workload_egress_packets_total`, `workload_ingress_bytes_total`, `workload_ingress_packets_total`, `workload_flows_total`. When the `PacketDrop` feature is enabled in `spec.agent.ebpf.features`, these additional metrics are available: `namespace_drop_bytes_total`, `namespace_drop_packets_total`, `node_drop_bytes_total`, `node_drop_packets_total`, `workload_drop_bytes_total`, `workload_drop_packets_total`. When the `FlowRTT` feature is enabled in `spec.agent.ebpf.features`, these additional metrics are available: `namespace_rtt_seconds`, `node_rtt_seconds`, `workload_rtt_seconds`.'
                           items:
                             type: string
                           type: array

pkg/dashboards/dashboard.go

@@ -16,19 +16,21 @@ type rowInfo struct {
 
 // Queries
 const (
-	layerApps           = "Applications"
-	layerInfra          = "Infrastructure"
-	appsFilters1        = `SrcK8S_Namespace!~"|$NETOBSERV_NS|openshift.*"`
-	appsFilters2        = `SrcK8S_Namespace=~"$NETOBSERV_NS|openshift.*",DstK8S_Namespace!~"|$NETOBSERV_NS|openshift.*"`
-	infraFilters1       = `SrcK8S_Namespace=~"$NETOBSERV_NS|openshift.*"`
-	infraFilters2       = `SrcK8S_Namespace!~"$NETOBSERV_NS|openshift.*",DstK8S_Namespace=~"$NETOBSERV_NS|openshift.*"`
-	metricTagNamespaces = "namespaces"
-	metricTagNodes      = "nodes"
-	metricTagWorkloads  = "workloads"
-	metricTagIngress    = "ingress"
-	metricTagEgress     = "egress"
-	metricTagBytes      = "bytes"
-	metricTagPackets    = "packets"
+	layerApps            = "Applications"
+	layerInfra           = "Infrastructure"
+	appsFilters1         = `SrcK8S_Namespace!~"|$NETOBSERV_NS|openshift.*"`
+	appsFilters2         = `SrcK8S_Namespace=~"$NETOBSERV_NS|openshift.*",DstK8S_Namespace!~"|$NETOBSERV_NS|openshift.*"`
+	infraFilters1        = `SrcK8S_Namespace=~"$NETOBSERV_NS|openshift.*"`
+	infraFilters2        = `SrcK8S_Namespace!~"$NETOBSERV_NS|openshift.*",DstK8S_Namespace=~"$NETOBSERV_NS|openshift.*"`
+	metricTagNamespaces  = "namespaces"
+	metricTagNodes       = "nodes"
+	metricTagWorkloads   = "workloads"
+	metricTagIngress     = "ingress"
+	metricTagEgress      = "egress"
+	metricTagBytes       = "bytes"
+	metricTagPackets     = "packets"
+	metricTagDropBytes   = "drop_bytes"
+	metricTagDropPackets = "drop_packets"
 )
 
 var (
@@ -85,6 +87,7 @@ var (
 func init() {
 	for _, group := range []string{metricTagNodes, metricTagNamespaces, metricTagWorkloads} {
 		groupTrimmed := strings.TrimSuffix(group, "s")
+		// byte/pkt rates
 		for _, vt := range []string{metricTagBytes, metricTagPackets} {
 			for _, dir := range []string{metricTagEgress, metricTagIngress} {
 				rowsInfo = append(rowsInfo, rowInfo{
@@ -95,6 +98,15 @@ func init() {
 				})
 			}
 		}
+		// drops
+		for _, vt := range []string{metricTagDropBytes, metricTagDropPackets} {
+			rowsInfo = append(rowsInfo, rowInfo{
+				metric:    fmt.Sprintf("netobserv_%s_%s_total", groupTrimmed, vt),
+				group:     group,
+				valueType: vt,
+			})
+		}
+		// TODO: RTT dashboard (after dashboard refactoring for exposed metrics; need to handle histogram queries)
 	}
 }
 
@@ -217,6 +229,10 @@ func flowMetricsRow(netobsNs string, rowInfo rowInfo) string {
 		vt = "byte"
 	case metricTagPackets:
 		vt = "packet"
+	case metricTagDropBytes:
+		vt = "drop bytes"
+	case metricTagDropPackets:
+		vt = "drop packets"
 	}
 	title := fmt.Sprintf("Top %s rates %s per source and destination %s", vt, verb, rowInfo.group)
 	var panels string

pkg/dashboards/dashboard_test.go

@@ -18,7 +18,7 @@ func TestCreateFlowMetricsDashboard_All(t *testing.T) {
 	assert.NoError(err)
 
 	assert.Equal("NetObserv", d.Title)
-	assert.Len(d.Rows, 12)
+	assert.Len(d.Rows, 18)
 
 	// First row
 	row := 0
@@ -28,8 +28,8 @@ func TestCreateFlowMetricsDashboard_All(t *testing.T) {
 	assert.Len(d.Rows[row].Panels[0].Targets, 1)
 	assert.Contains(d.Rows[row].Panels[0].Targets[0].Expr, "label_replace(label_replace(topk(10,sum(rate(netobserv_node_egress_bytes_total[1m])) by (SrcK8S_HostName, DstK8S_HostName))")
 
-	// 6th row
-	row = 5
+	// 8th row
+	row = 7
 	assert.Equal("Top byte rates received per source and destination namespaces", d.Rows[row].Title)
 	assert.Len(d.Rows[row].Panels, 2)
 	assert.Equal("Applications", d.Rows[row].Panels[0].Title)
@@ -42,8 +42,8 @@ func TestCreateFlowMetricsDashboard_All(t *testing.T) {
 		`label_replace(label_replace(topk(10,sum(rate(netobserv_namespace_ingress_bytes_total{SrcK8S_Namespace=~"netobserv|openshift.*"}[1m]) or rate(netobserv_namespace_ingress_bytes_total{SrcK8S_Namespace!~"netobserv|openshift.*",DstK8S_Namespace=~"netobserv|openshift.*"}[1m])) by (SrcK8S_Namespace, DstK8S_Namespace))`,
 	)
 
-	// 12th row
-	row = 11
+	// 16th row
+	row = 15
 	assert.Equal("Top packet rates received per source and destination workloads", d.Rows[row].Title)
 	assert.Len(d.Rows[row].Panels, 2)
 	assert.Equal("Applications", d.Rows[row].Panels[0].Title)
@@ -88,7 +88,7 @@ func TestCreateFlowMetricsDashboard_DefaultList(t *testing.T) {
 	assert.NoError(err)
 
 	assert.Equal("NetObserv", d.Title)
-	assert.Len(d.Rows, 3)
+	assert.Len(d.Rows, 4)
 
 	// First row
 	row := 0
@@ -113,8 +113,8 @@ func TestCreateFlowMetricsDashboard_DefaultList(t *testing.T) {
 		`label_replace(label_replace(topk(10,sum(rate(netobserv_workload_ingress_bytes_total{SrcK8S_Namespace=~"netobserv|openshift.*"}[1m]) or rate(netobserv_workload_ingress_bytes_total{SrcK8S_Namespace!~"netobserv|openshift.*",DstK8S_Namespace=~"netobserv|openshift.*"}[1m])) by (SrcK8S_Namespace, DstK8S_Namespace))`,
 	)
 
-	// 3rd row
-	row = 2
+	// 4th row
+	row = 3
 	assert.Equal("Top byte rates received per source and destination workloads", d.Rows[row].Title)
 	assert.Len(d.Rows[row].Panels, 2)
 	assert.Equal("Applications", d.Rows[row].Panels[0].Title)

pkg/metrics/predefined_metrics.go

@@ -34,7 +34,13 @@ var (
 	}
 	predefinedMetrics []taggedMetricDefinition
 	// Note that we set default in-code rather than in CRD, in order to keep track of value being unset or set intentionnally in FlowCollector
-	DefaultIncludeList = []string{"node_ingress_bytes_total", "workload_ingress_bytes_total", "namespace_flows_total"}
+	DefaultIncludeList = []string{
+		"node_ingress_bytes_total",
+		"workload_ingress_bytes_total",
+		"namespace_flows_total",
+		"namespace_drop_packets_total",
+		"namespace_rtt_seconds",
+	}
 	// Pre-deprecation default IgnoreTags list (1.4) - used before switching to whitelist approach,
 	// to make sure there is no unintended new metrics being collected
 	// Don't add anything here: this is not meant to evolve
@@ -61,7 +67,7 @@ func init() {
 						ValueKey: valueField,
 						Filters: []flpapi.PromMetricsFilter{
 							{Key: "Duplicate", Value: "false"},
-							{Key: "FlowDirection", Value: mapDirection[dir], Type: "regex"},
+							{Key: "FlowDirection", Value: mapDirection[dir], Type: flpapi.PromFilterRegex},
 						},
 						Labels: labels,
 					},
@@ -78,6 +84,47 @@ func init() {
 			},
 			tags: []string{group, group + "-flows", "flows"},
 		})
+		// RTT metrics
+		predefinedMetrics = append(predefinedMetrics, taggedMetricDefinition{
+			PromMetricsItem: flpapi.PromMetricsItem{
+				Name:     fmt.Sprintf("%s_rtt_seconds", groupTrimmed),
+				Type:     "histogram",
+				ValueKey: "TimeFlowRttNs",
+				Filters: []flpapi.PromMetricsFilter{
+					{Key: "TimeFlowRttNs", Type: flpapi.PromFilterPresence},
+				},
+				Labels:     labels,
+				ValueScale: 1_000_000_000, // ns => s
+			},
+			tags: []string{group, "rtt"},
+		})
+		// Drops metrics
+		predefinedMetrics = append(predefinedMetrics, taggedMetricDefinition{
+			PromMetricsItem: flpapi.PromMetricsItem{
+				Name:     fmt.Sprintf("%s_drop_packets_total", groupTrimmed),
+				Type:     "counter",
+				ValueKey: "PktDropPackets",
+				Filters: []flpapi.PromMetricsFilter{
+					{Key: "Duplicate", Value: "false"},
+					{Key: "PktDropPackets", Type: flpapi.PromFilterPresence},
+				},
+				Labels: labels,
+			},
+			tags: []string{group, tagPackets, "drops"},
+		})
+		predefinedMetrics = append(predefinedMetrics, taggedMetricDefinition{
+			PromMetricsItem: flpapi.PromMetricsItem{
+				Name:     fmt.Sprintf("%s_drop_bytes_total", groupTrimmed),
+				Type:     "counter",
+				ValueKey: "PktDropBytes",
+				Filters: []flpapi.PromMetricsFilter{
+					{Key: "Duplicate", Value: "false"},
+					{Key: "PktDropBytes", Type: flpapi.PromFilterPresence},
+				},
+				Labels: labels,
+			},
+			tags: []string{group, tagBytes, "drop"},
+		})
 	}
 }
 

pkg/metrics/predefined_metrics_test.go

@@ -11,7 +11,17 @@ func TestIncludeExclude(t *testing.T) {
 
 	// IgnoreTags set, Include list unset => resolving ignore tags
 	res := GetEnabledNames([]string{"egress", "packets", "flows"}, nil)
-	assert.Equal([]string{"node_ingress_bytes_total", "namespace_ingress_bytes_total", "workload_ingress_bytes_total"}, res)
+	assert.Equal([]string{
+		"node_ingress_bytes_total",
+		"node_rtt_seconds",
+		"node_drop_bytes_total",
+		"namespace_ingress_bytes_total",
+		"namespace_rtt_seconds",
+		"namespace_drop_bytes_total",
+		"workload_ingress_bytes_total",
+		"workload_rtt_seconds",
+		"workload_drop_bytes_total",
+	}, res)
 
 	// IgnoreTags set, Include list set => keep include list
 	res = GetEnabledNames([]string{"egress", "packets"}, &[]string{"namespace_flows_total"})