From 53ea2d889174a4aca22d6fc4ce17d0aa8dc14e36 Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Mon, 11 Sep 2023 12:01:50 +0200 Subject: [PATCH] rtt metrics --- api/v1beta1/flowcollector_types.go | 4 +- .../flows.netobserv.io_flowcollectors.yaml | 3 +- ...observ-operator.clusterserviceversion.yaml | 1 + .../flows.netobserv.io_flowcollectors.yaml | 3 +- .../samples/flows_v1beta1_flowcollector.yaml | 1 + controllers/flowlogspipeline/flp_test.go | 45 ++++++++++++------- .../namespace_egress_rtt_total.yaml | 28 ++++++++++++ .../namespace_ingress_rtt_total.yaml | 28 ++++++++++++ .../node_egress_rtt_total.yaml | 28 ++++++++++++ .../node_ingress_rtt_total.yaml | 28 ++++++++++++ .../workload_egress_rtt_total.yaml | 32 +++++++++++++ .../workload_ingress_rtt_total.yaml | 32 +++++++++++++ docs/FlowCollector.md | 4 +- 13 files changed, 214 insertions(+), 23 deletions(-) create mode 100644 controllers/flowlogspipeline/metrics_definitions/namespace_egress_rtt_total.yaml create mode 100644 controllers/flowlogspipeline/metrics_definitions/namespace_ingress_rtt_total.yaml create mode 100644 controllers/flowlogspipeline/metrics_definitions/node_egress_rtt_total.yaml create mode 100644 controllers/flowlogspipeline/metrics_definitions/node_ingress_rtt_total.yaml create mode 100644 controllers/flowlogspipeline/metrics_definitions/workload_egress_rtt_total.yaml create mode 100644 controllers/flowlogspipeline/metrics_definitions/workload_ingress_rtt_total.yaml diff --git a/api/v1beta1/flowcollector_types.go b/api/v1beta1/flowcollector_types.go index 3aa6877ba..cc360891d 100644 --- a/api/v1beta1/flowcollector_types.go +++ b/api/v1beta1/flowcollector_types.go @@ -348,9 +348,9 @@ type FLPMetrics struct { Server MetricsServerConfig `json:"server,omitempty"` // `ignoreTags` is a list of tags to specify which metrics to ignore. Each metric is associated with a list of tags. More details in https://github.com/netobserv/network-observability-operator/tree/main/controllers/flowlogspipeline/metrics_definitions . - // Available tags are: `egress`, `ingress`, `flows`, `bytes`, `packets`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, `namespaces-flows`, `workloads-flows`. + // Available tags are: `egress`, `ingress`, `flows`, `bytes`, `packets`, `rtt`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, `namespaces-flows`, `workloads-flows`. // Namespace-based metrics are covered by both `workloads` and `namespaces` tags, hence it is recommended to always ignore one of them (`workloads` offering a finer granularity). - //+kubebuilder:default:={"egress","packets","nodes-flows","namespaces-flows","workloads-flows","namespaces"} + //+kubebuilder:default:={"egress","packets","rtt","nodes-flows","namespaces-flows","workloads-flows","namespaces"} // +optional IgnoreTags []string `json:"ignoreTags"` diff --git a/bundle/manifests/flows.netobserv.io_flowcollectors.yaml b/bundle/manifests/flows.netobserv.io_flowcollectors.yaml index 6a6f1e19a..1bd238e49 100644 --- a/bundle/manifests/flows.netobserv.io_flowcollectors.yaml +++ b/bundle/manifests/flows.netobserv.io_flowcollectors.yaml @@ -4527,6 +4527,7 @@ spec: default: - egress - packets + - rtt - nodes-flows - namespaces-flows - workloads-flows @@ -4535,7 +4536,7 @@ spec: metrics to ignore. Each metric is associated with a list of tags. More details in https://github.com/netobserv/network-observability-operator/tree/main/controllers/flowlogspipeline/metrics_definitions . Available tags are: `egress`, `ingress`, `flows`, `bytes`, - `packets`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, + `packets`, `rtt`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, `namespaces-flows`, `workloads-flows`. Namespace-based metrics are covered by both `workloads` and `namespaces` tags, hence it is recommended to always ignore one of them (`workloads` diff --git a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml index 190c28e9b..6b22d04b5 100644 --- a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml +++ b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml @@ -328,6 +328,7 @@ metadata: "ignoreTags": [ "egress", "packets", + "rtt", "nodes-flows", "namespaces-flows", "workloads-flows", diff --git a/config/crd/bases/flows.netobserv.io_flowcollectors.yaml b/config/crd/bases/flows.netobserv.io_flowcollectors.yaml index ee232894d..4aa2271b0 100644 --- a/config/crd/bases/flows.netobserv.io_flowcollectors.yaml +++ b/config/crd/bases/flows.netobserv.io_flowcollectors.yaml @@ -4514,6 +4514,7 @@ spec: default: - egress - packets + - rtt - nodes-flows - namespaces-flows - workloads-flows @@ -4522,7 +4523,7 @@ spec: metrics to ignore. Each metric is associated with a list of tags. More details in https://github.com/netobserv/network-observability-operator/tree/main/controllers/flowlogspipeline/metrics_definitions . Available tags are: `egress`, `ingress`, `flows`, `bytes`, - `packets`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, + `packets`, `rtt`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, `namespaces-flows`, `workloads-flows`. Namespace-based metrics are covered by both `workloads` and `namespaces` tags, hence it is recommended to always ignore one of them (`workloads` diff --git a/config/samples/flows_v1beta1_flowcollector.yaml b/config/samples/flows_v1beta1_flowcollector.yaml index 8d4007e93..b22cea4a8 100644 --- a/config/samples/flows_v1beta1_flowcollector.yaml +++ b/config/samples/flows_v1beta1_flowcollector.yaml @@ -38,6 +38,7 @@ spec: ignoreTags: - egress - packets + - rtt - nodes-flows - namespaces-flows - workloads-flows diff --git a/controllers/flowlogspipeline/flp_test.go b/controllers/flowlogspipeline/flp_test.go index bd070ed22..08f984837 100644 --- a/controllers/flowlogspipeline/flp_test.go +++ b/controllers/flowlogspipeline/flp_test.go @@ -814,22 +814,33 @@ func TestMergeMetricsConfigurationNoIgnore(t *testing.T) { assert.True(validatePipelineConfig(stages, parameters)) jsonStages, _ := json.Marshal(stages) assert.Equal(`[{"name":"ipfix"},{"name":"extract_conntrack","follows":"ipfix"},{"name":"enrich","follows":"extract_conntrack"},{"name":"loki","follows":"enrich"},{"name":"stdout","follows":"enrich"},{"name":"prometheus","follows":"enrich"}]`, string(jsonStages)) - assert.Len(parameters[5].Encode.Prom.Metrics, 15) - assert.Equal("namespace_egress_bytes_total", parameters[5].Encode.Prom.Metrics[0].Name) - assert.Equal("namespace_egress_packets_total", parameters[5].Encode.Prom.Metrics[1].Name) - assert.Equal("namespace_flows_total", parameters[5].Encode.Prom.Metrics[2].Name) - assert.Equal("namespace_ingress_bytes_total", parameters[5].Encode.Prom.Metrics[3].Name) - assert.Equal("namespace_ingress_packets_total", parameters[5].Encode.Prom.Metrics[4].Name) - assert.Equal("node_egress_bytes_total", parameters[5].Encode.Prom.Metrics[5].Name) - assert.Equal("node_egress_packets_total", parameters[5].Encode.Prom.Metrics[6].Name) - assert.Equal("node_flows_total", parameters[5].Encode.Prom.Metrics[7].Name) - assert.Equal("node_ingress_bytes_total", parameters[5].Encode.Prom.Metrics[8].Name) - assert.Equal("node_ingress_packets_total", parameters[5].Encode.Prom.Metrics[9].Name) - assert.Equal("workload_egress_bytes_total", parameters[5].Encode.Prom.Metrics[10].Name) - assert.Equal("workload_egress_packets_total", parameters[5].Encode.Prom.Metrics[11].Name) - assert.Equal("workload_flows_total", parameters[5].Encode.Prom.Metrics[12].Name) - assert.Equal("workload_ingress_bytes_total", parameters[5].Encode.Prom.Metrics[13].Name) - assert.Equal("workload_ingress_packets_total", parameters[5].Encode.Prom.Metrics[14].Name) + assert.Len(parameters[5].Encode.Prom.Metrics, 21) + definitions := []string{ + "namespace_egress_bytes_total", + "namespace_egress_packets_total", + "namespace_egress_rtt_total", + "namespace_flows_total", + "namespace_ingress_bytes_total", + "namespace_ingress_packets_total", + "namespace_ingress_rtt_total", + "node_egress_bytes_total", + "node_egress_packets_total", + "node_egress_rtt_total", + "node_flows_total", + "node_ingress_bytes_total", + "node_ingress_packets_total", + "node_ingress_rtt_total", + "workload_egress_bytes_total", + "workload_egress_packets_total", + "workload_egress_rtt_total", + "workload_flows_total", + "workload_ingress_bytes_total", + "workload_ingress_packets_total", + "workload_ingress_rtt_total", + } + for i, def := range definitions { + assert.Equal(def, parameters[5].Encode.Prom.Metrics[i].Name) + } assert.Equal("netobserv_", parameters[5].Encode.Prom.Prefix) } @@ -845,7 +856,7 @@ func TestMergeMetricsConfigurationWithIgnore(t *testing.T) { assert.True(validatePipelineConfig(stages, parameters)) jsonStages, _ := json.Marshal(stages) assert.Equal(`[{"name":"ipfix"},{"name":"extract_conntrack","follows":"ipfix"},{"name":"enrich","follows":"extract_conntrack"},{"name":"loki","follows":"enrich"},{"name":"stdout","follows":"enrich"},{"name":"prometheus","follows":"enrich"}]`, string(jsonStages)) - assert.Len(parameters[5].Encode.Prom.Metrics, 10) + assert.Len(parameters[5].Encode.Prom.Metrics, 14) assert.Equal("namespace_egress_bytes_total", parameters[5].Encode.Prom.Metrics[0].Name) assert.Equal("netobserv_", parameters[5].Encode.Prom.Prefix) } diff --git a/controllers/flowlogspipeline/metrics_definitions/namespace_egress_rtt_total.yaml b/controllers/flowlogspipeline/metrics_definitions/namespace_egress_rtt_total.yaml new file mode 100644 index 000000000..2c20714ea --- /dev/null +++ b/controllers/flowlogspipeline/metrics_definitions/namespace_egress_rtt_total.yaml @@ -0,0 +1,28 @@ +#flp_confgen +description: + This metric observes the namespaces outgoing traffic +details: + Sum RTT for outgoing traffic per source and destination namespaces +usage: + Evaluate network outgoing TCP hanshakes Round Trip Time per source and destination namespaces +tags: + - egress + - rtt + - namespaces +encode: + type: prom + prom: + metrics: + - name: namespace_egress_rtt_total + type: histogram + valuekey: TimeFlowRttNs + filters: + - key: FlowDirection + value: "1" + - key: Duplicate + value: "false" + - key: TimeFlowRttNs + value: "!nil" + labels: + - SrcK8S_Namespace + - DstK8S_Namespace diff --git a/controllers/flowlogspipeline/metrics_definitions/namespace_ingress_rtt_total.yaml b/controllers/flowlogspipeline/metrics_definitions/namespace_ingress_rtt_total.yaml new file mode 100644 index 000000000..82008a7c8 --- /dev/null +++ b/controllers/flowlogspipeline/metrics_definitions/namespace_ingress_rtt_total.yaml @@ -0,0 +1,28 @@ +#flp_confgen +description: + This metric observes the namespaces incoming traffic +details: + Sum RTT for incoming traffic per source and destination namespaces +usage: + Evaluate network incoming TCP hanshakes Round Trip Time per source and destination namespaces +tags: + - ingress + - rtt + - namespaces +encode: + type: prom + prom: + metrics: + - name: namespace_ingress_rtt_total + type: histogram + valuekey: TimeFlowRttNs + filters: + - key: FlowDirection + value: "0" + - key: Duplicate + value: "false" + - key: TimeFlowRttNs + value: "!nil" + labels: + - SrcK8S_Namespace + - DstK8S_Namespace diff --git a/controllers/flowlogspipeline/metrics_definitions/node_egress_rtt_total.yaml b/controllers/flowlogspipeline/metrics_definitions/node_egress_rtt_total.yaml new file mode 100644 index 000000000..8353b4c45 --- /dev/null +++ b/controllers/flowlogspipeline/metrics_definitions/node_egress_rtt_total.yaml @@ -0,0 +1,28 @@ +#flp_confgen +description: + This metric observes the nodes outgoing traffic +details: + Sum RTT for outgoing traffic per source and destination nodes +usage: + Evaluate network outgoing TCP hanshakes Round Trip Time per source and destination nodes +tags: + - egress + - rtt + - nodes +encode: + type: prom + prom: + metrics: + - name: node_egress_rtt_total + type: histogram + valuekey: TimeFlowRttNs + filters: + - key: FlowDirection + value: "1" + - key: Duplicate + value: "false" + - key: TimeFlowRttNs + value: "!nil" + labels: + - SrcK8S_HostName + - DstK8S_HostName diff --git a/controllers/flowlogspipeline/metrics_definitions/node_ingress_rtt_total.yaml b/controllers/flowlogspipeline/metrics_definitions/node_ingress_rtt_total.yaml new file mode 100644 index 000000000..62568e8d9 --- /dev/null +++ b/controllers/flowlogspipeline/metrics_definitions/node_ingress_rtt_total.yaml @@ -0,0 +1,28 @@ +#flp_confgen +description: + This metric observes the nodes incoming traffic +details: + Sum RTT for incoming traffic per source and destination nodes +usage: + Evaluate network incoming TCP hanshakes Round Trip Time per source and destination nodes +tags: + - ingress + - rtt + - nodes +encode: + type: prom + prom: + metrics: + - name: node_ingress_rtt_total + type: histogram + valuekey: TimeFlowRttNs + filters: + - key: FlowDirection + value: "0" + - key: Duplicate + value: "false" + - key: TimeFlowRttNs + value: "!nil" + labels: + - SrcK8S_HostName + - DstK8S_HostName diff --git a/controllers/flowlogspipeline/metrics_definitions/workload_egress_rtt_total.yaml b/controllers/flowlogspipeline/metrics_definitions/workload_egress_rtt_total.yaml new file mode 100644 index 000000000..4f7ce6888 --- /dev/null +++ b/controllers/flowlogspipeline/metrics_definitions/workload_egress_rtt_total.yaml @@ -0,0 +1,32 @@ +#flp_confgen +description: + This metric observes the outgoing traffic +details: + Sum RTT for outgoing traffic per source and destination namespaces and owners +usage: + Evaluate network outgoing TCP hanshakes Round Trip Time per source and destination namespaces and owners +tags: + - egress + - rtt + - workloads +encode: + type: prom + prom: + metrics: + - name: workload_egress_rtt_total + type: histogram + valuekey: TimeFlowRttNs + filters: + - key: FlowDirection + value: "1" + - key: Duplicate + value: "false" + - key: TimeFlowRttNs + value: "!nil" + labels: + - SrcK8S_Namespace + - DstK8S_Namespace + - SrcK8S_OwnerName + - DstK8S_OwnerName + - SrcK8S_OwnerType + - DstK8S_OwnerType diff --git a/controllers/flowlogspipeline/metrics_definitions/workload_ingress_rtt_total.yaml b/controllers/flowlogspipeline/metrics_definitions/workload_ingress_rtt_total.yaml new file mode 100644 index 000000000..0e8264785 --- /dev/null +++ b/controllers/flowlogspipeline/metrics_definitions/workload_ingress_rtt_total.yaml @@ -0,0 +1,32 @@ +#flp_confgen +description: + This metric observes the incoming traffic +details: + Sum RTT for incoming traffic per source and destination namespaces and owners +usage: + Evaluate network incoming TCP hanshakes Round Trip Time per source and destination namespaces and owners +tags: + - ingress + - rtt + - workloads +encode: + type: prom + prom: + metrics: + - name: workload_ingress_rtt_total + type: histogram + valuekey: TimeFlowRttNs + filters: + - key: FlowDirection + value: "0" + - key: Duplicate + value: "false" + - key: TimeFlowRttNs + value: "!nil" + labels: + - SrcK8S_Namespace + - DstK8S_Namespace + - SrcK8S_OwnerName + - DstK8S_OwnerName + - SrcK8S_OwnerType + - DstK8S_OwnerType diff --git a/docs/FlowCollector.md b/docs/FlowCollector.md index cfed772fd..7f533ab9c 100644 --- a/docs/FlowCollector.md +++ b/docs/FlowCollector.md @@ -8002,9 +8002,9 @@ target specifies the target value for the given metric ignoreTags []string - `ignoreTags` is a list of tags to specify which metrics to ignore. Each metric is associated with a list of tags. More details in https://github.com/netobserv/network-observability-operator/tree/main/controllers/flowlogspipeline/metrics_definitions . Available tags are: `egress`, `ingress`, `flows`, `bytes`, `packets`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, `namespaces-flows`, `workloads-flows`. Namespace-based metrics are covered by both `workloads` and `namespaces` tags, hence it is recommended to always ignore one of them (`workloads` offering a finer granularity).
+ `ignoreTags` is a list of tags to specify which metrics to ignore. Each metric is associated with a list of tags. More details in https://github.com/netobserv/network-observability-operator/tree/main/controllers/flowlogspipeline/metrics_definitions . Available tags are: `egress`, `ingress`, `flows`, `bytes`, `packets`, `rtt`, `namespaces`, `nodes`, `workloads`, `nodes-flows`, `namespaces-flows`, `workloads-flows`. Namespace-based metrics are covered by both `workloads` and `namespaces` tags, hence it is recommended to always ignore one of them (`workloads` offering a finer granularity).

- Default: [egress packets nodes-flows namespaces-flows workloads-flows namespaces]
+ Default: [egress packets rtt nodes-flows namespaces-flows workloads-flows namespaces]
false