From 4601245bbbfd3373176d79387a4375246bd76160 Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Wed, 21 Feb 2024 12:54:15 +0100 Subject: [PATCH] NodeDirection / IfDirection --- apis/flowmetrics/v1alpha1/flowmetric_types.go | 16 ++--- .../flows.netobserv.io_flowmetrics.yaml | 6 +- ...observ-operator.clusterserviceversion.yaml | 2 +- .../bases/flows.netobserv.io_flowmetrics.yaml | 6 +- config/samples/flows_v1alpha1_flowmetric.yaml | 2 +- .../config/static-frontend-config.yaml | 58 +++++++++++-------- controllers/constants/constants.go | 3 +- controllers/flp/flp_pipeline_builder.go | 9 ++- controllers/flp/flp_test.go | 3 +- controllers/flp/metrics_api_test.go | 2 +- docs/flows-format.adoc | 2 +- ...cloned.flows.netobserv.io_flowmetrics.yaml | 6 +- pkg/loki/labels.go | 15 ----- pkg/metrics/predefined_metrics.go | 2 +- 14 files changed, 64 insertions(+), 68 deletions(-) diff --git a/apis/flowmetrics/v1alpha1/flowmetric_types.go b/apis/flowmetrics/v1alpha1/flowmetric_types.go index 850df8d54..43d8e6c9c 100644 --- a/apis/flowmetrics/v1alpha1/flowmetric_types.go +++ b/apis/flowmetrics/v1alpha1/flowmetric_types.go @@ -22,7 +22,7 @@ import ( type MetricType string type FilterMatchType string -type FlowDirection string +type NodeDirection string const ( CounterMetric MetricType = "Counter" @@ -33,9 +33,9 @@ const ( MatchRegex FilterMatchType = "Regex" MatchPresence FilterMatchType = "Presence" MatchAbsence FilterMatchType = "Absence" - Egress FlowDirection = "Egress" - Ingress FlowDirection = "Ingress" - AnyDirection FlowDirection = "Any" + Egress NodeDirection = "Egress" + Ingress NodeDirection = "Ingress" + AnyDirection NodeDirection = "Any" ) type MetricFilter struct { @@ -77,7 +77,7 @@ type FlowMetricSpec struct { ValueField string `json:"valueField,omitempty"` // `filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must - // be used to eliminate duplicates: `Duplicate:"false"` and `FlowDirection: "0"`. + // be used to eliminate duplicates: `Duplicate:"false"` and `NodeDirection: "0"`. // Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/networking/network_observability/json-flows-format-reference.html. // +optional Filters []MetricFilter `json:"filters"` @@ -97,12 +97,12 @@ type FlowMetricSpec struct { IncludeDuplicates bool `json:"includeDuplicates,omitempty"` // Filter for ingress, egress or any direction flows. - // When set to `Ingress`, it is equivalent to adding the regex filter on `FlowDirection`: `0|2`. - // When set to `Egress`, it is equivalent to adding the regex filter on `FlowDirection`: `1|2`. + // When set to `Ingress`, it is equivalent to adding the regex filter on `NodeDirection`: `0|2`. + // When set to `Egress`, it is equivalent to adding the regex filter on `NodeDirection`: `1|2`. // +kubebuilder:validation:Enum:="Any";"Egress";"Ingress" // +kubebuilder:default:="Any" // +optional - Direction FlowDirection `json:"direction,omitempty"` + Direction NodeDirection `json:"direction,omitempty"` // A list of buckets to use when `type` is "Histogram". The list must be parseable as floats. Prometheus default buckets will be used if unset. // +optional diff --git a/bundle/manifests/flows.netobserv.io_flowmetrics.yaml b/bundle/manifests/flows.netobserv.io_flowmetrics.yaml index 3ff7c05f1..25b8d1c71 100644 --- a/bundle/manifests/flows.netobserv.io_flowmetrics.yaml +++ b/bundle/manifests/flows.netobserv.io_flowmetrics.yaml @@ -51,8 +51,8 @@ spec: default: Any description: 'Filter for ingress, egress or any direction flows. When set to `Ingress`, it is equivalent to adding the regex filter on - `FlowDirection`: `0|2`. When set to `Egress`, it is equivalent to - adding the regex filter on `FlowDirection`: `1|2`.' + `NodeDirection`: `0|2`. When set to `Egress`, it is equivalent to + adding the regex filter on `NodeDirection`: `1|2`.' enum: - Any - Egress @@ -61,7 +61,7 @@ spec: filters: description: '`filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must - be used to eliminate duplicates: `Duplicate:"false"` and `FlowDirection: + be used to eliminate duplicates: `Duplicate:"false"` and `NodeDirection: "0"`. Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/networking/network_observability/json-flows-format-reference.html.' items: diff --git a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml index 8814b9729..360f0170f 100644 --- a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml +++ b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml @@ -39,7 +39,7 @@ metadata: "value": "false" }, { - "field": "FlowDirection", + "field": "NodeDirection", "matchType": "Regex", "value": "1|2" } diff --git a/config/crd/bases/flows.netobserv.io_flowmetrics.yaml b/config/crd/bases/flows.netobserv.io_flowmetrics.yaml index 50469918a..733ad630c 100644 --- a/config/crd/bases/flows.netobserv.io_flowmetrics.yaml +++ b/config/crd/bases/flows.netobserv.io_flowmetrics.yaml @@ -53,8 +53,8 @@ spec: default: Any description: 'Filter for ingress, egress or any direction flows. When set to `Ingress`, it is equivalent to adding the regex filter on - `FlowDirection`: `0|2`. When set to `Egress`, it is equivalent to - adding the regex filter on `FlowDirection`: `1|2`.' + `NodeDirection`: `0|2`. When set to `Egress`, it is equivalent to + adding the regex filter on `NodeDirection`: `1|2`.' enum: - Any - Egress @@ -63,7 +63,7 @@ spec: filters: description: '`filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must - be used to eliminate duplicates: `Duplicate:"false"` and `FlowDirection: + be used to eliminate duplicates: `Duplicate:"false"` and `NodeDirection: "0"`. Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/networking/network_observability/json-flows-format-reference.html.' items: diff --git a/config/samples/flows_v1alpha1_flowmetric.yaml b/config/samples/flows_v1alpha1_flowmetric.yaml index a0da6cd68..1a5663b55 100644 --- a/config/samples/flows_v1alpha1_flowmetric.yaml +++ b/config/samples/flows_v1alpha1_flowmetric.yaml @@ -20,6 +20,6 @@ spec: matchType: Regex - field: Duplicate value: "false" - - field: FlowDirection + - field: NodeDirection value: "1|2" matchType: Regex diff --git a/controllers/consoleplugin/config/static-frontend-config.yaml b/controllers/consoleplugin/config/static-frontend-config.yaml index 10225e666..b0033a929 100644 --- a/controllers/consoleplugin/config/static-frontend-config.yaml +++ b/controllers/consoleplugin/config/static-frontend-config.yaml @@ -380,24 +380,31 @@ columns: filter: icmp_code default: false width: 10 - - id: FlowDirection - name: Direction - tooltip: The direction of the flow observed at the Node observation point. - field: FlowDirection - filter: direction + - id: NodeDirection + name: Node Direction + tooltip: The interpreted direction of the flow observed at the Node observation point. + field: NodeDirection + filter: nodedirection default: false width: 10 - - id: Interface - name: Interface - tooltip: The network interface of the Flow. - field: Interface - filter: interface + - id: Interfaces + name: Interfaces + tooltip: The network interfaces of the Flow. + field: Interfaces + filter: interfaces + default: false + width: 10 + - id: IfDirections + name: Interface Directions + tooltip: The directions of the Flow observed at the network interface observation point. + field: IfDirections + filter: ifdirections default: false width: 10 - id: FlowDirInts name: Interfaces and Directions - tooltip: Pairs of network interface and direction of the Flow observed at the Node observation point. - field: FlowDirection + tooltip: Pairs of network interface and direction of the Flow observed at the network interface observation point. + field: Interfaces default: false width: 15 - id: Bytes @@ -757,21 +764,26 @@ filters: name: ICMP code component: number hint: Specify an ICMP code value as integer number. - - id: direction - name: Direction + - id: nodedirection + name: Node Direction component: autocomplete placeholder: 'E.g: Ingress, Egress, Inner' - hint: Specify the direction of the Flow observed at the Node observation point. + hint: Specify the interpreted direction of the Flow observed at the Node observation point. - id: flow_layer name: Flow layer component: text placeholder: 'Either infra or app' hint: Specify the layer of Flow. - - id: interface - name: Network interface + - id: interfaces + name: Network interfaces component: text placeholder: 'E.g: br-ex, ovn-k8s-mp0' hint: Specify a network interface. + - id: ifdirections + name: Interface Directions + component: autocomplete + placeholder: 'E.g: Ingress, Egress' + hint: Specify the direction of the Flow observed at the network interface observation point. - id: id name: Conversation Id component: text @@ -927,23 +939,23 @@ fields: type: boolean description: Indicates if this flow was also captured from another interface on the same host lokiLabel: true - - name: FlowDirection + - name: NodeDirection type: number description: | - Flow direction from the node observation point. Can be one of: + + Flow interpreted direction from the node observation point. Can be one of: + - 0: Ingress (incoming traffic, from the node observation point) + - 1: Egress (outgoing traffic, from the node observation point) + - 2: Inner (with the same source and destination node) lokiLabel: true - - name: IfDirection + - name: IfDirections type: number description: | - Flow direction from the network interface observation point. Can be one of: + + Flow directions from the network interface observation point. Can be one of: + - 0: Ingress (interface incoming traffic) + - 1: Egress (interface outgoing traffic) - - name: Interface + - name: Interfaces type: string - description: Network interface + description: Network interfaces - name: Flags type: number description: | diff --git a/controllers/constants/constants.go b/controllers/constants/constants.go index 90a87b404..bc2277426 100644 --- a/controllers/constants/constants.go +++ b/controllers/constants/constants.go @@ -47,9 +47,8 @@ const ( LokiCRReader = "netobserv-reader" ) -var LokiIndexFields = []string{"SrcK8S_Namespace", "SrcK8S_OwnerName", "SrcK8S_Type", "DstK8S_Namespace", "DstK8S_OwnerName", "DstK8S_Type", "K8S_FlowLayer"} +var LokiIndexFields = []string{"SrcK8S_Namespace", "SrcK8S_OwnerName", "SrcK8S_Type", "DstK8S_Namespace", "DstK8S_OwnerName", "DstK8S_Type", "K8S_FlowLayer", "NodeDirection"} var LokiConnectionIndexFields = []string{"_RecordType"} -var LokiDeduperMarkIndexFields = []string{"FlowDirection"} var LokiZoneIndexFields = []string{"SrcK8S_Zone", "DstK8S_Zone"} var FlowCollectorName = types.NamespacedName{Name: "cluster"} var EnvNoHTTP2 = corev1.EnvVar{ diff --git a/controllers/flp/flp_pipeline_builder.go b/controllers/flp/flp_pipeline_builder.go index 38fecf90a..13740626b 100644 --- a/controllers/flp/flp_pipeline_builder.go +++ b/controllers/flp/flp_pipeline_builder.go @@ -87,8 +87,7 @@ func (b *PipelineBuilder) AddProcessorStages() error { ReporterIPField: "AgentIP", SrcHostField: "SrcK8S_HostIP", DstHostField: "DstK8S_HostIP", - FlowDirectionField: "FlowDirection", - IfDirectionField: "IfDirection", + FlowDirectionField: "NodeDirection", }, }) @@ -194,9 +193,9 @@ func flowMetricToFLP(flowMetric *metricslatest.FlowMetricSpec) (*api.MetricsItem m.Filters = append(m.Filters, api.MetricsFilter{Key: "Duplicate", Value: "false", Type: "exact"}) } if flowMetric.Direction == metricslatest.Egress { - m.Filters = append(m.Filters, api.MetricsFilter{Key: "FlowDirection", Value: "1|2", Type: "regex"}) + m.Filters = append(m.Filters, api.MetricsFilter{Key: "NodeDirection", Value: "1|2", Type: "regex"}) } else if flowMetric.Direction == metricslatest.Ingress { - m.Filters = append(m.Filters, api.MetricsFilter{Key: "FlowDirection", Value: "0|2", Type: "regex"}) + m.Filters = append(m.Filters, api.MetricsFilter{Key: "NodeDirection", Value: "0|2", Type: "regex"}) } for _, b := range flowMetric.Buckets { f, err := strconv.ParseFloat(b, 64) @@ -243,7 +242,7 @@ func (b *PipelineBuilder) addConnectionTracking(lastStage config.PipelineBuilder ReportMissing: true, }, { - Name: "FlowDirection", + Name: "NodeDirection", Operation: "first", ReportMissing: true, }, diff --git a/controllers/flp/flp_test.go b/controllers/flp/flp_test.go index afdb92176..776644320 100644 --- a/controllers/flp/flp_test.go +++ b/controllers/flp/flp_test.go @@ -656,6 +656,7 @@ func TestConfigMapShouldDeserializeAsJSONWithLokiManual(t *testing.T) { "DstK8S_OwnerName", "DstK8S_Type", "K8S_FlowLayer", + "NodeDirection", "_RecordType", }, lokiCfg.Labels) assert.Equal(`{app="netobserv-flowcollector"}`, fmt.Sprintf("%v", lokiCfg.StaticLabels)) @@ -711,8 +712,8 @@ func TestConfigMapShouldDeserializeAsJSONWithLokiStack(t *testing.T) { "DstK8S_OwnerName", "DstK8S_Type", "K8S_FlowLayer", + "NodeDirection", "_RecordType", - "FlowDirection", }, lokiCfg.Labels) assert.Equal(`{app="netobserv-flowcollector"}`, fmt.Sprintf("%v", lokiCfg.StaticLabels)) diff --git a/controllers/flp/metrics_api_test.go b/controllers/flp/metrics_api_test.go index dc4baadb2..b6f05e2a9 100644 --- a/controllers/flp/metrics_api_test.go +++ b/controllers/flp/metrics_api_test.go @@ -106,7 +106,7 @@ func TestFlowMetricToFLP(t *testing.T) { Filters: []api.MetricsFilter{ {Key: "f", Value: "v", Type: api.PromFilterRegex}, {Key: "f2", Type: api.PromFilterAbsence}, - {Key: "FlowDirection", Value: "1|2", Type: api.PromFilterRegex}, + {Key: "NodeDirection", Value: "1|2", Type: api.PromFilterRegex}, }, Labels: []string{"by_field"}, Buckets: []float64{1, 5, 10, 50, 100}, diff --git a/docs/flows-format.adoc b/docs/flows-format.adoc index 9311b2b83..794febd6c 100644 --- a/docs/flows-format.adoc +++ b/docs/flows-format.adoc @@ -116,7 +116,7 @@ The "Loki label" column is useful when querying Loki directly: label fields need - RST+ACK (0x400) | n/a | no -| `FlowDirection` +| `NodeDirection` | number | Flow direction from the node observation point. Can be one of: + - 0: Ingress (incoming traffic, from the node observation point) + diff --git a/hack/cloned.flows.netobserv.io_flowmetrics.yaml b/hack/cloned.flows.netobserv.io_flowmetrics.yaml index 50469918a..733ad630c 100644 --- a/hack/cloned.flows.netobserv.io_flowmetrics.yaml +++ b/hack/cloned.flows.netobserv.io_flowmetrics.yaml @@ -53,8 +53,8 @@ spec: default: Any description: 'Filter for ingress, egress or any direction flows. When set to `Ingress`, it is equivalent to adding the regex filter on - `FlowDirection`: `0|2`. When set to `Egress`, it is equivalent to - adding the regex filter on `FlowDirection`: `1|2`.' + `NodeDirection`: `0|2`. When set to `Egress`, it is equivalent to + adding the regex filter on `NodeDirection`: `1|2`.' enum: - Any - Egress @@ -63,7 +63,7 @@ spec: filters: description: '`filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must - be used to eliminate duplicates: `Duplicate:"false"` and `FlowDirection: + be used to eliminate duplicates: `Duplicate:"false"` and `NodeDirection: "0"`. Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/networking/network_observability/json-flows-format-reference.html.' items: diff --git a/pkg/loki/labels.go b/pkg/loki/labels.go index 2d25fded5..395441e6f 100644 --- a/pkg/loki/labels.go +++ b/pkg/loki/labels.go @@ -1,11 +1,8 @@ package loki import ( - "strconv" - flowslatest "github.com/netobserv/network-observability-operator/apis/flowcollector/v1beta2" "github.com/netobserv/network-observability-operator/controllers/constants" - "github.com/netobserv/network-observability-operator/controllers/ebpf" "github.com/netobserv/network-observability-operator/pkg/helper" ) @@ -24,17 +21,5 @@ func GetLokiLabels(desired *flowslatest.FlowCollectorSpec) []string { indexFields = append(indexFields, constants.LokiZoneIndexFields...) } - if helper.UseEBPF(desired) { - dedupJustMark, _ := strconv.ParseBool(ebpf.DedupeJustMarkDefault) - if desired.Agent.EBPF.Advanced != nil { - if v, ok := desired.Agent.EBPF.Advanced.Env[ebpf.EnvDedupeJustMark]; ok { - dedupJustMark, _ = strconv.ParseBool(v) - } - } - if dedupJustMark { - indexFields = append(indexFields, constants.LokiDeduperMarkIndexFields...) - } - } - return indexFields } diff --git a/pkg/metrics/predefined_metrics.go b/pkg/metrics/predefined_metrics.go index f9c52cfa5..f20ca5190 100644 --- a/pkg/metrics/predefined_metrics.go +++ b/pkg/metrics/predefined_metrics.go @@ -70,7 +70,7 @@ func init() { ValueKey: valueField, Filters: []flpapi.MetricsFilter{ {Key: "Duplicate", Value: "false"}, - {Key: "FlowDirection", Value: mapDirection[dir], Type: flpapi.PromFilterRegex}, + {Key: "NodeDirection", Value: mapDirection[dir], Type: flpapi.PromFilterRegex}, }, Labels: labels, },