From 13c0e7e31603ea9c18c94a532407a9b33b01470d Mon Sep 17 00:00:00 2001 From: Olivier Cazade Date: Thu, 2 Nov 2023 16:23:23 +0100 Subject: [PATCH] Added flag to enable multiCluster configuration --- api/v1alpha1/flowcollector_webhook.go | 7 ++++ api/v1alpha1/zz_generated.conversion.go | 1 + api/v1beta1/flowcollector_types.go | 4 +++ api/v1beta1/zz_generated.conversion.go | 2 ++ api/v1beta1/zz_generated.deepcopy.go | 5 +++ api/v1beta2/flowcollector_types.go | 4 +++ api/v1beta2/zz_generated.deepcopy.go | 5 +++ .../flows.netobserv.io_flowcollectors.yaml | 12 +++++++ .../flows.netobserv.io_flowcollectors.yaml | 12 +++++++ .../flowcollector_controller_iso_test.go | 1 + .../flowlogspipeline/flp_pipeline_builder.go | 33 +++++++++++-------- docs/FlowCollector.md | 18 ++++++++++ ...ned.flows.netobserv.io_flowcollectors.yaml | 8 +++++ 13 files changed, 99 insertions(+), 13 deletions(-) diff --git a/api/v1alpha1/flowcollector_webhook.go b/api/v1alpha1/flowcollector_webhook.go index 45dabf072..cddb3a08b 100644 --- a/api/v1alpha1/flowcollector_webhook.go +++ b/api/v1alpha1/flowcollector_webhook.go @@ -65,6 +65,13 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error { if restored.Spec.Processor.Metrics.DisableAlerts != nil { dst.Spec.Processor.Metrics.DisableAlerts = restored.Spec.Processor.Metrics.DisableAlerts } + if restored.Spec.Processor.ClusterName != "" { + dst.Spec.Processor.ClusterName = restored.Spec.Processor.ClusterName + } + if restored.Spec.Processor.MultiClusterDeployment != nil { + dst.Spec.Processor.MultiClusterDeployment = restored.Spec.Processor.MultiClusterDeployment + } + dst.Spec.Processor.Metrics.Server.TLS.InsecureSkipVerify = restored.Spec.Processor.Metrics.Server.TLS.InsecureSkipVerify dst.Spec.Processor.Metrics.Server.TLS.ProvidedCaFile = restored.Spec.Processor.Metrics.Server.TLS.ProvidedCaFile diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go index 7f3b86a99..bf0a143b9 100644 --- a/api/v1alpha1/zz_generated.conversion.go +++ b/api/v1alpha1/zz_generated.conversion.go @@ -720,6 +720,7 @@ func autoConvert_v1beta2_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in *v1bet // WARNING: in.ConversationEndTimeout requires manual conversion: does not exist in peer-type // WARNING: in.ConversationTerminatingTimeout requires manual conversion: does not exist in peer-type // WARNING: in.ClusterName requires manual conversion: does not exist in peer-type + // WARNING: in.MultiClusterDeployment requires manual conversion: does not exist in peer-type if err := Convert_v1beta2_DebugConfig_To_v1alpha1_DebugConfig(&in.Debug, &out.Debug, s); err != nil { return err } diff --git a/api/v1beta1/flowcollector_types.go b/api/v1beta1/flowcollector_types.go index 040b1dfec..0ead28652 100644 --- a/api/v1beta1/flowcollector_types.go +++ b/api/v1beta1/flowcollector_types.go @@ -480,6 +480,10 @@ type FlowCollectorFLP struct { // `clusterName` is the name of the cluster to appear in the flows data. This is useful in a multi-cluster context. When using OpenShift, leave empty to make it automatically determined. ClusterName string `json:"clusterName,omitempty"` + //+kubebuilder:default:=false + // Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data + MultiClusterDeployment *bool `json:"multiClusterDeployment,omitempty"` + // `debug` allows setting some aspects of the internal configuration of the flow processor. // This section is aimed exclusively for debugging and fine-grained performance optimizations, // such as `GOGC` and `GOMAXPROCS` env vars. Users setting its values do it at their own risk. diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index e150d3944..cedfec1f5 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -681,6 +681,7 @@ func autoConvert_v1beta1_FlowCollectorFLP_To_v1beta2_FlowCollectorFLP(in *FlowCo out.ConversationEndTimeout = (*v1.Duration)(unsafe.Pointer(in.ConversationEndTimeout)) out.ConversationTerminatingTimeout = (*v1.Duration)(unsafe.Pointer(in.ConversationTerminatingTimeout)) out.ClusterName = in.ClusterName + out.MultiClusterDeployment = (*bool)(unsafe.Pointer(in.MultiClusterDeployment)) if err := Convert_v1beta1_DebugConfig_To_v1beta2_DebugConfig(&in.Debug, &out.Debug, s); err != nil { return err } @@ -715,6 +716,7 @@ func autoConvert_v1beta2_FlowCollectorFLP_To_v1beta1_FlowCollectorFLP(in *v1beta out.ConversationEndTimeout = (*v1.Duration)(unsafe.Pointer(in.ConversationEndTimeout)) out.ConversationTerminatingTimeout = (*v1.Duration)(unsafe.Pointer(in.ConversationTerminatingTimeout)) out.ClusterName = in.ClusterName + out.MultiClusterDeployment = (*bool)(unsafe.Pointer(in.MultiClusterDeployment)) if err := Convert_v1beta2_DebugConfig_To_v1beta1_DebugConfig(&in.Debug, &out.Debug, s); err != nil { return err } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index ee5647448..f44cc3cb9 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -352,6 +352,11 @@ func (in *FlowCollectorFLP) DeepCopyInto(out *FlowCollectorFLP) { *out = new(v1.Duration) **out = **in } + if in.MultiClusterDeployment != nil { + in, out := &in.MultiClusterDeployment, &out.MultiClusterDeployment + *out = new(bool) + **out = **in + } in.Debug.DeepCopyInto(&out.Debug) } diff --git a/api/v1beta2/flowcollector_types.go b/api/v1beta2/flowcollector_types.go index 7839ccdfe..c75e85bdf 100644 --- a/api/v1beta2/flowcollector_types.go +++ b/api/v1beta2/flowcollector_types.go @@ -472,6 +472,10 @@ type FlowCollectorFLP struct { // `clusterName` is the name of the cluster to appear in the flows data. This is useful in a multi-cluster context. When using OpenShift, leave empty to make it automatically determined. ClusterName string `json:"clusterName,omitempty"` + //+kubebuilder:default:=false + // Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data + MultiClusterDeployment *bool `json:"multiClusterDeployment,omitempty"` + // `debug` allows setting some aspects of the internal configuration of the flow processor. // This section is aimed exclusively for debugging and fine-grained performance optimizations, // such as `GOGC` and `GOMAXPROCS` env vars. Users setting its values do it at their own risk. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 74f820cfc..9b14de746 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -347,6 +347,11 @@ func (in *FlowCollectorFLP) DeepCopyInto(out *FlowCollectorFLP) { *out = new(v1.Duration) **out = **in } + if in.MultiClusterDeployment != nil { + in, out := &in.MultiClusterDeployment, &out.MultiClusterDeployment + *out = new(bool) + **out = **in + } in.Debug.DeepCopyInto(&out.Debug) } diff --git a/bundle/manifests/flows.netobserv.io_flowcollectors.yaml b/bundle/manifests/flows.netobserv.io_flowcollectors.yaml index a0a7678e5..a788c24c0 100644 --- a/bundle/manifests/flows.netobserv.io_flowcollectors.yaml +++ b/bundle/manifests/flows.netobserv.io_flowcollectors.yaml @@ -4941,6 +4941,12 @@ spec: type: object type: object type: object + multiClusterDeployment: + default: false + description: Set `multiClusterDeployment` to `true` to enable + multi clusters feature. This will add clusterName label to flows + data + type: boolean port: default: 2055 description: Port of the flow collector (host port). By convention, @@ -7817,6 +7823,12 @@ spec: type: object type: object type: object + multiClusterDeployment: + default: false + description: Set `multiClusterDeployment` to `true` to enable + multi clusters feature. This will add clusterName label to flows + data + type: boolean port: default: 2055 description: Port of the flow collector (host port). By convention, diff --git a/config/crd/bases/flows.netobserv.io_flowcollectors.yaml b/config/crd/bases/flows.netobserv.io_flowcollectors.yaml index 699acb2e9..b8f31cba8 100644 --- a/config/crd/bases/flows.netobserv.io_flowcollectors.yaml +++ b/config/crd/bases/flows.netobserv.io_flowcollectors.yaml @@ -4927,6 +4927,12 @@ spec: type: object type: object type: object + multiClusterDeployment: + default: false + description: Set `multiClusterDeployment` to `true` to enable + multi clusters feature. This will add clusterName label to flows + data + type: boolean port: default: 2055 description: Port of the flow collector (host port). By convention, @@ -7803,6 +7809,12 @@ spec: type: object type: object type: object + multiClusterDeployment: + default: false + description: Set `multiClusterDeployment` to `true` to enable + multi clusters feature. This will add clusterName label to flows + data + type: boolean port: default: 2055 description: Port of the flow collector (host port). By convention, diff --git a/controllers/flowcollector_controller_iso_test.go b/controllers/flowcollector_controller_iso_test.go index 563c85ead..491448238 100644 --- a/controllers/flowcollector_controller_iso_test.go +++ b/controllers/flowcollector_controller_iso_test.go @@ -70,6 +70,7 @@ func flowCollectorIsoSpecs() { ConversationHeartbeatInterval: &metav1.Duration{Duration: time.Second}, ConversationEndTimeout: &metav1.Duration{Duration: time.Second}, ConversationTerminatingTimeout: &metav1.Duration{Duration: time.Second}, + MultiClusterDeployment: ptr.To(true), ClusterName: "testCluster", Debug: flowslatest.DebugConfig{}, LogTypes: &outputRecordTypes, diff --git a/controllers/flowlogspipeline/flp_pipeline_builder.go b/controllers/flowlogspipeline/flp_pipeline_builder.go index bb842191f..8eb2cc7d9 100644 --- a/controllers/flowlogspipeline/flp_pipeline_builder.go +++ b/controllers/flowlogspipeline/flp_pipeline_builder.go @@ -22,6 +22,7 @@ const ( conntrackTerminatingTimeout = 5 * time.Second conntrackEndTimeout = 10 * time.Second conntrackHeartbeatInterval = 30 * time.Second + clusterNameLabelName = "K8S_ClusterName" ) type PipelineBuilder struct { @@ -53,6 +54,10 @@ func (b *PipelineBuilder) AddProcessorStages() error { indexFields, lastStage = b.addConnectionTracking(indexFields, lastStage) + if b.desired.Processor.MultiClusterDeployment != nil && *b.desired.Processor.MultiClusterDeployment { + indexFields = append(indexFields, clusterNameLabelName) + } + // enrich stage (transform) configuration enrichedStage := lastStage.TransformNetwork("enrich", api.TransformNetwork{ Rules: api.NetworkTransformRules{{ @@ -316,19 +321,21 @@ func (b *PipelineBuilder) addTransformFilter(lastStage config.PipelineBuilderSta var clusterName string transformFilterRules := []api.TransformFilterRule{} - if b.desired.Processor.ClusterName != "" { - clusterName = b.desired.Processor.ClusterName - } else { - //take clustername from openshift - clusterName = string(globals.DefaultClusterID) - } - if clusterName != "" { - transformFilterRules = []api.TransformFilterRule{ - { - Input: "K8S_ClusterName", - Type: "add_field_if_doesnt_exist", - Value: clusterName, - }, + if b.desired.Processor.MultiClusterDeployment != nil && *b.desired.Processor.MultiClusterDeployment { + if b.desired.Processor.ClusterName != "" { + clusterName = b.desired.Processor.ClusterName + } else { + //take clustername from openshift + clusterName = string(globals.DefaultClusterID) + } + if clusterName != "" { + transformFilterRules = []api.TransformFilterRule{ + { + Input: clusterNameLabelName, + Type: "add_field_if_doesnt_exist", + Value: clusterName, + }, + } } } diff --git a/docs/FlowCollector.md b/docs/FlowCollector.md index 97affae55..d4d0e62b9 100644 --- a/docs/FlowCollector.md +++ b/docs/FlowCollector.md @@ -7537,6 +7537,15 @@ TLS client configuration for Loki URL. `Metrics` define the processor configuration regarding metrics
false + + multiClusterDeployment + boolean + + Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data
+
+ Default: false
+ + false port integer @@ -12707,6 +12716,15 @@ TLS client configuration for Loki URL. `Metrics` define the processor configuration regarding metrics
false + + multiClusterDeployment + boolean + + Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data
+
+ Default: false
+ + false port integer diff --git a/hack/cloned.flows.netobserv.io_flowcollectors.yaml b/hack/cloned.flows.netobserv.io_flowcollectors.yaml index d0bf5a406..25afda988 100644 --- a/hack/cloned.flows.netobserv.io_flowcollectors.yaml +++ b/hack/cloned.flows.netobserv.io_flowcollectors.yaml @@ -3414,6 +3414,10 @@ spec: type: object type: object type: object + multiClusterDeployment: + default: false + description: Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data + type: boolean port: default: 2055 description: Port of the flow collector (host port). By convention, some values are forbidden. It must be greater than 1024 and different from 4500, 4789 and 6081. @@ -5395,6 +5399,10 @@ spec: type: object type: object type: object + multiClusterDeployment: + default: false + description: Set `multiClusterDeployment` to `true` to enable multi clusters feature. This will add clusterName label to flows data + type: boolean port: default: 2055 description: Port of the flow collector (host port). By convention, some values are forbidden. It must be greater than 1024 and different from 4500, 4789 and 6081.