NETOBSERV-764 - ADD v1beta2 api version with simplified Loki configur…

…ation options (#329)

* ADD v1beta2 with new Loki integration fields

Signed-off-by: acmenezes <[email protected]>

* flp & console plugin helpers

* manual mode as default + optionnal lokistack

* tests for v1beta2 MANUAL and LOKISTACK modes

Signed-off-by: acmenezes <[email protected]>

* cp test + conversions + update bundle

* preserve loki config in conversions

* set v1beta2 as current

* FIX comments on v1beta2 and regenerate

Signed-off-by: acmenezes <[email protected]>

* ADD conversion on v1alpha1 to v1beta2 for loki modes

Signed-off-by: acmenezes <[email protected]>

* update bundle and fix test

* fix webhooks conversions

* addressed feedback

* remove v1beta2 from autogen

* fix tests & misc + deprecate v1alpha1

* moved hack crd & set webhook path for tests

* Update api/v1beta2/flowcollector_types.go

Co-authored-by: Joel Takvorian <[email protected]>

* addressed feedback + update bundle

---------

Signed-off-by: acmenezes <[email protected]>
Co-authored-by: Julien Pinsonneau <[email protected]>
Co-authored-by: Julien Pinsonneau <[email protected]>
Co-authored-by: Joel Takvorian <[email protected]>

.mk/sample.mk

@@ -3,9 +3,9 @@
 deploy-sample-cr:
 	@echo -e "\n==> Deploy sample CR"
 ifeq (main,$(VERSION))
-	kubectl apply -f ./config/samples/flows_v1beta1_flowcollector.yaml || true
+	kubectl apply -f ./config/samples/flows_v1beta2_flowcollector.yaml || true
 else
-	kubectl apply -f ./config/samples/flows_v1beta1_flowcollector_versioned.yaml || true
+	kubectl apply -f ./config/samples/flows_v1beta2_flowcollector_versioned.yaml || true
 endif
 
 # Undeploy the sample FlowCollector CR

Makefile

@@ -252,13 +252,14 @@ doc: crdoc ## Generate markdown documentation
 	$(CRDOC) --resources config/crd/bases/flows.netobserv.io_flowcollectors.yaml --output docs/FlowCollector.md
 
 generate-go-conversions: $(CONVERSION_GEN) ## Run all generate-go-conversions
-		$(MAKE) clean-generated-conversions SRC_DIRS="./api/v1alpha1"
+		$(MAKE) clean-generated-conversions SRC_DIRS="./api/v1alpha1,./api/v1beta1"
 		$(CONVERSION_GEN) \
 		--input-dirs=./api/v1alpha1 \
+		--input-dirs=./api/v1beta1 \
 		--build-tag=ignore_autogenerated_core \
 		--output-file-base=zz_generated.conversion \
 		$(CONVERSION_GEN_OUTPUT_BASE) \
-		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
+		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt		
 
 generate: gencode manifests doc generate-go-conversions ## Run all code/file generators
 

PROJECT

@@ -12,8 +12,8 @@ resources:
   domain: netobserv.io
   group: flows
   kind: FlowCollector
-  path: github.com/netobserv/network-observability-operator/api/v1beta1
-  version: v1beta1
+  path: github.com/netobserv/network-observability-operator/api/v1beta2
+  version: v1beta2
   webhooks:
     conversion: true
     webhookVersion: v1

README.md

@@ -44,7 +44,7 @@ To deploy the monitoring pipeline, this `make` target installs a `FlowCollector`
 make deploy-sample-cr
 ```
 
-Alternatively, you can [grab and edit](./config/samples/flows_v1beta1_flowcollector.yaml) this config before installing it.
+Alternatively, you can [grab and edit](./config/samples/flows_v1beta2_flowcollector.yaml) this config before installing it.
 
 You can still edit the `FlowCollector` after it's installed: the operator will take care about reconciling everything with the updated configuration:
 
@@ -61,7 +61,7 @@ To deploy a specific version of the operator, you need to switch to the related
 ```bash
 git checkout 0.1.2
 VERSION=0.1.2 make deploy deploy-loki deploy-grafana
-kubectl apply -f ./config/samples/flows_v1beta1_flowcollector_versioned.yaml
+kubectl apply -f ./config/samples/flows_v1beta2_flowcollector_versioned.yaml
 ```
 
 Beware that the version of the underlying components, such as flowlogs-pipeline, may be tied to the version of the operator (this is why we recommend switching the git branch). Breaking this correlation may result in crashes. The versions of the underlying components are defined in the `FlowCollector` resource as image tags.
@@ -110,7 +110,7 @@ To get dashboards, import [this file](./config/samples/dashboards/Network%20Obse
 
 ## Configuration
 
-The `FlowCollector` resource is used to configure the operator and its managed components. A comprehensive documentation is [available here](./docs/FlowCollector.md), and a full sample file [there](./config/samples/flows_v1beta1_flowcollector.yaml).
+The `FlowCollector` resource is used to configure the operator and its managed components. A comprehensive documentation is [available here](./docs/FlowCollector.md), and a full sample file [there](./config/samples/flows_v1beta2_flowcollector.yaml).
 
 To edit configuration in cluster, run:
 

api/v1alpha1/doc.go

@@ -12,5 +12,7 @@ limitations under the License.
 */
 
 // Package v1aplha1 contains the v1alpha1 API implementation.
-// +k8s:conversion-gen=github.com/netobserv/network-observability-operator/api/v1beta1
+// +k8s:conversion-gen=github.com/netobserv/network-observability-operator/api/v1beta2
+//
+// Deprecated: This package will be removed in one of the next releases.
 package v1alpha1

api/v1alpha1/flowcollector_types.go

@@ -672,6 +672,8 @@ type FlowCollectorStatus struct {
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[*].reason"
 
 // FlowCollector is the Schema for the flowcollectors API, which pilots and configures netflow collection.
+//
+// Deprecated: This package will be removed in one of the next releases.
 type FlowCollector struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -680,9 +682,11 @@ type FlowCollector struct {
 	Status FlowCollectorStatus `json:"status,omitempty"`
 }
 
-//+kubebuilder:object:root=true
+// +kubebuilder:object:root=true
 
 // FlowCollectorList contains a list of FlowCollector
+//
+// Deprecated: This package will be removed in one of the next releases.
 type FlowCollectorList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`

api/v1alpha1/flowcollector_webhook.go

@@ -20,63 +20,75 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/netobserv/network-observability-operator/api/v1beta1"
+	"github.com/netobserv/network-observability-operator/api/v1beta2"
 	utilconversion "github.com/netobserv/network-observability-operator/pkg/conversion"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apiconversion "k8s.io/apimachinery/pkg/conversion"
 	"sigs.k8s.io/controller-runtime/pkg/conversion"
 )
 
-// ConvertTo converts this v1alpha1 FlowCollector to its v1beta1 equivalent (the conversion Hub)
+// ConvertTo converts this v1alpha1 FlowCollector to its v1beta2 equivalent (the conversion Hub)
 // https://book.kubebuilder.io/multiversion-tutorial/conversion.html
 func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error {
-	dst := dstRaw.(*v1beta1.FlowCollector)
+	dst := dstRaw.(*v1beta2.FlowCollector)
 
-	if err := Convert_v1alpha1_FlowCollector_To_v1beta1_FlowCollector(r, dst, nil); err != nil {
-		return fmt.Errorf("copying v1alpha1.FlowCollector into v1beta1.FlowCollector: %w", err)
+	if err := Convert_v1alpha1_FlowCollector_To_v1beta2_FlowCollector(r, dst, nil); err != nil {
+		return fmt.Errorf("copying v1alpha1.FlowCollector into v1beta2.FlowCollector: %w", err)
 	}
 	dst.Status.Conditions = make([]v1.Condition, len(r.Status.Conditions))
 	copy(dst.Status.Conditions, r.Status.Conditions)
 
 	// Manually restore data.
-	restored := &v1beta1.FlowCollector{}
+	restored := &v1beta2.FlowCollector{}
 	if ok, err := utilconversion.UnmarshalData(r, restored); err != nil || !ok {
+		// fallback on current loki config as Manual mode if metadata are not available
+		dst.Spec.Loki.Mode = v1beta2.LokiModeManual
+		dst.Spec.Loki.Manual.IngesterURL = r.Spec.Loki.URL
+		dst.Spec.Loki.Manual.QuerierURL = r.Spec.Loki.QuerierURL
+		dst.Spec.Loki.Manual.StatusURL = r.Spec.Loki.StatusURL
+		dst.Spec.Loki.Manual.TenantID = r.Spec.Loki.TenantID
+		dst.Spec.Loki.Manual.AuthToken = r.Spec.Loki.AuthToken
+		if err := Convert_v1alpha1_ClientTLS_To_v1beta2_ClientTLS(&r.Spec.Loki.TLS, &dst.Spec.Loki.Manual.TLS, nil); err != nil {
+			return fmt.Errorf("copying v1alplha1.Loki.TLS into v1beta2.Loki.Manual.TLS: %w", err)
+		}
 		return err
 	}
 
-	dst.Spec.Processor.LogTypes = restored.Spec.Processor.LogTypes
+	// Agent
+	if restored.Spec.Agent.EBPF.Features != nil {
+		dst.Spec.Agent.EBPF.Features = make([]v1beta2.AgentFeature, len(restored.Spec.Agent.EBPF.Features))
+		copy(dst.Spec.Agent.EBPF.Features, restored.Spec.Agent.EBPF.Features)
+	}
 
+	// Processor
+	dst.Spec.Processor.LogTypes = restored.Spec.Processor.LogTypes
 	if restored.Spec.Processor.ConversationHeartbeatInterval != nil {
 		dst.Spec.Processor.ConversationHeartbeatInterval = restored.Spec.Processor.ConversationHeartbeatInterval
 	}
-
 	if restored.Spec.Processor.ConversationEndTimeout != nil {
 		dst.Spec.Processor.ConversationEndTimeout = restored.Spec.Processor.ConversationEndTimeout
 	}
-
 	if restored.Spec.Processor.ConversationTerminatingTimeout != nil {
 		dst.Spec.Processor.ConversationTerminatingTimeout = restored.Spec.Processor.ConversationTerminatingTimeout
 	}
-
 	if restored.Spec.Processor.Metrics.DisableAlerts != nil {
 		dst.Spec.Processor.Metrics.DisableAlerts = restored.Spec.Processor.Metrics.DisableAlerts
 	}
+	dst.Spec.Processor.Metrics.Server.TLS.InsecureSkipVerify = restored.Spec.Processor.Metrics.Server.TLS.InsecureSkipVerify
+	dst.Spec.Processor.Metrics.Server.TLS.ProvidedCaFile = restored.Spec.Processor.Metrics.Server.TLS.ProvidedCaFile
 
-	dst.Spec.Loki.Enable = restored.Spec.Loki.Enable
-
-	if restored.Spec.Agent.EBPF.Features != nil {
-		dst.Spec.Agent.EBPF.Features = make([]v1beta1.AgentFeature, len(restored.Spec.Agent.EBPF.Features))
-		copy(dst.Spec.Agent.EBPF.Features, restored.Spec.Agent.EBPF.Features)
-	}
-
-	dst.Spec.Loki.StatusTLS = restored.Spec.Loki.StatusTLS
+	// Kafka
 	dst.Spec.Kafka.SASL = restored.Spec.Kafka.SASL
 
-	dst.Spec.ConsolePlugin.Enable = restored.Spec.ConsolePlugin.Enable
-
-	dst.Spec.Processor.Metrics.Server.TLS.InsecureSkipVerify = restored.Spec.Processor.Metrics.Server.TLS.InsecureSkipVerify
-	dst.Spec.Processor.Metrics.Server.TLS.ProvidedCaFile = restored.Spec.Processor.Metrics.Server.TLS.ProvidedCaFile
+	// Loki
+	dst.Spec.Loki.Enable = restored.Spec.Loki.Enable
+	dst.Spec.Loki.Mode = restored.Spec.Loki.Mode
+	dst.Spec.Loki.Manual = restored.Spec.Loki.Manual
+	if restored.Spec.Loki.LokiStack != nil {
+		dst.Spec.Loki.LokiStack = restored.Spec.Loki.LokiStack
+	}
 
+	// Exporters
 	if restored.Spec.Exporters != nil {
 		for _, restoredExp := range restored.Spec.Exporters {
 			if !isExporterIn(restoredExp, dst.Spec.Exporters) {
@@ -88,7 +100,7 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error {
 	return nil
 }
 
-func isExporterIn(restoredExporter *v1beta1.FlowCollectorExporter, dstExporters []*v1beta1.FlowCollectorExporter) bool {
+func isExporterIn(restoredExporter *v1beta2.FlowCollectorExporter, dstExporters []*v1beta2.FlowCollectorExporter) bool {
 
 	for _, dstExp := range dstExporters {
 		if reflect.DeepEqual(restoredExporter, dstExp) {
@@ -98,12 +110,12 @@ func isExporterIn(restoredExporter *v1beta1.FlowCollectorExporter, dstExporters
 	return false
 }
 
-// ConvertFrom converts the hub version v1beta1 FlowCollector object to v1alpha1
+// ConvertFrom converts the hub version v1beta2 FlowCollector object to v1alpha1
 func (r *FlowCollector) ConvertFrom(srcRaw conversion.Hub) error {
-	src := srcRaw.(*v1beta1.FlowCollector)
+	src := srcRaw.(*v1beta2.FlowCollector)
 
-	if err := Convert_v1beta1_FlowCollector_To_v1alpha1_FlowCollector(src, r, nil); err != nil {
-		return fmt.Errorf("copying v1beta1.FlowCollector into v1alpha1.FlowCollector: %w", err)
+	if err := Convert_v1beta2_FlowCollector_To_v1alpha1_FlowCollector(src, r, nil); err != nil {
+		return fmt.Errorf("copying v1beta2.FlowCollector into v1alpha1.FlowCollector: %w", err)
 	}
 	r.Status.Conditions = make([]v1.Condition, len(src.Status.Conditions))
 	copy(r.Status.Conditions, src.Status.Conditions)
@@ -113,60 +125,67 @@ func (r *FlowCollector) ConvertFrom(srcRaw conversion.Hub) error {
 }
 
 func (r *FlowCollectorList) ConvertTo(dstRaw conversion.Hub) error {
-	dst := dstRaw.(*v1beta1.FlowCollectorList)
-	return Convert_v1alpha1_FlowCollectorList_To_v1beta1_FlowCollectorList(r, dst, nil)
+	dst := dstRaw.(*v1beta2.FlowCollectorList)
+	return Convert_v1alpha1_FlowCollectorList_To_v1beta2_FlowCollectorList(r, dst, nil)
 }
 
 func (r *FlowCollectorList) ConvertFrom(srcRaw conversion.Hub) error {
-	src := srcRaw.(*v1beta1.FlowCollectorList)
-	return Convert_v1beta1_FlowCollectorList_To_v1alpha1_FlowCollectorList(src, r, nil)
+	src := srcRaw.(*v1beta2.FlowCollectorList)
+	return Convert_v1beta2_FlowCollectorList_To_v1alpha1_FlowCollectorList(src, r, nil)
 }
 
 // This function need to be manually created because conversion-gen not able to create it intentionally because
-// we have new defined fields in v1beta1 not in v1alpha1
+// we have new defined fields in v1beta2 not in v1alpha1
 // nolint:golint,stylecheck,revive
-func Convert_v1beta1_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in *v1beta1.FlowCollectorFLP, out *FlowCollectorFLP, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in, out, s)
+func Convert_v1beta2_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in *v1beta2.FlowCollectorFLP, out *FlowCollectorFLP, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_FlowCollectorFLP_To_v1alpha1_FlowCollectorFLP(in, out, s)
 }
 
 // This function need to be manually created because conversion-gen not able to create it intentionally because
-// we have new defined fields in v1beta1 not in v1alpha1
+// we have new defined fields in v1beta2 not in v1alpha1
 // nolint:golint,stylecheck,revive
-func Convert_v1beta1_FLPMetrics_To_v1alpha1_FLPMetrics(in *v1beta1.FLPMetrics, out *FLPMetrics, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_FLPMetrics_To_v1alpha1_FLPMetrics(in, out, s)
+func Convert_v1beta2_FLPMetrics_To_v1alpha1_FLPMetrics(in *v1beta2.FLPMetrics, out *FLPMetrics, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_FLPMetrics_To_v1alpha1_FLPMetrics(in, out, s)
 }
 
 // This function need to be manually created because conversion-gen not able to create it intentionally because
-// we have new defined fields in v1beta1 not in v1alpha1
+// we have new defined fields in v1beta2 not in v1alpha1
+// nolint:golint,stylecheck,revive
+func Convert_v1beta2_FlowCollectorLoki_To_v1alpha1_FlowCollectorLoki(in *v1beta2.FlowCollectorLoki, out *FlowCollectorLoki, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_FlowCollectorLoki_To_v1alpha1_FlowCollectorLoki(in, out, s)
+}
+
+// This function need to be manually created because conversion-gen not able to create it intentionally because
+// we have new defined fields in v1beta2 not in v1alpha1
 // nolint:golint,stylecheck,revive
-func Convert_v1beta1_FlowCollectorLoki_To_v1alpha1_FlowCollectorLoki(in *v1beta1.FlowCollectorLoki, out *FlowCollectorLoki, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_FlowCollectorLoki_To_v1alpha1_FlowCollectorLoki(in, out, s)
+func Convert_v1alpha1_FlowCollectorLoki_To_v1beta2_FlowCollectorLoki(in *FlowCollectorLoki, out *v1beta2.FlowCollectorLoki, s apiconversion.Scope) error {
+	return autoConvert_v1alpha1_FlowCollectorLoki_To_v1beta2_FlowCollectorLoki(in, out, s)
 }
 
 // This function need to be manually created because conversion-gen not able to create it intentionally because
 // we have new defined fields in v1beta1 not in v1alpha1
 // nolint:golint,stylecheck,revive
-func Convert_v1beta1_FlowCollectorConsolePlugin_To_v1alpha1_FlowCollectorConsolePlugin(in *v1beta1.FlowCollectorConsolePlugin, out *FlowCollectorConsolePlugin, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_FlowCollectorConsolePlugin_To_v1alpha1_FlowCollectorConsolePlugin(in, out, s)
+func Convert_v1beta2_FlowCollectorConsolePlugin_To_v1alpha1_FlowCollectorConsolePlugin(in *v1beta2.FlowCollectorConsolePlugin, out *FlowCollectorConsolePlugin, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_FlowCollectorConsolePlugin_To_v1alpha1_FlowCollectorConsolePlugin(in, out, s)
 }
 
 // This function need to be manually created because conversion-gen not able to create it intentionally because
 // we have new defined fields in v1beta1 not in v1alpha1
 // nolint:golint,stylecheck,revive
-func Convert_v1beta1_FlowCollectorExporter_To_v1alpha1_FlowCollectorExporter(in *v1beta1.FlowCollectorExporter, out *FlowCollectorExporter, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_FlowCollectorExporter_To_v1alpha1_FlowCollectorExporter(in, out, s)
+func Convert_v1beta2_FlowCollectorExporter_To_v1alpha1_FlowCollectorExporter(in *v1beta2.FlowCollectorExporter, out *FlowCollectorExporter, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_FlowCollectorExporter_To_v1alpha1_FlowCollectorExporter(in, out, s)
 }
 
 // This function need to be manually created because conversion-gen not able to create it intentionally because
 // we have new defined fields in v1beta1 not in v1alpha1
 // nolint:golint,stylecheck,revive
-func Convert_v1beta1_FlowCollectorEBPF_To_v1alpha1_FlowCollectorEBPF(in *v1beta1.FlowCollectorEBPF, out *FlowCollectorEBPF, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_FlowCollectorEBPF_To_v1alpha1_FlowCollectorEBPF(in, out, s)
+func Convert_v1beta2_FlowCollectorEBPF_To_v1alpha1_FlowCollectorEBPF(in *v1beta2.FlowCollectorEBPF, out *FlowCollectorEBPF, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_FlowCollectorEBPF_To_v1alpha1_FlowCollectorEBPF(in, out, s)
 }
 
 // // This function need to be manually created because conversion-gen not able to create it intentionally because
-// // we have new defined fields in v1beta1 not in v1alpha1
+// // we have new defined fields in v1beta2 not in v1alpha1
 // // nolint:golint,stylecheck,revive
-func Convert_v1beta1_ServerTLS_To_v1alpha1_ServerTLS(in *v1beta1.ServerTLS, out *ServerTLS, s apiconversion.Scope) error {
-	return autoConvert_v1beta1_ServerTLS_To_v1alpha1_ServerTLS(in, out, s)
+func Convert_v1beta2_ServerTLS_To_v1alpha1_ServerTLS(in *v1beta2.ServerTLS, out *ServerTLS, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_ServerTLS_To_v1alpha1_ServerTLS(in, out, s)
 }