Introduce Beat CRD

Makefile

@@ -152,9 +152,13 @@ shellcheck:
 install-crds: generate-crds
 	kubectl apply -f $(ALL_CRDS)
 
+# Install roles that operator expects to be present by default.
+install-roles:
+	kubectl apply -f ./config/operator/all-in-one/beat-roles.yaml
+
 # Run locally against the configured Kubernetes cluster, with port-forwarding enabled so that
 # the operator can reach services running in the cluster through k8s port-forward feature
-run: install-crds go-run
+run: install-crds install-roles go-run
 
 go-run:
 	# Run the operator locally with debug logs and operator image set to latest

cmd/manager/main.go

@@ -24,6 +24,8 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/controller/apmserver"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/association"
 	associationctl "github.com/elastic/cloud-on-k8s/pkg/controller/association/controller"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/beat"
+	beatcommon "github.com/elastic/cloud-on-k8s/pkg/controller/beat/common"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/container"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/operator"
@@ -166,6 +168,11 @@ func init() {
 		"",
 		fmt.Sprintf("K8s secret mounted into the path designated by %s to be used for webhook certificates", operator.WebhookCertDirFlag),
 	)
+	Cmd.Flags().Bool(
+		operator.ManageBeatAutodiscoverRBACFlag,
+		true,
+		"Determines whether the operator should set up bindings and service accounts for the Beats autodiscover feature",
+	)
 
 	// enable using dashed notation in flags and underscores in env
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -250,6 +257,8 @@ func execute() {
 		CertDir: viper.GetString(operator.WebhookCertDirFlag),
 	}
 
+	manageBeatAutodiscoverRBAC := viper.GetBool(operator.ManageBeatAutodiscoverRBACFlag)
+
 	// configure the manager cache based on the number of managed namespaces
 	managedNamespaces := viper.GetStringSlice(operator.NamespacesFlag)
 	switch {
@@ -260,8 +269,15 @@ func execute() {
 		opts.Namespace = managedNamespaces[0]
 	default:
 		log.Info("Operator configured to manage multiple namespaces", "namespaces", managedNamespaces, "operator_namespace", operatorNamespace)
-		// always include the operator namespace into the manager cache so that we can work with operator-internal resources in there
-		opts.NewCache = cache.MultiNamespacedCacheBuilder(append(managedNamespaces, operatorNamespace))
+		// the manager cache should always include the operator namespace so that we can work with operator-internal resources
+		cachedNamespaces := append(managedNamespaces, operatorNamespace)
+
+		// include empty namespace so that we can work with cluster-wide (non-namespaced) resources needed with autodiscover
+		if manageBeatAutodiscoverRBAC {
+			cachedNamespaces = append(cachedNamespaces, "")
+		}
+
+		opts.NewCache = cache.MultiNamespacedCacheBuilder(cachedNamespaces)
 	}
 
 	// only expose prometheus metrics if provided a non-zero port
@@ -330,6 +346,10 @@ func execute() {
 		accessReviewer = rbac.NewPermissiveAccessReviewer()
 	}
 
+	if manageBeatAutodiscoverRBAC {
+		beatcommon.EnableAutodiscoverRBACManagement()
+	}
+
 	if err = apmserver.Add(mgr, params); err != nil {
 		log.Error(err, "unable to create controller", "controller", "ApmServer")
 		os.Exit(1)
@@ -346,6 +366,10 @@ func execute() {
 		log.Error(err, "unable to create controller", "controller", "EnterpriseSearch")
 		os.Exit(1)
 	}
+	if err = beat.Add(mgr, params); err != nil {
+		log.Error(err, "unable to create controller", "controller", "Beat")
+		os.Exit(1)
+	}
 	if err = associationctl.AddApmES(mgr, accessReviewer, params); err != nil {
 		log.Error(err, "unable to create controller", "controller", "apm-es-association")
 		os.Exit(1)
@@ -362,6 +386,11 @@ func execute() {
 		log.Error(err, "unable to create controller", "controller", "ent-es-association")
 		os.Exit(1)
 	}
+	if err = associationctl.AddBeatES(mgr, accessReviewer, params); err != nil {
+		log.Error(err, "unable to create controller", "controller", "beat-es-association")
+		os.Exit(1)
+	}
+
 	if err = remoteca.Add(mgr, accessReviewer, params); err != nil {
 		log.Error(err, "unable to create controller", "controller", "RemoteClusterCertificateAuthorites")
 		os.Exit(1)

config/crds/all-crds.yaml

@@ -457,6 +457,151 @@ status:
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+  creationTimestamp: null
+  name: beats.beat.k8s.elastic.co
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.health
+    name: health
+    type: string
+  - JSONPath: .status.availableNodes
+    description: Available nodes
+    name: available
+    type: integer
+  - JSONPath: .status.expectedNodes
+    description: Expected nodes
+    name: expected
+    type: integer
+  - JSONPath: .spec.type
+    description: Beat type
+    name: type
+    type: string
+  - JSONPath: .spec.version
+    description: Beat version
+    name: version
+    type: string
+  - JSONPath: .metadata.creationTimestamp
+    name: age
+    type: date
+  group: beat.k8s.elastic.co
+  names:
+    categories:
+    - elastic
+    kind: Beat
+    listKind: BeatList
+    plural: beats
+    shortNames:
+    - beat
+    singular: beat
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: Beat is the Schema for the Beats API.
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: BeatSpec defines the desired state of a Beat.
+          properties:
+            config:
+              description: Config holds the Beat configuration. If provided, it will
+                override the default configuration.
+              type: object
+            daemonSet:
+              description: DaemonSet specifies the Beat should be deployed as a DaemonSet,
+                and allows providing its spec. Cannot be used along with `deployment`.
+                If both are absent a default for the Type is used.
+              properties: {}
+              type: object
+            deployment:
+              description: Deployment specifies the Beat should be deployed as a Deployment,
+                and allows providing its spec. Cannot be used along with `daemonSet`.
+                If both are absent a default for the Type is used.
+              properties:
+                replicas:
+                  format: int32
+                  type: integer
+              type: object
+            elasticsearchRef:
+              description: ElasticsearchRef is a reference to an Elasticsearch cluster
+                running in the same Kubernetes cluster.
+              properties:
+                name:
+                  description: Name of the Kubernetes object.
+                  type: string
+                namespace:
+                  description: Namespace of the Kubernetes object. If empty, defaults
+                    to the current namespace.
+                  type: string
+              required:
+              - name
+              type: object
+            image:
+              description: Image is the Beat Docker image to deploy. Version and Type
+                have to match the Beat in the image.
+              type: string
+            serviceAccountName:
+              description: ServiceAccountName is used to check access from the current
+                resource to Elasticsearch resource in a different namespace. Can only
+                be used if ECK is enforcing RBAC on references.
+              type: string
+            type:
+              description: Type is the type of the Beat to deploy (filebeat, metricbeat,
+                etc.). Any string can be used, but well-known types will be recognized
+                and will allow to provide sane default configurations.
+              maxLength: 20
+              type: string
+            version:
+              description: Version of the Beat.
+              type: string
+          required:
+          - type
+          - version
+          type: object
+        status:
+          description: BeatStatus defines the observed state of a Beat.
+          properties:
+            associationStatus:
+              description: AssociationStatus is the status of an association resource.
+              type: string
+            availableNodes:
+              format: int32
+              type: integer
+            expectedNodes:
+              format: int32
+              type: integer
+            health:
+              type: string
+          type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.5