Beat CRD presets

Makefile

@@ -212,7 +212,7 @@ apply-operator:
 		$(MAKE) --no-print-directory -sC config/operator generate-namespace | kubectl apply -f -
 
 apply-psp:
-	kubectl apply -f config/dev/elastic-psp.yaml
+	kubectl apply -f config/psp/elastic-psp.yaml
 
 ALL_IN_ONE_OUTPUT_FILE=config/all-in-one.yaml
 

cmd/manager/main.go

@@ -169,9 +169,9 @@ func init() {
 		fmt.Sprintf("K8s secret mounted into the path designated by %s to be used for webhook certificates", operator.WebhookCertDirFlag),
 	)
 	Cmd.Flags().Bool(
-		operator.ManageBeatAutodiscoverRBACFlag,
+		operator.ManageBeatRBACFlag,
 		true,
-		"Determines whether the operator should set up bindings and service accounts for the Beats autodiscover feature",
+		"Determines whether the operator should set up service accounts and bindings for the Beats autodiscover feature and Beat CRD presets",
 	)
 
 	// enable using dashed notation in flags and underscores in env
@@ -257,7 +257,7 @@ func execute() {
 		CertDir: viper.GetString(operator.WebhookCertDirFlag),
 	}
 
-	manageBeatAutodiscoverRBAC := viper.GetBool(operator.ManageBeatAutodiscoverRBACFlag)
+	manageBeatRBAC := viper.GetBool(operator.ManageBeatRBACFlag)
 
 	// configure the manager cache based on the number of managed namespaces
 	managedNamespaces := viper.GetStringSlice(operator.NamespacesFlag)
@@ -272,8 +272,8 @@ func execute() {
 		// the manager cache should always include the operator namespace so that we can work with operator-internal resources
 		cachedNamespaces := append(managedNamespaces, operatorNamespace)
 
-		// include empty namespace so that we can work with cluster-wide (non-namespaced) resources needed with autodiscover
-		if manageBeatAutodiscoverRBAC {
+		// include empty namespace so that we can work with cluster-wide (non-namespaced) resources needed for Beat role management
+		if manageBeatRBAC {
 			cachedNamespaces = append(cachedNamespaces, "")
 		}
 
@@ -346,8 +346,8 @@ func execute() {
 		accessReviewer = rbac.NewPermissiveAccessReviewer()
 	}
 
-	if manageBeatAutodiscoverRBAC {
-		beatcommon.EnableAutodiscoverRBACManagement()
+	if manageBeatRBAC {
+		beatcommon.EnableRBACManagement()
 	}
 
 	if err = apmserver.Add(mgr, params); err != nil {

config/crds/all-crds.yaml

@@ -555,6 +555,20 @@ spec:
               description: Image is the Beat Docker image to deploy. Version and Type
                 have to match the Beat in the image.
               type: string
+            preset:
+              description: 'Preset specifies which built-in configuration the operator
+                should use. The configuration provided in a preset consists of: Beat
+                config, roles with permissions required by that config and podTemplate
+                for DaemonSet or Deployment. Preset must match the Beat `type`. If
+                `config` is provided, it replaces the config from preset entirely.
+                If `deployment` or `daemonSet` is provided it has to match Deployment
+                or DaemonSet in the preset. The `podTemplate` is then merged with
+                PodTemplate from the preset. If preset is not provided, both `config`
+                and `daemonSet` or `deployment` must be specified.'
+              enum:
+              - filebeat-k8s-autodiscover
+              - metricbeat-k8s-hosts
+              type: string
             serviceAccountName:
               description: ServiceAccountName is used to check access from the current
                 resource to Elasticsearch resource in a different namespace. Can only

config/crds/bases/beat.k8s.elastic.co_beats.yaml

@@ -12160,6 +12160,20 @@ spec:
               description: Image is the Beat Docker image to deploy. Version and Type
                 have to match the Beat in the image.
               type: string
+            preset:
+              description: 'Preset specifies which built-in configuration the operator
+                should use. The configuration provided in a preset consists of: Beat
+                config, roles with permissions required by that config and podTemplate
+                for DaemonSet or Deployment. Preset must match the Beat `type`. If
+                `config` is provided, it replaces the config from preset entirely.
+                If `deployment` or `daemonSet` is provided it has to match Deployment
+                or DaemonSet in the preset. The `podTemplate` is then merged with
+                PodTemplate from the preset. If preset is not provided, both `config`
+                and `daemonSet` or `deployment` must be specified.'
+              enum:
+              - filebeat-k8s-autodiscover
+              - metricbeat-k8s-hosts
+              type: string
             serviceAccountName:
               description: ServiceAccountName is used to check access from the current
                 resource to Elasticsearch resource in a different namespace. Can only

config/e2e/operator.yaml

@@ -60,8 +60,8 @@ rules:
   - update
   - patch
   - delete
-# required to allow the operator to bind service accounts it manages
-# to role that holds permissions needed for Beat autodiscover feature
+# required to allow the operator to bind service accounts it manages to role
+# that holds permissions needed for Beat autodiscover feature and Beat CRD presets
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:
@@ -70,6 +70,7 @@ rules:
   - bind
   resourceNames:
   - elastic-beat-autodiscover
+  - elastic-beat-preset-metricbeat-k8s-hosts
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:

config/e2e/rbac.yaml

@@ -27,7 +27,6 @@ rules:
       - podsecuritypolicies
     resourceNames:
       - elastic.restricted
-      - elastic.beat.restricted
     verbs:
       - use
   - apiGroups:
@@ -61,6 +60,23 @@ subjects:
     name: {{ .Operator.Name }}
     namespace: {{ .Operator.Namespace }}
 ---
+# ClusterRole allowing use of Beat-specific PSP
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: elastic-beat-restricted
+  labels:
+    test-run: {{ .TestRun }}
+rules:
+- apiGroups:
+  - extensions
+  resources:
+  - podsecuritypolicies
+  resourceNames:
+  - elastic.beat.restricted
+  verbs:
+  - use
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -92,6 +108,34 @@ rules:
       - delete
       - create
       - update
+  - apiGroups:
+    - policy
+    resources:
+    - podsecuritypolicies
+    verbs:
+    - get
+  # to dynamically bind service accounts to ClusterRole allowing to use PSP
+  - apiGroups:
+    - rbac.authorization.k8s.io
+    resources:
+    - clusterroles
+    verbs:
+    - bind
+    - get
+    resourceNames:
+    - elastic-beat-restricted
+  - apiGroups:
+    - "rbac.authorization.k8s.io"
+    resources:
+    - rolebindings
+    verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    - delete
   - apiGroups:
       - "security.openshift.io"
     resources:

config/operator/all-in-one/beat-roles.yaml

@@ -4,11 +4,60 @@ kind: ClusterRole
 metadata:
   name: elastic-beat-autodiscover
 rules:
-- apiGroups: [""]
+- apiGroups:
+  - ""
   resources:
   - namespaces
   - pods
   verbs:
   - get
   - watch
   - list
+---
+# permissions needed for Beat metricbeat-k8s-hosts preset
+# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: elastic-beat-preset-metricbeat-k8s-hosts
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - namespaces
+  - events
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "extensions"
+  resources:
+  - replicasets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  - deployments
+  - replicasets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/stats
+  verbs:
+  - get
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
+

config/operator/all-in-one/cluster_role.template.yaml

@@ -12,8 +12,8 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
-# required to allow the operator to bind service accounts it manages
-# to role that holds permissions needed for Beat autodiscover feature
+# required to allow the operator to bind service accounts it manages to role
+# that holds permissions needed for Beat autodiscover feature and Beat CRD presets
 - apiGroups:
   - "rbac.authorization.k8s.io"
   resources:
@@ -34,6 +34,7 @@ rules:
   - bind
   resourceNames:
   - elastic-beat-autodiscover
+  - elastic-beat-preset-metricbeat-k8s-hosts
 - apiGroups:
   - ""
   resources:

config/operator/namespace/cluster_role.template.yaml

@@ -42,8 +42,8 @@ rules:
   - update
   - patch
   - delete
-# required to allow the operator to bind service accounts it manages
-# to role that holds permissions needed for Beat autodiscover feature
+# required to allow the operator to bind service accounts it manages to role
+# that holds permissions needed for Beat autodiscover feature and Beat CRD presets
 - apiGroups:
   - "rbac.authorization.k8s.io"
   resources:
@@ -64,6 +64,7 @@ rules:
   - bind
   resourceNames:
   - elastic-beat-autodiscover
+  - elastic-beat-preset-metricbeat-k8s-hosts
 - apiGroups:
   - apps
   resources: