Enable operator metrics collection (#273)

* Enable operator metrics collection * Added operator metrics ssl and auth configuration
netobserv · Mar 14, 2023 · 4e15d42 · 4e15d42
1 parent 84c7fe8
commit 4e15d42
Show file tree

Hide file tree

Showing 16 changed files with 154 additions and 32 deletions.
diff --git a/bundle/manifests/netobserv-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml b/bundle/manifests/netobserv-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml
@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: netobserv-metrics-monitor
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    path: /metrics
+    port: https
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: netobserv-metrics-service.openshift-operators.svc
+  selector:
+    matchLabels:
+      control-plane: controller-manager
diff --git a/bundle/manifests/netobserv-metrics-service_v1_service.yaml b/bundle/manifests/netobserv-metrics-service_v1_service.yaml
@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: manager-metrics-tls
   creationTimestamp: null
   labels:
     control-plane: controller-manager

diff --git a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml
@@ -432,6 +432,10 @@ spec:
     spec:
       clusterPermissions:
       - rules:
+        - nonResourceURLs:
+          - /metrics
+          verbs:
+          - get
         - apiGroups:
           - apiregistration.k8s.io
           resources:
@@ -700,6 +704,8 @@ spec:
                 - --upstream=http://127.0.0.1:8080/
                 - --logtostderr=true
                 - --v=10
+                - --tls-cert-file=/etc/tls/private/tls.crt
+                - --tls-private-key-file=/etc/tls/private/tls.key
                 image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
                 name: kube-rbac-proxy
                 ports:
@@ -713,6 +719,10 @@ spec:
                   requests:
                     cpu: 5m
                     memory: 64Mi
+                volumeMounts:
+                - mountPath: /etc/tls/private
+                  name: manager-metric-tls
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: netobserv-controller-manager
@@ -722,6 +732,10 @@ spec:
                 secret:
                   defaultMode: 420
                   secretName: webhook-server-cert
+              - name: manager-metric-tls
+                secret:
+                  defaultMode: 420
+                  secretName: manager-metrics-tls
       permissions:
       - rules:
         - apiGroups:

diff --git a/config/openshift-olm/kustomization.yaml b/config/openshift-olm/kustomization.yaml
@@ -8,6 +8,14 @@ namespace: netobserv
 # field above.
 namePrefix: netobserv-
 
+resources:
+- monitor.yaml
+
+patches:
+  - path: ./monitoring-patch.yaml
+    target:
+      kind: Deployment
+      name: controller-manager
 # Labels to add to all resources and selectors.
 #commonLabels:
 #  someName: someValue

diff --git a/config/openshift-olm/monitor.yaml b/config/openshift-olm/monitor.yaml
@@ -0,0 +1,21 @@
+# Prometheus Monitor Service (Metrics)
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: metrics-monitor
+  namespace: system
+spec:
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      path: /metrics
+      interval: 30s
+      port: https
+      scheme: https
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: netobserv-metrics-service.openshift-operators.svc
+  selector:
+    matchLabels:
+      control-plane: controller-manager
diff --git a/config/openshift-olm/monitoring-patch.yaml b/config/openshift-olm/monitoring-patch.yaml
@@ -0,0 +1,27 @@
+# Check that the 0 container is the expected one
+- op: test
+  path: /spec/template/spec/containers/1/name
+  value: kube-rbac-proxy
+
+- op: add
+  path: "/spec/template/spec/volumes/-"
+  value:
+    name: manager-metric-tls
+    secret:
+      secretName: manager-metrics-tls
+      defaultMode: 420
+
+- op: add
+  path: "/spec/template/spec/containers/1/volumeMounts"
+  value:
+    - name: manager-metric-tls
+      readOnly: true
+      mountPath: /etc/tls/private
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-cert-file=/etc/tls/private/tls.crt'
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-private-key-file=/etc/tls/private/tls.key'
diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml
@@ -8,10 +8,14 @@ namespace: netobserv
 # field above.
 namePrefix: netobserv-
 
-# Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
+resources:
+- monitor.yaml
 
+patches:
+  - path: ./monitoring-patch.yaml
+    target:
+      kind: Deployment
+      name: controller-manager
 bases:
 - ../crd
 - ../rbac

diff --git a/config/prometheus/monitor.yaml → config/openshift/monitor.yaml b/config/prometheus/monitor.yaml → config/openshift/monitor.yaml
@@ -1,4 +1,3 @@
-
 # Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -9,12 +8,14 @@ metadata:
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      path: /metrics
+      interval: 30s
       port: https
       scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
       tlsConfig:
-        insecureSkipVerify: true
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: netobserv-metrics-service.netobserv.svc
   selector:
     matchLabels:
       control-plane: controller-manager
diff --git a/config/openshift/monitoring-patch.yaml b/config/openshift/monitoring-patch.yaml
@@ -0,0 +1,27 @@
+# Check that the 0 container is the expected one
+- op: test
+  path: /spec/template/spec/containers/1/name
+  value: kube-rbac-proxy
+
+- op: add
+  path: "/spec/template/spec/volumes/-"
+  value:
+    name: manager-metric-tls
+    secret:
+      secretName: manager-metrics-tls
+      defaultMode: 420
+
+- op: add
+  path: "/spec/template/spec/containers/1/volumeMounts"
+  value:
+    - name: manager-metric-tls
+      readOnly: true
+      mountPath: /etc/tls/private
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-cert-file=/etc/tls/private/tls.crt'
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-private-key-file=/etc/tls/private/tls.key'
diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml
diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml
diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml
@@ -3,6 +3,8 @@ kind: Service
 metadata:
   labels:
     control-plane: controller-manager
+  annotations:
+        service.beta.openshift.io/serving-cert-secret-name: manager-metrics-tls
   name: metrics-service
   namespace: system
 spec:

diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -15,4 +15,3 @@ resources:
 - auth_proxy_service.yaml
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -6,6 +6,10 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
 - apiGroups:
   - apiregistration.k8s.io
   resources:

diff --git a/controllers/flowcollector_controller.go b/controllers/flowcollector_controller.go
@@ -75,6 +75,7 @@ func NewFlowCollectorReconciler(client client.Client, scheme *runtime.Scheme, co
 //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=list;create;update;watch
 //+kubebuilder:rbac:groups=apiregistration.k8s.io,resources=apiservices,verbs=list;get;watch
 //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;create;delete;update;patch;list;watch
+//+kubebuilder:rbac:urls="/metrics",verbs=get
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -325,11 +326,11 @@ func (r *FlowCollectorReconciler) reconcileOperator(ctx context.Context, clientH
 	}
 	if r.config.DownstreamDeployment {
 		desiredRole := buildRoleMonitoringReader(ns)
-		if err := clientHelper.ReconcileRole(ctx, desiredRole); err != nil {
+		if err := clientHelper.ReconcileClusterRole(ctx, desiredRole); err != nil {
 			return err
 		}
 		desiredBinding := buildRoleBindingMonitoringReader(ns)
-		if err := clientHelper.ReconcileRoleBinding(ctx, desiredBinding); err != nil {
+		if err := clientHelper.ReconcileClusterRoleBinding(ctx, desiredBinding); err != nil {
 			return err
 		}
 	}

diff --git a/controllers/flowcollector_objects.go b/controllers/flowcollector_objects.go
@@ -28,30 +28,34 @@ func buildNamespace(ns string, isDownstream bool) *corev1.Namespace {
 	}
 }
 
-func buildRoleMonitoringReader(ns string) *rbacv1.Role {
-	cr := rbacv1.Role{
+func buildRoleMonitoringReader(ns string) *rbacv1.ClusterRole {
+	cr := rbacv1.ClusterRole{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      constants.OperatorName + roleSuffix,
-			Namespace: ns,
-		},
-		Rules: []rbacv1.PolicyRule{{APIGroups: []string{""},
-			Verbs:     []string{"get", "list", "watch"},
-			Resources: []string{"pods", "services", "endpoints"},
+			Name: constants.OperatorName + roleSuffix,
 		},
+		Rules: []rbacv1.PolicyRule{
+			{APIGroups: []string{""},
+				Verbs:     []string{"get", "list", "watch"},
+				Resources: []string{"pods", "services", "endpoints"},
+			},
+			{
+				NonResourceURLs: []string{"/metrics"},
+				Verbs:           []string{"get"},
+			},
 		},
 	}
 	return &cr
 }
 
-func buildRoleBindingMonitoringReader(ns string) *rbacv1.RoleBinding {
-	return &rbacv1.RoleBinding{
+func buildRoleBindingMonitoringReader(ns string) *rbacv1.ClusterRoleBinding {
+	return &rbacv1.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      constants.OperatorName + roleSuffix,
 			Namespace: ns,
 		},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "Role",
+			Kind:     "ClusterRole",
 			Name:     constants.OperatorName + roleSuffix,
 		},
 		Subjects: []rbacv1.Subject{{