diff --git a/bundle/manifests/netobserv-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml b/bundle/manifests/netobserv-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml new file mode 100644 index 000000000..11ab4a0f1 --- /dev/null +++ b/bundle/manifests/netobserv-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: netobserv-metrics-monitor +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 30s + path: /metrics + port: https + scheme: https + tlsConfig: + caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt + serverName: netobserv-metrics-service.openshift-operators.svc + selector: + matchLabels: + control-plane: controller-manager diff --git a/bundle/manifests/netobserv-metrics-service_v1_service.yaml b/bundle/manifests/netobserv-metrics-service_v1_service.yaml index 8123bb69f..59b4d16b0 100644 --- a/bundle/manifests/netobserv-metrics-service_v1_service.yaml +++ b/bundle/manifests/netobserv-metrics-service_v1_service.yaml @@ -1,6 +1,8 @@ apiVersion: v1 kind: Service metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: manager-metrics-tls creationTimestamp: null labels: control-plane: controller-manager diff --git a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml index 33bc76745..8878d9a0d 100644 --- a/bundle/manifests/netobserv-operator.clusterserviceversion.yaml +++ b/bundle/manifests/netobserv-operator.clusterserviceversion.yaml @@ -432,6 +432,10 @@ spec: spec: clusterPermissions: - rules: + - nonResourceURLs: + - /metrics + verbs: + - get - apiGroups: - apiregistration.k8s.io resources: @@ -700,6 +704,8 @@ spec: - --upstream=http://127.0.0.1:8080/ - --logtostderr=true - --v=10 + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 name: kube-rbac-proxy ports: @@ -713,6 +719,10 @@ spec: requests: cpu: 5m memory: 64Mi + volumeMounts: + - mountPath: /etc/tls/private + name: manager-metric-tls + readOnly: true securityContext: runAsNonRoot: true serviceAccountName: netobserv-controller-manager @@ -722,6 +732,10 @@ spec: secret: defaultMode: 420 secretName: webhook-server-cert + - name: manager-metric-tls + secret: + defaultMode: 420 + secretName: manager-metrics-tls permissions: - rules: - apiGroups: diff --git a/config/openshift-olm/kustomization.yaml b/config/openshift-olm/kustomization.yaml index 92ba38763..2f30e6654 100644 --- a/config/openshift-olm/kustomization.yaml +++ b/config/openshift-olm/kustomization.yaml @@ -8,6 +8,14 @@ namespace: netobserv # field above. namePrefix: netobserv- +resources: +- monitor.yaml + +patches: + - path: ./monitoring-patch.yaml + target: + kind: Deployment + name: controller-manager # Labels to add to all resources and selectors. #commonLabels: # someName: someValue diff --git a/config/openshift-olm/monitor.yaml b/config/openshift-olm/monitor.yaml new file mode 100644 index 000000000..8eb75986a --- /dev/null +++ b/config/openshift-olm/monitor.yaml @@ -0,0 +1,21 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: metrics-monitor + namespace: system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + path: /metrics + interval: 30s + port: https + scheme: https + tlsConfig: + caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt + serverName: netobserv-metrics-service.openshift-operators.svc + selector: + matchLabels: + control-plane: controller-manager diff --git a/config/openshift-olm/monitoring-patch.yaml b/config/openshift-olm/monitoring-patch.yaml new file mode 100644 index 000000000..fa68cbbc1 --- /dev/null +++ b/config/openshift-olm/monitoring-patch.yaml @@ -0,0 +1,27 @@ +# Check that the 0 container is the expected one +- op: test + path: /spec/template/spec/containers/1/name + value: kube-rbac-proxy + +- op: add + path: "/spec/template/spec/volumes/-" + value: + name: manager-metric-tls + secret: + secretName: manager-metrics-tls + defaultMode: 420 + +- op: add + path: "/spec/template/spec/containers/1/volumeMounts" + value: + - name: manager-metric-tls + readOnly: true + mountPath: /etc/tls/private + +- op: add + path: "/spec/template/spec/containers/1/args/-" + value: '--tls-cert-file=/etc/tls/private/tls.crt' + +- op: add + path: "/spec/template/spec/containers/1/args/-" + value: '--tls-private-key-file=/etc/tls/private/tls.key' diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml index 92ba38763..c09c4ffab 100644 --- a/config/openshift/kustomization.yaml +++ b/config/openshift/kustomization.yaml @@ -8,10 +8,14 @@ namespace: netobserv # field above. namePrefix: netobserv- -# Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue +resources: +- monitor.yaml +patches: + - path: ./monitoring-patch.yaml + target: + kind: Deployment + name: controller-manager bases: - ../crd - ../rbac diff --git a/config/prometheus/monitor.yaml b/config/openshift/monitor.yaml similarity index 58% rename from config/prometheus/monitor.yaml rename to config/openshift/monitor.yaml index a2f501836..f3c6a618f 100644 --- a/config/prometheus/monitor.yaml +++ b/config/openshift/monitor.yaml @@ -1,4 +1,3 @@ - # Prometheus Monitor Service (Metrics) apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -9,12 +8,14 @@ metadata: namespace: system spec: endpoints: - - path: /metrics + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + path: /metrics + interval: 30s port: https scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: - insecureSkipVerify: true + caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt + serverName: netobserv-metrics-service.netobserv.svc selector: matchLabels: control-plane: controller-manager diff --git a/config/openshift/monitoring-patch.yaml b/config/openshift/monitoring-patch.yaml new file mode 100644 index 000000000..fa68cbbc1 --- /dev/null +++ b/config/openshift/monitoring-patch.yaml @@ -0,0 +1,27 @@ +# Check that the 0 container is the expected one +- op: test + path: /spec/template/spec/containers/1/name + value: kube-rbac-proxy + +- op: add + path: "/spec/template/spec/volumes/-" + value: + name: manager-metric-tls + secret: + secretName: manager-metrics-tls + defaultMode: 420 + +- op: add + path: "/spec/template/spec/containers/1/volumeMounts" + value: + - name: manager-metric-tls + readOnly: true + mountPath: /etc/tls/private + +- op: add + path: "/spec/template/spec/containers/1/args/-" + value: '--tls-cert-file=/etc/tls/private/tls.crt' + +- op: add + path: "/spec/template/spec/containers/1/args/-" + value: '--tls-private-key-file=/etc/tls/private/tls.key' diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml deleted file mode 100644 index ed137168a..000000000 --- a/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- monitor.yaml diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db47..000000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 03016d377..41c869640 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -3,6 +3,8 @@ kind: Service metadata: labels: control-plane: controller-manager + annotations: + service.beta.openshift.io/serving-cert-secret-name: manager-metrics-tls name: metrics-service namespace: system spec: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a6a..d0aab63f7 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -15,4 +15,3 @@ resources: - auth_proxy_service.yaml - auth_proxy_role.yaml - auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 81a7e4f38..d61fcc9de 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -6,6 +6,10 @@ metadata: creationTimestamp: null name: manager-role rules: +- nonResourceURLs: + - /metrics + verbs: + - get - apiGroups: - apiregistration.k8s.io resources: diff --git a/controllers/flowcollector_controller.go b/controllers/flowcollector_controller.go index 9eae3b4d1..02ebeb392 100644 --- a/controllers/flowcollector_controller.go +++ b/controllers/flowcollector_controller.go @@ -75,6 +75,7 @@ func NewFlowCollectorReconciler(client client.Client, scheme *runtime.Scheme, co //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=list;create;update;watch //+kubebuilder:rbac:groups=apiregistration.k8s.io,resources=apiservices,verbs=list;get;watch //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;create;delete;update;patch;list;watch +//+kubebuilder:rbac:urls="/metrics",verbs=get // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -325,11 +326,11 @@ func (r *FlowCollectorReconciler) reconcileOperator(ctx context.Context, clientH } if r.config.DownstreamDeployment { desiredRole := buildRoleMonitoringReader(ns) - if err := clientHelper.ReconcileRole(ctx, desiredRole); err != nil { + if err := clientHelper.ReconcileClusterRole(ctx, desiredRole); err != nil { return err } desiredBinding := buildRoleBindingMonitoringReader(ns) - if err := clientHelper.ReconcileRoleBinding(ctx, desiredBinding); err != nil { + if err := clientHelper.ReconcileClusterRoleBinding(ctx, desiredBinding); err != nil { return err } } diff --git a/controllers/flowcollector_objects.go b/controllers/flowcollector_objects.go index fbb90b2da..931181dc8 100644 --- a/controllers/flowcollector_objects.go +++ b/controllers/flowcollector_objects.go @@ -28,30 +28,34 @@ func buildNamespace(ns string, isDownstream bool) *corev1.Namespace { } } -func buildRoleMonitoringReader(ns string) *rbacv1.Role { - cr := rbacv1.Role{ +func buildRoleMonitoringReader(ns string) *rbacv1.ClusterRole { + cr := rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: constants.OperatorName + roleSuffix, - Namespace: ns, - }, - Rules: []rbacv1.PolicyRule{{APIGroups: []string{""}, - Verbs: []string{"get", "list", "watch"}, - Resources: []string{"pods", "services", "endpoints"}, + Name: constants.OperatorName + roleSuffix, }, + Rules: []rbacv1.PolicyRule{ + {APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch"}, + Resources: []string{"pods", "services", "endpoints"}, + }, + { + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get"}, + }, }, } return &cr } -func buildRoleBindingMonitoringReader(ns string) *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ +func buildRoleBindingMonitoringReader(ns string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: constants.OperatorName + roleSuffix, Namespace: ns, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", + Kind: "ClusterRole", Name: constants.OperatorName + roleSuffix, }, Subjects: []rbacv1.Subject{{