Expose controller metrics in an isolated port (#1369)

The metrics will be exposed on the port 8081.

Signed-off-by: Alvaro Neira Ayuso <[email protected]>

contrib/prometheus-mixin/README.md

@@ -1,7 +1,7 @@
 # Sealed Secrets Metrics
 
 The Sealed Secrets Controller running in Kubernetes exposes Prometheus
-metrics on `*:8080/metrics`.
+metrics on `*:8081/metrics`.
 
 These metrics enable operators to observe how it is performing. For example 
 how many `SealedSecret` unseals have been attempted and how many errors may 
@@ -30,13 +30,13 @@ After installing the Sealed Secrets Controller you can access the metrics via
 Kubernetes port-forward to your pod:
 
 ```
-$ kubectl port-forward sealed-secrets-controller-6566dc69c6-lqr6x 8080 &
+$ kubectl port-forward sealed-secrets-controller-6566dc69c6-lqr6x 8081 &
 [1] 293283
 ```
 
 Then query the metrics endpoint:
 ```
-$ curl localhost:8080/metrics
+$ curl localhost:8081/metrics
 
 <snip>
 # HELP sealed_secrets_controller_build_info Build information.

controller-norbac.jsonnet

@@ -34,6 +34,21 @@ local namespace = 'kube-system';
     target_pod: $.controller.spec.template,
   },
 
+  service_metrics: kube.Service('sealed-secrets-controller-metrics') + $.namespace {
+    local service = self,
+    target_pod: $.controller.spec.template,
+    spec: {
+      selector: service.target_pod.metadata.labels,
+      ports: [
+        {
+          port: 8081,
+          targetPort: 8081,
+        },
+      ],
+      type: "ClusterIP",
+    },
+  },
+
   controller: kube.Deployment('sealed-secrets-controller') + $.namespace {
     spec+: {
       template+: {
@@ -57,6 +72,7 @@ local namespace = 'kube-system';
               livenessProbe: self.readinessProbe,
               ports_+: {
                 http: { containerPort: 8080 },
+                metrics: { containerPort: 8081 },
               },
               securityContext+: {
                 allowPrivilegeEscalation: false,

docker/controller.Dockerfile

@@ -6,6 +6,6 @@ USER 1001
 ARG TARGETARCH
 COPY dist/controller_linux_${TARGETARCH}*/controller /usr/local/bin/
 
-EXPOSE 8080
+EXPOSE 8080 8081
 
 ENTRYPOINT ["controller"]

docs/GKE.md

@@ -82,3 +82,14 @@ gcloud compute firewall-rules create gke-to-kubeseal-8080 \
   --target-tags "$NETWORK_TARGET_TAG" \
   --priority 1000
 ```
+
+Create the firewall rule to see the metrics
+
+```bash
+gcloud compute firewall-rules create gke-to-metrics-8081 \
+  --network "$NETWORK" \
+  --allow "tcp:8081" \
+  --source-ranges "$CP_IPV4_CIDR" \
+  --target-tags "$NETWORK_TARGET_TAG" \
+  --priority 1000
+```

helm/sealed-secrets/README.md

@@ -188,21 +188,25 @@ The command removes all the Kubernetes components associated with the chart and
 
 ### Metrics parameters
 
-| Name                                       | Description                                                                            | Value   |
-| ------------------------------------------ | -------------------------------------------------------------------------------------- | ------- |
-| `metrics.serviceMonitor.enabled`           | Specify if a ServiceMonitor will be deployed for Prometheus Operator                   | `false` |
-| `metrics.serviceMonitor.namespace`         | Namespace where Prometheus Operator is running in                                      | `""`    |
-| `metrics.serviceMonitor.labels`            | Extra labels for the ServiceMonitor                                                    | `{}`    |
-| `metrics.serviceMonitor.annotations`       | Extra annotations for the ServiceMonitor                                               | `{}`    |
-| `metrics.serviceMonitor.interval`          | How frequently to scrape metrics                                                       | `""`    |
-| `metrics.serviceMonitor.scrapeTimeout`     | Timeout after which the scrape is ended                                                | `""`    |
-| `metrics.serviceMonitor.honorLabels`       | Specify if ServiceMonitor endPoints will honor labels                                  | `true`  |
-| `metrics.serviceMonitor.metricRelabelings` | Specify additional relabeling of metrics                                               | `[]`    |
-| `metrics.serviceMonitor.relabelings`       | Specify general relabeling                                                             | `[]`    |
-| `metrics.dashboards.create`                | Specifies whether a ConfigMap with a Grafana dashboard configuration should be created | `false` |
-| `metrics.dashboards.labels`                | Extra labels to be added to the Grafana dashboard ConfigMap                            | `{}`    |
-| `metrics.dashboards.annotations`           | Annotations to be added to the Grafana dashboard ConfigMap                             | `{}`    |
-| `metrics.dashboards.namespace`             | Namespace where Grafana dashboard ConfigMap is deployed                                | `""`    |
+| Name                                       | Description                                                                            | Value       |
+| ------------------------------------------ | -------------------------------------------------------------------------------------- | ----------- |
+| `metrics.serviceMonitor.enabled`           | Specify if a ServiceMonitor will be deployed for Prometheus Operator                   | `false`     |
+| `metrics.serviceMonitor.namespace`         | Namespace where Prometheus Operator is running in                                      | `""`        |
+| `metrics.serviceMonitor.labels`            | Extra labels for the ServiceMonitor                                                    | `{}`        |
+| `metrics.serviceMonitor.annotations`       | Extra annotations for the ServiceMonitor                                               | `{}`        |
+| `metrics.serviceMonitor.interval`          | How frequently to scrape metrics                                                       | `""`        |
+| `metrics.serviceMonitor.scrapeTimeout`     | Timeout after which the scrape is ended                                                | `""`        |
+| `metrics.serviceMonitor.honorLabels`       | Specify if ServiceMonitor endPoints will honor labels                                  | `true`      |
+| `metrics.serviceMonitor.metricRelabelings` | Specify additional relabeling of metrics                                               | `[]`        |
+| `metrics.serviceMonitor.relabelings`       | Specify general relabeling                                                             | `[]`        |
+| `metrics.dashboards.create`                | Specifies whether a ConfigMap with a Grafana dashboard configuration should be created | `false`     |
+| `metrics.dashboards.labels`                | Extra labels to be added to the Grafana dashboard ConfigMap                            | `{}`        |
+| `metrics.dashboards.annotations`           | Annotations to be added to the Grafana dashboard ConfigMap                             | `{}`        |
+| `metrics.dashboards.namespace`             | Namespace where Grafana dashboard ConfigMap is deployed                                | `""`        |
+| `metrics.service.type`                     | Sealed Secret Metrics service type                                                     | `ClusterIP` |
+| `metrics.service.port`                     | Sealed Secret service Metrics HTTP port                                                | `8081`      |
+| `metrics.service.nodePort`                 | Node port for HTTP                                                                     | `""`        |
+| `metrics.service.annotations`              | Additional custom annotations for Sealed Secret Metrics service                        | `{}`        |
 
 ### PodDisruptionBudget Parameters
 

helm/sealed-secrets/templates/deployment.yaml

@@ -123,6 +123,8 @@ spec:
           ports:
             - containerPort: 8080
               name: http
+            - containerPort: 8081
+              name: metrics
           {{- if .Values.startupProbe.enabled }}
           startupProbe: {{- include "sealed-secrets.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }}
             tcpSocket:

helm/sealed-secrets/templates/service.yaml

@@ -29,4 +29,35 @@ spec:
       nodePort: null
       {{- end }}
   selector: {{- include "sealed-secrets.matchLabels" . | nindent 4 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "sealed-secrets.fullname" . }}-metrics
+  namespace: {{ include "sealed-secrets.namespace" . }}
+  {{- if or .Values.metrics.service.annotations .Values.commonAnnotations }}
+  annotations:
+    {{- if .Values.metrics.service.annotations }}
+    {{- include "sealed-secrets.render" (dict "value" .Values.metrics.service.annotations "context" $) | nindent 4 }}
+    {{- end }}
+    {{- if .Values.commonAnnotations }}
+    {{- include "sealed-secrets.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- end }}
+  labels: {{- include "sealed-secrets.labels" . | nindent 4 }}
+    {{- if .Values.metrics.service.labels }}
+    {{- include "sealed-secrets.render" ( dict "value" .Values.metrics.service.labels "context" $) | nindent 4 }}
+    {{- end }}
+spec:
+  type: {{ .Values.metrics.service.type }}
+  ports:
+    - name: http
+      port: {{ .Values.metrics.service.port }}
+      targetPort: http
+      {{- if and (or (eq .Values.metrics.service.type "NodePort") (eq .Values.metrics.service.type "LoadBalancer")) (not (empty .Values.metrics.service.nodePort)) }}
+      nodePort: {{ .Values.metrics.service.nodePort }}
+      {{- else if eq .Values.metrics.service.type "ClusterIP" }}
+      nodePort: null
+      {{- end }}
+  selector: {{- include "sealed-secrets.matchLabels" . | nindent 4 }}
 {{- end }}

helm/sealed-secrets/values.yaml

@@ -448,6 +448,25 @@ metrics:
     ##
     namespace: ""
 
+  ## Sealed Secret Metrics service parameters
+  ##
+  service:
+    ## @param metrics.service.type Sealed Secret Metrics service type
+    ##
+    type: ClusterIP
+    ## @param metrics.service.port Sealed Secret service Metrics HTTP port
+    ##
+    port: 8081
+    ## @param metrics.service.nodePort Node port for HTTP
+    ## Specify the nodePort value for the LoadBalancer and NodePort service types
+    ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+    ## NOTE: choose port between <30000-32767>
+    ##
+    nodePort: ""
+    ## @param metrics.service.annotations [object] Additional custom annotations for Sealed Secret Metrics service
+    ##
+    annotations: {}
+
 ## @section PodDisruptionBudget Parameters
 
 pdb:

pkg/controller/main.go

@@ -248,12 +248,21 @@ func Main(f *Flags, version string) error {
 	}
 
 	server := httpserver(cp, controller.AttemptUnseal, controller.Rotate, f.RateLimitBurst, f.RateLimitPerSecond)
+	serverMetrics := httpserverMetrics()
 
 	sigterm := make(chan os.Signal, 1)
 	signal.Notify(sigterm, syscall.SIGTERM)
 	<-sigterm
 
-	return server.Shutdown(context.Background())
+	if err := server.Shutdown(context.Background()); err != nil {
+		return err
+	}
+
+	if err := serverMetrics.Shutdown(context.Background()); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func prepareController(clientset kubernetes.Interface, namespace string, tweakopts func(*metav1.ListOptions), f *Flags, ssclientset versioned.Interface, keyRegistry *KeyRegistry) (*Controller, error) {

pkg/controller/server.go

@@ -17,9 +17,10 @@ import (
 )
 
 var (
-	listenAddr   = flag.String("listen-addr", ":8080", "HTTP serving address.")
-	readTimeout  = flag.Duration("read-timeout", 2*time.Minute, "HTTP request timeout.")
-	writeTimeout = flag.Duration("write-timeout", 2*time.Minute, "HTTP response timeout.")
+	listenAddr        = flag.String("listen-addr", ":8080", "HTTP serving address.")
+	listenMetricsAddr = flag.String("listen-metrics-addr", ":8081", "HTTP metrics serving address.")
+	readTimeout       = flag.Duration("read-timeout", 2*time.Minute, "HTTP request timeout.")
+	writeTimeout      = flag.Duration("write-timeout", 2*time.Minute, "HTTP response timeout.")
 )
 
 // Called on every request to /cert.  Errors will be logged and return a 500.
@@ -44,8 +45,6 @@ func httpserver(cp certProvider, sc secretChecker, sr secretRotator, burst int,
 		}
 	})
 
-	mux.Handle("/metrics", promhttp.Handler())
-
 	mux.Handle("/v1/verify", Instrument("/v1/verify", httpRateLimiter.RateLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		content, err := io.ReadAll(r.Body)
 		if err != nil {
@@ -119,6 +118,26 @@ func httpserver(cp certProvider, sc secretChecker, sr secretRotator, burst int,
 	return &server
 }
 
+func httpserverMetrics() *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+
+	server := http.Server{
+		Addr:              *listenMetricsAddr,
+		Handler:           mux,
+		ReadTimeout:       *readTimeout,
+		ReadHeaderTimeout: *readTimeout,
+		WriteTimeout:      *writeTimeout,
+	}
+
+	log.Infof("HTTP metrics server serving on %s", server.Addr)
+	go func() {
+		err := server.ListenAndServe()
+		log.Errorf("HTTP metrics server exiting: %v", err)
+	}()
+	return &server
+}
+
 func rateLimiter(burst int, rate int) throttled.HTTPRateLimiter {
 	store, err := memstore.New(65536)
 	if err != nil {