From 587a65c9837bd36228001b378e8cf644c9412518 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Wed, 8 Mar 2023 15:37:41 +0900 Subject: [PATCH] [release-v1.8] add probes to all deployments (#199) * add probes to all deployments (#176) * add missing probes (#13563) * Bump manifests --------- Co-authored-by: Stavros Kontopoulos --- config/core/deployments/controller.yaml | 17 +++ .../deployments/domainmapping-controller.yaml | 17 +++ config/hpa-autoscaling/controller.yaml | 17 +++ openshift/release/artifacts/serving-core.yaml | 34 ++++++ openshift/release/artifacts/serving-hpa.yaml | 17 +++ .../knative.dev/pkg/injection/health_check.go | 109 ++++++++++++++++++ .../pkg/injection/sharedmain/main.go | 18 +++ 7 files changed, 229 insertions(+) create mode 100644 vendor/knative.dev/pkg/injection/health_check.go diff --git a/config/core/deployments/controller.yaml b/config/core/deployments/controller.yaml index e8bf992b5327..940a6bdf4627 100644 --- a/config/core/deployments/controller.yaml +++ b/config/core/deployments/controller.yaml @@ -89,11 +89,28 @@ spec: seccompProfile: type: RuntimeDefault + livenessProbe: + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + ports: - name: metrics containerPort: 9090 - name: profiling containerPort: 8008 + - name: probes + containerPort: 8080 --- apiVersion: v1 diff --git a/config/core/deployments/domainmapping-controller.yaml b/config/core/deployments/domainmapping-controller.yaml index e220d15b03e7..0694317c853a 100644 --- a/config/core/deployments/domainmapping-controller.yaml +++ b/config/core/deployments/domainmapping-controller.yaml @@ -85,8 +85,25 @@ spec: seccompProfile: type: RuntimeDefault + livenessProbe: + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + ports: - name: metrics containerPort: 9090 - name: profiling containerPort: 8008 + - name: probes + containerPort: 8080 diff --git a/config/hpa-autoscaling/controller.yaml b/config/hpa-autoscaling/controller.yaml index 8e95aab3e999..eb2a650aba8b 100644 --- a/config/hpa-autoscaling/controller.yaml +++ b/config/hpa-autoscaling/controller.yaml @@ -86,11 +86,28 @@ spec: seccompProfile: type: RuntimeDefault + livenessProbe: + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + ports: - name: metrics containerPort: 9090 - name: profiling containerPort: 8008 + - name: probes + containerPort: 8080 --- apiVersion: v1 diff --git a/openshift/release/artifacts/serving-core.yaml b/openshift/release/artifacts/serving-core.yaml index 904a5d34c332..81b4092b37d3 100644 --- a/openshift/release/artifacts/serving-core.yaml +++ b/openshift/release/artifacts/serving-core.yaml @@ -6150,11 +6150,28 @@ spec: + livenessProbe: + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + ports: - name: metrics containerPort: 9090 - name: profiling containerPort: 8008 + - name: probes + containerPort: 8080 --- apiVersion: v1 @@ -6266,11 +6283,28 @@ spec: + livenessProbe: + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + ports: - name: metrics containerPort: 9090 - name: profiling containerPort: 8008 + - name: probes + containerPort: 8080 --- # Copyright 2020 The Knative Authors # diff --git a/openshift/release/artifacts/serving-hpa.yaml b/openshift/release/artifacts/serving-hpa.yaml index 7c70bc360662..f1c0a1c65d19 100644 --- a/openshift/release/artifacts/serving-hpa.yaml +++ b/openshift/release/artifacts/serving-hpa.yaml @@ -87,11 +87,28 @@ spec: + livenessProbe: + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + failureThreshold: 5 + ports: - name: metrics containerPort: 9090 - name: profiling containerPort: 8008 + - name: probes + containerPort: 8080 --- apiVersion: v1 diff --git a/vendor/knative.dev/pkg/injection/health_check.go b/vendor/knative.dev/pkg/injection/health_check.go new file mode 100644 index 000000000000..2899c7e35b47 --- /dev/null +++ b/vendor/knative.dev/pkg/injection/health_check.go @@ -0,0 +1,109 @@ +/* +Copyright 2023 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package injection + +import ( + "context" + "errors" + "net/http" + "strconv" + "time" + + "knative.dev/pkg/logging" +) + +// HealthCheckDefaultPort defines the default port number for health probes +const HealthCheckDefaultPort = 8080 + +// ServeHealthProbes sets up liveness and readiness probes. +// If user sets no probes explicitly via the context then defaults are added. +func ServeHealthProbes(ctx context.Context, port int) error { + logger := logging.FromContext(ctx) + server := http.Server{ReadHeaderTimeout: time.Minute, Handler: muxWithHandles(ctx), Addr: ":" + strconv.Itoa(port)} + go func() { + <-ctx.Done() + _ = server.Shutdown(ctx) + }() + + // start the web server on port and accept requests + logger.Infof("Probes server listening on port %s", port) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} + +func muxWithHandles(ctx context.Context) *http.ServeMux { + mux := http.NewServeMux() + readiness := getReadinessHandleOrDefault(ctx) + liveness := getLivenessHandleOrDefault(ctx) + mux.HandleFunc("/readiness", *readiness) + mux.HandleFunc("/health", *liveness) + return mux +} + +func newDefaultProbesHandle(sigCtx context.Context) http.HandlerFunc { + logger := logging.FromContext(sigCtx) + return func(w http.ResponseWriter, r *http.Request) { + f := func() error { + select { + // When we get SIGTERM (sigCtx done), let readiness probes start failing. + case <-sigCtx.Done(): + logger.Info("Signal context canceled") + return errors.New("received SIGTERM from kubelet") + default: + return nil + } + } + if err := f(); err != nil { + logger.Errorf("Healthcheck failed: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } else { + w.WriteHeader(http.StatusOK) + } + } +} + +type addReadinessKey struct{} + +// AddReadiness signals to probe setup logic to add a user provided probe handler +func AddReadiness(ctx context.Context, handlerFunc http.HandlerFunc) context.Context { + return context.WithValue(ctx, addReadinessKey{}, &handlerFunc) +} + +func getReadinessHandleOrDefault(ctx context.Context) *http.HandlerFunc { + if ctx.Value(addReadinessKey{}) != nil { + return ctx.Value(addReadinessKey{}).(*http.HandlerFunc) + } + defaultHandle := newDefaultProbesHandle(ctx) + return &defaultHandle +} + +type addLivenessKey struct{} + +// AddLiveness signals to probe setup logic to add a user provided probe handler +func AddLiveness(ctx context.Context, handlerFunc http.HandlerFunc) context.Context { + return context.WithValue(ctx, addLivenessKey{}, &handlerFunc) +} + +func getLivenessHandleOrDefault(ctx context.Context) *http.HandlerFunc { + if ctx.Value(addLivenessKey{}) != nil { + return ctx.Value(addLivenessKey{}).(*http.HandlerFunc) + } + defaultHandle := newDefaultProbesHandle(ctx) + return &defaultHandle +} diff --git a/vendor/knative.dev/pkg/injection/sharedmain/main.go b/vendor/knative.dev/pkg/injection/sharedmain/main.go index 3b8ab3f40bdf..8311f1d91e9f 100644 --- a/vendor/knative.dev/pkg/injection/sharedmain/main.go +++ b/vendor/knative.dev/pkg/injection/sharedmain/main.go @@ -292,6 +292,13 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto return controller.StartAll(ctx, controllers...) }) + // Setup default health checks to catch issues with cache sync etc. + if !healthProbesDisabled(ctx) { + eg.Go(func() error { + return injection.ServeHealthProbes(ctx, injection.HealthCheckDefaultPort) + }) + } + // This will block until either a signal arrives or one of the grouped functions // returns an error. <-egCtx.Done() @@ -303,6 +310,17 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto } } +type healthProbesDisabledKey struct{} + +// WithHealthProbesDisabled signals to MainWithContext that it should disable default probes (readiness and liveness). +func WithHealthProbesDisabled(ctx context.Context) context.Context { + return context.WithValue(ctx, healthProbesDisabledKey{}, struct{}{}) +} + +func healthProbesDisabled(ctx context.Context) bool { + return ctx.Value(healthProbesDisabledKey{}) != nil +} + func flush(logger *zap.SugaredLogger) { logger.Sync() metrics.FlushExporter()