Skip to content

Commit

Permalink
[release-v1.8] add probes to all deployments (knative#199)
Browse files Browse the repository at this point in the history
* add probes to all deployments (knative#176)

* add missing probes (knative#13563)

* Bump manifests

---------

Co-authored-by: Stavros Kontopoulos <[email protected]>
  • Loading branch information
nak3 and skonto authored Mar 8, 2023
1 parent a2d1813 commit 587a65c
Show file tree
Hide file tree
Showing 7 changed files with 229 additions and 0 deletions.
17 changes: 17 additions & 0 deletions config/core/deployments/controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,28 @@ spec:
seccompProfile:
type: RuntimeDefault

livenessProbe:
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5

ports:
- name: metrics
containerPort: 9090
- name: profiling
containerPort: 8008
- name: probes
containerPort: 8080

---
apiVersion: v1
Expand Down
17 changes: 17 additions & 0 deletions config/core/deployments/domainmapping-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,25 @@ spec:
seccompProfile:
type: RuntimeDefault

livenessProbe:
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5

ports:
- name: metrics
containerPort: 9090
- name: profiling
containerPort: 8008
- name: probes
containerPort: 8080
17 changes: 17 additions & 0 deletions config/hpa-autoscaling/controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,28 @@ spec:
seccompProfile:
type: RuntimeDefault

livenessProbe:
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5

ports:
- name: metrics
containerPort: 9090
- name: profiling
containerPort: 8008
- name: probes
containerPort: 8080

---
apiVersion: v1
Expand Down
34 changes: 34 additions & 0 deletions openshift/release/artifacts/serving-core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6150,11 +6150,28 @@ spec:



livenessProbe:
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5

ports:
- name: metrics
containerPort: 9090
- name: profiling
containerPort: 8008
- name: probes
containerPort: 8080

---
apiVersion: v1
Expand Down Expand Up @@ -6266,11 +6283,28 @@ spec:



livenessProbe:
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5

ports:
- name: metrics
containerPort: 9090
- name: profiling
containerPort: 8008
- name: probes
containerPort: 8080
---
# Copyright 2020 The Knative Authors
#
Expand Down
17 changes: 17 additions & 0 deletions openshift/release/artifacts/serving-hpa.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,28 @@ spec:



livenessProbe:
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
failureThreshold: 5

ports:
- name: metrics
containerPort: 9090
- name: profiling
containerPort: 8008
- name: probes
containerPort: 8080

---
apiVersion: v1
Expand Down
109 changes: 109 additions & 0 deletions vendor/knative.dev/pkg/injection/health_check.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
Copyright 2023 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package injection

import (
"context"
"errors"
"net/http"
"strconv"
"time"

"knative.dev/pkg/logging"
)

// HealthCheckDefaultPort defines the default port number for health probes
const HealthCheckDefaultPort = 8080

// ServeHealthProbes sets up liveness and readiness probes.
// If user sets no probes explicitly via the context then defaults are added.
func ServeHealthProbes(ctx context.Context, port int) error {
logger := logging.FromContext(ctx)
server := http.Server{ReadHeaderTimeout: time.Minute, Handler: muxWithHandles(ctx), Addr: ":" + strconv.Itoa(port)}
go func() {
<-ctx.Done()
_ = server.Shutdown(ctx)
}()

// start the web server on port and accept requests
logger.Infof("Probes server listening on port %s", port)
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}

func muxWithHandles(ctx context.Context) *http.ServeMux {
mux := http.NewServeMux()
readiness := getReadinessHandleOrDefault(ctx)
liveness := getLivenessHandleOrDefault(ctx)
mux.HandleFunc("/readiness", *readiness)
mux.HandleFunc("/health", *liveness)
return mux
}

func newDefaultProbesHandle(sigCtx context.Context) http.HandlerFunc {
logger := logging.FromContext(sigCtx)
return func(w http.ResponseWriter, r *http.Request) {
f := func() error {
select {
// When we get SIGTERM (sigCtx done), let readiness probes start failing.
case <-sigCtx.Done():
logger.Info("Signal context canceled")
return errors.New("received SIGTERM from kubelet")
default:
return nil
}
}
if err := f(); err != nil {
logger.Errorf("Healthcheck failed: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
} else {
w.WriteHeader(http.StatusOK)
}
}
}

type addReadinessKey struct{}

// AddReadiness signals to probe setup logic to add a user provided probe handler
func AddReadiness(ctx context.Context, handlerFunc http.HandlerFunc) context.Context {
return context.WithValue(ctx, addReadinessKey{}, &handlerFunc)
}

func getReadinessHandleOrDefault(ctx context.Context) *http.HandlerFunc {
if ctx.Value(addReadinessKey{}) != nil {
return ctx.Value(addReadinessKey{}).(*http.HandlerFunc)
}
defaultHandle := newDefaultProbesHandle(ctx)
return &defaultHandle
}

type addLivenessKey struct{}

// AddLiveness signals to probe setup logic to add a user provided probe handler
func AddLiveness(ctx context.Context, handlerFunc http.HandlerFunc) context.Context {
return context.WithValue(ctx, addLivenessKey{}, &handlerFunc)
}

func getLivenessHandleOrDefault(ctx context.Context) *http.HandlerFunc {
if ctx.Value(addLivenessKey{}) != nil {
return ctx.Value(addLivenessKey{}).(*http.HandlerFunc)
}
defaultHandle := newDefaultProbesHandle(ctx)
return &defaultHandle
}
18 changes: 18 additions & 0 deletions vendor/knative.dev/pkg/injection/sharedmain/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,13 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto
return controller.StartAll(ctx, controllers...)
})

// Setup default health checks to catch issues with cache sync etc.
if !healthProbesDisabled(ctx) {
eg.Go(func() error {
return injection.ServeHealthProbes(ctx, injection.HealthCheckDefaultPort)
})
}

// This will block until either a signal arrives or one of the grouped functions
// returns an error.
<-egCtx.Done()
Expand All @@ -303,6 +310,17 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto
}
}

type healthProbesDisabledKey struct{}

// WithHealthProbesDisabled signals to MainWithContext that it should disable default probes (readiness and liveness).
func WithHealthProbesDisabled(ctx context.Context) context.Context {
return context.WithValue(ctx, healthProbesDisabledKey{}, struct{}{})
}

func healthProbesDisabled(ctx context.Context) bool {
return ctx.Value(healthProbesDisabledKey{}) != nil
}

func flush(logger *zap.SugaredLogger) {
logger.Sync()
metrics.FlushExporter()
Expand Down

0 comments on commit 587a65c

Please sign in to comment.