From 9069dea2c49057ab3011f028106ad9ce615814b1 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Wed, 8 Mar 2023 01:33:22 -0600 Subject: [PATCH] Add Circuit Breaker Module support --- api/v1alpha1/httpsedge_types.go | 3 ++ api/v1alpha1/ngrok_common.go | 24 ++++++++++ api/v1alpha1/ngrokmoduleset_types.go | 15 ++++--- api/v1alpha1/zz_generated.deepcopy.go | 26 +++++++++++ .../ingress.k8s.ngrok.com_httpsedges.yaml | 36 +++++++++++++++ ...ingress.k8s.ngrok.com_ngrokmodulesets.yaml | 44 ++++++++++++++++--- internal/controllers/httpsedge_controller.go | 37 ++++++++++++++++ internal/store/driver.go | 1 + 8 files changed, 176 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/httpsedge_types.go b/api/v1alpha1/httpsedge_types.go index 50cb9c1b..6d40abd5 100644 --- a/api/v1alpha1/httpsedge_types.go +++ b/api/v1alpha1/httpsedge_types.go @@ -51,6 +51,9 @@ type HTTPSEdgeRouteSpec struct { // +kubebuilder:validation:Required Backend TunnelGroupBackend `json:"backend,omitempty"` + // CircuitBreaker is a circuit breaker configuration to apply to this route + CircuitBreaker *EndpointCircuitBreaker `json:"circuitBreaker,omitempty"` + // Compression is whether or not to enable compression for this route Compression *EndpointCompression `json:"compression,omitempty"` diff --git a/api/v1alpha1/ngrok_common.go b/api/v1alpha1/ngrok_common.go index 9b23f711..4e573759 100644 --- a/api/v1alpha1/ngrok_common.go +++ b/api/v1alpha1/ngrok_common.go @@ -1,5 +1,7 @@ package v1alpha1 +import "k8s.io/apimachinery/pkg/api/resource" + // common ngrok API/Dashboard fields type ngrokAPICommon struct { // Description is a human-readable description of the object in the ngrok API/Dashboard @@ -71,3 +73,25 @@ type EndpointWebhookVerification struct { // requests from the given provider. All providers except AWS SNS require a secret SecretRef *SecretKeyRef `json:"secret,omitempty"` } + +type EndpointCircuitBreaker struct { + // Integer number of seconds after which the circuit is tripped to wait before + // re-evaluating upstream health + TrippedDuration uint32 `json:"trippedDuration,omitempty"` + + // Integer number of seconds in the statistical rolling window that metrics are + // retained for. + RollingWindow uint32 `json:"rollingWindow,omitempty"` + + // Integer number of buckets into which metrics are retained. Max 128. + //+kubebuilder:validation:Minimum=1 + //+kubebuilder:validation:Maximum=128 + NumBuckets uint32 `json:"numBuckets,omitempty"` + + // Integer number of requests in a rolling window that will trip the circuit. + // Helpful if traffic volume is low. + VolumeThreshold uint32 `json:"volumeThreshold,omitempty"` + + // Error threshold percentage should be between 0 - 1.0, not 0-100.0 + ErrorThresholdPercentage resource.Quantity `json:"errorThresholdPercentage,omitempty"` +} diff --git a/api/v1alpha1/ngrokmoduleset_types.go b/api/v1alpha1/ngrokmoduleset_types.go index b62bf9de..fd4dec1a 100644 --- a/api/v1alpha1/ngrokmoduleset_types.go +++ b/api/v1alpha1/ngrokmoduleset_types.go @@ -29,15 +29,17 @@ import ( ) type NgrokModuleSetModules struct { - // Compression configuration for this module + // CircuitBreaker configuration for this module set + CircuitBreaker *EndpointCircuitBreaker `json:"circuitBreaker,omitempty"` + // Compression configuration for this module set Compression *EndpointCompression `json:"compression,omitempty"` - // Header configuration for this module + // Header configuration for this module set Headers *EndpointHeaders `json:"headers,omitempty"` - // IPRestriction configuration for this module + // IPRestriction configuration for this module set IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` - // TLSTermination configuration for this module + // TLSTermination configuration for this module set TLSTermination *EndpointTLSTerminationAtEdge `json:"tlsTermination,omitempty"` - // WebhookVerification configuration for this module + // WebhookVerification configuration for this module set WebhookVerification *EndpointWebhookVerification `json:"webhookVerification,omitempty"` } @@ -60,6 +62,9 @@ func (ms *NgrokModuleSet) Merge(o *NgrokModuleSet) { msmod := &ms.Modules omod := o.Modules + if omod.CircuitBreaker != nil { + msmod.CircuitBreaker = omod.CircuitBreaker + } if omod.Compression != nil { msmod.Compression = omod.Compression } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 94c4bb09..bf2c6c45 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -128,6 +128,22 @@ func (in *DomainStatus) DeepCopy() *DomainStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointCircuitBreaker) DeepCopyInto(out *EndpointCircuitBreaker) { + *out = *in + out.ErrorThresholdPercentage = in.ErrorThresholdPercentage.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointCircuitBreaker. +func (in *EndpointCircuitBreaker) DeepCopy() *EndpointCircuitBreaker { + if in == nil { + return nil + } + out := new(EndpointCircuitBreaker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointCompression) DeepCopyInto(out *EndpointCompression) { *out = *in @@ -341,6 +357,11 @@ func (in *HTTPSEdgeRouteSpec) DeepCopyInto(out *HTTPSEdgeRouteSpec) { *out = *in out.ngrokAPICommon = in.ngrokAPICommon in.Backend.DeepCopyInto(&out.Backend) + if in.CircuitBreaker != nil { + in, out := &in.CircuitBreaker, &out.CircuitBreaker + *out = new(EndpointCircuitBreaker) + (*in).DeepCopyInto(*out) + } if in.Compression != nil { in, out := &in.Compression, &out.Compression *out = new(EndpointCompression) @@ -634,6 +655,11 @@ func (in *NgrokModuleSetList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NgrokModuleSetModules) DeepCopyInto(out *NgrokModuleSetModules) { *out = *in + if in.CircuitBreaker != nil { + in, out := &in.CircuitBreaker, &out.CircuitBreaker + *out = new(EndpointCircuitBreaker) + (*in).DeepCopyInto(*out) + } if in.Compression != nil { in, out := &in.Compression, &out.Compression *out = new(EndpointCompression) diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml index f25d4c23..e8a47cb5 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml @@ -74,6 +74,42 @@ spec: with the object in the ngrok API/Dashboard type: string type: object + circuitBreaker: + description: CircuitBreaker is a circuit breaker configuration + to apply to this route + properties: + errorThresholdPercentage: + anyOf: + - type: integer + - type: string + description: Error threshold percentage should be between + 0 - 1.0, not 0-100.0 + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + numBuckets: + description: Integer number of buckets into which metrics + are retained. Max 128. + format: int32 + maximum: 128 + minimum: 1 + type: integer + rollingWindow: + description: Integer number of seconds in the statistical + rolling window that metrics are retained for. + format: int32 + type: integer + trippedDuration: + description: Integer number of seconds after which the circuit + is tripped to wait before re-evaluating upstream health + format: int32 + type: integer + volumeThreshold: + description: Integer number of requests in a rolling window + that will trip the circuit. Helpful if traffic volume + is low. + format: int32 + type: integer + type: object compression: description: Compression is whether or not to enable compression for this route diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml index 94811a25..d5eb021d 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml @@ -34,8 +34,42 @@ spec: type: object modules: properties: + circuitBreaker: + description: CircuitBreaker configuration for this module set + properties: + errorThresholdPercentage: + anyOf: + - type: integer + - type: string + description: Error threshold percentage should be between 0 - + 1.0, not 0-100.0 + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + numBuckets: + description: Integer number of buckets into which metrics are + retained. Max 128. + format: int32 + maximum: 128 + minimum: 1 + type: integer + rollingWindow: + description: Integer number of seconds in the statistical rolling + window that metrics are retained for. + format: int32 + type: integer + trippedDuration: + description: Integer number of seconds after which the circuit + is tripped to wait before re-evaluating upstream health + format: int32 + type: integer + volumeThreshold: + description: Integer number of requests in a rolling window that + will trip the circuit. Helpful if traffic volume is low. + format: int32 + type: integer + type: object compression: - description: Compression configuration for this module + description: Compression configuration for this module set properties: enabled: description: Enabled is whether or not to enable compression for @@ -43,7 +77,7 @@ spec: type: boolean type: object headers: - description: Header configuration for this module + description: Header configuration for this module set properties: request: description: Request headers are the request headers module configuration @@ -84,7 +118,7 @@ spec: type: object type: object ipRestriction: - description: IPRestriction configuration for this module + description: IPRestriction configuration for this module set properties: policies: items: @@ -92,7 +126,7 @@ spec: type: array type: object tlsTermination: - description: TLSTermination configuration for this module + description: TLSTermination configuration for this module set properties: minVersion: description: MinVersion is the minimum TLS version to allow for @@ -100,7 +134,7 @@ spec: type: string type: object webhookVerification: - description: WebhookVerification configuration for this module + description: WebhookVerification configuration for this module set properties: provider: description: a string indicating which webhook provider will be diff --git a/internal/controllers/httpsedge_controller.go b/internal/controllers/httpsedge_controller.go index 5c21fd9d..120a0d78 100644 --- a/internal/controllers/httpsedge_controller.go +++ b/internal/controllers/httpsedge_controller.go @@ -440,6 +440,7 @@ type edgeRouteModuleUpdater struct { func (u *edgeRouteModuleUpdater) updateModulesForRoute(ctx context.Context, route *ngrok.HTTPSEdgeRoute, routeSpec *ingressv1alpha1.HTTPSEdgeRouteSpec) error { funcs := []func(context.Context, *ngrok.HTTPSEdgeRoute, *ingressv1alpha1.HTTPSEdgeRouteSpec) error{ + u.setEdgeRouteCircuitBreaker, u.setEdgeRouteCompression, u.setEdgeRouteIPRestriction, u.setEdgeRouteRequestHeaders, @@ -462,6 +463,42 @@ func (u *edgeRouteModuleUpdater) edgeRouteItem(route *ngrok.HTTPSEdgeRoute) *ngr } } +func (u *edgeRouteModuleUpdater) setEdgeRouteCircuitBreaker(ctx context.Context, route *ngrok.HTTPSEdgeRoute, routeSpec *ingressv1alpha1.HTTPSEdgeRouteSpec) error { + circuitBreaker := routeSpec.CircuitBreaker + + client := u.clientset.CircuitBreaker() + + // Early return if nothing to be done + if circuitBreaker == nil { + if route.CircuitBreaker == nil { + u.log.Info("CircuitBreaker matches desired state, skipping update") + return nil + } + + return client.Delete(ctx, u.edgeRouteItem(route)) + } + + module := ngrok.EndpointCircuitBreaker{ + TrippedDuration: circuitBreaker.TrippedDuration, + RollingWindow: circuitBreaker.RollingWindow, + NumBuckets: circuitBreaker.NumBuckets, + VolumeThreshold: circuitBreaker.VolumeThreshold, + ErrorThresholdPercentage: circuitBreaker.ErrorThresholdPercentage.AsApproximateFloat64(), + } + + if reflect.DeepEqual(module, route.CircuitBreaker) { + u.log.Info("CircuitBreaker matches desired state, skipping update") + return nil + } + + _, err := client.Replace(ctx, &ngrok.EdgeRouteCircuitBreakerReplace{ + EdgeID: route.EdgeID, + ID: route.ID, + Module: module, + }) + return err +} + func (u *edgeRouteModuleUpdater) setEdgeRouteCompression(ctx context.Context, route *ngrok.HTTPSEdgeRoute, routeSpec *ingressv1alpha1.HTTPSEdgeRouteSpec) error { compression := routeSpec.Compression diff --git a/internal/store/driver.go b/internal/store/driver.go index bf7e381a..a7d3a1ff 100644 --- a/internal/store/driver.go +++ b/internal/store/driver.go @@ -403,6 +403,7 @@ func (d *Driver) calculateHTTPSEdges() []ingressv1alpha1.HTTPSEdge { Backend: ingressv1alpha1.TunnelGroupBackend{ Labels: backendToLabelMap(httpIngressPath.Backend, ingress.Namespace), }, + CircuitBreaker: modSet.Modules.CircuitBreaker, Compression: modSet.Modules.Compression, IPRestriction: modSet.Modules.IPRestriction, Headers: modSet.Modules.Headers,