From 37f47a47c0b2896144dfb1c1b532acdbf734371a Mon Sep 17 00:00:00 2001 From: Alex Bezek Date: Thu, 9 Feb 2023 18:14:22 -0800 Subject: [PATCH 1/2] ingress controller uses the driver to sync all changes --- internal/controllers/ingress_controller.go | 365 ++---------------- .../controllers/ingress_controller_test.go | 304 --------------- internal/controllers/utils.go | 1 - 3 files changed, 22 insertions(+), 648 deletions(-) diff --git a/internal/controllers/ingress_controller.go b/internal/controllers/ingress_controller.go index ac70a578..22e9cca3 100644 --- a/internal/controllers/ingress_controller.go +++ b/internal/controllers/ingress_controller.go @@ -2,27 +2,17 @@ package controllers import ( "context" - "fmt" - "reflect" - "strconv" - "strings" - "time" "github.com/go-logr/logr" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" "github.com/ngrok/kubernetes-ingress-controller/internal/annotations" internalerrors "github.com/ngrok/kubernetes-ingress-controller/internal/errors" "github.com/ngrok/kubernetes-ingress-controller/internal/store" - v1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) @@ -83,12 +73,11 @@ func (irec *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - // TODO: Uncomment when we want the sync call to manage api resources - // err = irec.Driver.Sync(ctx, irec.Client) - // if err != nil { - // log.Error(err, "Failed to sync after removing ingress from store") - // return ctrl.Result{}, err - // } + err = irec.Driver.Sync(ctx, irec.Client) + if err != nil { + log.Error(err, "Failed to sync after removing ingress from store") + return ctrl.Result{}, err + } return ctrl.Result{}, nil } @@ -122,16 +111,10 @@ func (irec *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } } else { - // The object is being deleted if hasFinalizer(ingress) { - log.Info("Deleting ingress") - - if err = irec.DeleteDependents(ctx, ingress); err != nil { - return ctrl.Result{}, err - } - - if err := removeAndSyncFinalizer(ctx, irec.Client, ingress); err != nil { - log.Error(err, "Failed to remove finalizer") + log.Info("Deleting ingress from store") + if err := irec.delete(ctx, ingress); err != nil { + log.Error(err, "Failed to delete ingress") return ctrl.Result{}, err } } @@ -143,334 +126,30 @@ func (irec *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) return irec.reconcileAll(ctx, ingress) } -func (irec *IngressReconciler) DeleteDependents(ctx context.Context, ingress *netv1.Ingress) error { - // TODO: Currently this controller "owns" the HTTPSEdge and Tunnel objects so deleting an ingress - // will delete the HTTPSEdge and Tunnel objects. Once multiple ingress objects combine to form 1 edge - // this logic will need to be smarter - return nil +// Delete is called when the ingress object is being deleted +func (irec *IngressReconciler) delete(ctx context.Context, ingress *netv1.Ingress) error { + if err := removeAndSyncFinalizer(ctx, irec.Client, ingress); err != nil { + irec.Log.Error(err, "Failed to remove finalizer") + return err + } + // Remove the ingress object from the store + return irec.Driver.Delete(ingress) } func (irec *IngressReconciler) reconcileAll(ctx context.Context, ingress *netv1.Ingress) (reconcile.Result, error) { - err := irec.reconcileDomains(ctx, ingress) - if err != nil { - if internalerrors.IsNotAllDomainsReadyYet(err) { - irec.Recorder.Event(ingress, v1.EventTypeNormal, "Provisioning domains", "Waiting for domains to be ready") - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - irec.Recorder.Event(ingress, v1.EventTypeWarning, "Failed to reconcile reserved domains", err.Error()) - return ctrl.Result{}, err - } - - err = irec.reconcileTunnels(ctx, ingress) + log := irec.Log + // First Update the store + err := irec.Driver.Update(ingress) if err != nil { - irec.Recorder.Event(ingress, v1.EventTypeWarning, "Failed to reconcile tunnels", err.Error()) + log.Error(err, "Failed to add ingress to store") return ctrl.Result{}, err } - err = irec.reconcileEdges(ctx, ingress) + err = irec.Driver.Sync(ctx, irec.Client) if err != nil { - irec.Recorder.Event(ingress, v1.EventTypeWarning, "Failed to reconcile edges", err.Error()) + log.Error(err, "Failed to sync ingress to store") return ctrl.Result{}, err } return ctrl.Result{}, nil } - -// Converts a k8s Ingress Rule to and ngrok Route configuration. -func (irec *IngressReconciler) routesPlanner(ctx context.Context, ingress *netv1.Ingress, parsedRouteModules *annotations.RouteModules) ([]ingressv1alpha1.HTTPSEdgeRouteSpec, error) { - namespace := ingress.Namespace - rule := ingress.Spec.Rules[0] - - var matchType string - var ngrokRoutes []ingressv1alpha1.HTTPSEdgeRouteSpec - - for _, httpIngressPath := range rule.HTTP.Paths { - switch *httpIngressPath.PathType { - case netv1.PathTypePrefix: - matchType = "path_prefix" - case netv1.PathTypeExact: - matchType = "exact_path" - case netv1.PathTypeImplementationSpecific: - matchType = "path_prefix" // Path Prefix seems like a sane default for most cases - default: - return nil, fmt.Errorf("unsupported path type: %v", httpIngressPath.PathType) - } - - route := ingressv1alpha1.HTTPSEdgeRouteSpec{ - Match: httpIngressPath.Path, - MatchType: matchType, - Backend: ingressv1alpha1.TunnelGroupBackend{ - Labels: backendToLabelMap(httpIngressPath.Backend, namespace), - }, - Compression: parsedRouteModules.Compression, - Headers: parsedRouteModules.Headers, - IPRestriction: parsedRouteModules.IPRestriction, - WebhookVerification: parsedRouteModules.WebhookVerification, - } - - ngrokRoutes = append(ngrokRoutes, route) - } - - return ngrokRoutes, nil -} - -// Converts a k8s ingress object into an ngrok Edge with all its configurations and sub-resources -// TODO: Support multiple Rules per Ingress -func (irec *IngressReconciler) ingressToEdge(ctx context.Context, ingress *netv1.Ingress) (*ingressv1alpha1.HTTPSEdge, error) { - if ingress == nil { - return nil, nil - } - - // An ingress with no rules sends all traffic to a single default backend(.spec.defaultBackend) - // and must be specified. TODO: Implement this. - if len(ingress.Spec.Rules) == 0 { - return nil, nil - } - - parsedRouteModules := irec.AnnotationsExtractor.Extract(ingress) - - ngrokRoutes, err := irec.routesPlanner(ctx, ingress, parsedRouteModules) - if err != nil { - return nil, err - } - - return &ingressv1alpha1.HTTPSEdge{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ingress.Namespace, - Name: ingress.Name, - }, - Spec: ingressv1alpha1.HTTPSEdgeSpec{ - Hostports: []string{ingress.Spec.Rules[0].Host + ":443"}, - Routes: ngrokRoutes, - TLSTermination: parsedRouteModules.TLSTermination, - }, - }, nil -} - -func (irec *IngressReconciler) reconcileEdges(ctx context.Context, ingress *netv1.Ingress) error { - edge, err := irec.ingressToEdge(ctx, ingress) - if err != nil { - return err - } - - if err := controllerutil.SetControllerReference(ingress, edge, irec.Scheme); err != nil { - return err - } - - found := &ingressv1alpha1.HTTPSEdge{} - err = irec.Client.Get(ctx, types.NamespacedName{Name: edge.Name, Namespace: edge.Namespace}, found) - if err != nil && errors.IsNotFound(err) { - err = irec.Create(ctx, edge) - if err != nil { - return err - } - } else if err != nil { - return err - } - - if !reflect.DeepEqual(edge.Spec, found.Spec) { - found.Spec = edge.Spec - err = irec.Update(ctx, found) - if err != nil { - return err - } - } - - return nil -} - -func (irec *IngressReconciler) reconcileDomains(ctx context.Context, ingress *netv1.Ingress) error { - reservedDomains, err := irec.ingressToDomains(ctx, ingress) - if err != nil { - return err - } - - loadBalancerIngressStatuses := []netv1.IngressLoadBalancerIngress{} - hasDomainsWithoutStatus := false - - for _, reservedDomain := range reservedDomains { - found := &ingressv1alpha1.Domain{} - err := irec.Client.Get(ctx, types.NamespacedName{Name: reservedDomain.Name, Namespace: reservedDomain.Namespace}, found) - if err != nil { - if !errors.IsNotFound(err) { - return err - } - - irec.Log.Info("Creating domain", "namespace", reservedDomain.Namespace, "name", reservedDomain.Name) - err = irec.Create(ctx, &reservedDomain) - if err != nil { - return err - } - } - - if !reflect.DeepEqual(reservedDomain.Spec, found.Spec) { - found.Spec = reservedDomain.Spec - err = irec.Update(ctx, found) - if err != nil { - return err - } - } - - var loadBalancerHostname string - if found.Status.CNAMETarget != nil { - loadBalancerHostname = *found.Status.CNAMETarget - } else if found.Status.Domain != "" { - loadBalancerHostname = found.Status.Domain - } else { - hasDomainsWithoutStatus = true - } - - loadBalancerIngressStatuses = append(loadBalancerIngressStatuses, netv1.IngressLoadBalancerIngress{ - Hostname: loadBalancerHostname, - }) - } - - if hasDomainsWithoutStatus { - return internalerrors.NewNotAllDomainsReadyYetError() - } - - irec.Log.Info("Updating Ingress status with load balancer ingress statuses") - ingress.Status.LoadBalancer.Ingress = loadBalancerIngressStatuses - return irec.Status().Update(ctx, ingress) -} - -func (irec *IngressReconciler) reconcileTunnels(ctx context.Context, ingress *netv1.Ingress) error { - tunnels := ingressToTunnels(ingress) - - for _, tunnel := range tunnels { - if err := controllerutil.SetControllerReference(ingress, &tunnel, irec.Scheme); err != nil { - return err - } - - found := &ingressv1alpha1.Tunnel{} - err := irec.Client.Get(ctx, types.NamespacedName{Name: tunnel.Name, Namespace: tunnel.Namespace}, found) - if err != nil && errors.IsNotFound(err) { - err = irec.Create(ctx, &tunnel) - if err != nil { - return err - } - } else if err != nil { - return err - } - - if !reflect.DeepEqual(tunnel.Spec, found.Spec) { - found.Spec = tunnel.Spec - err = irec.Update(ctx, found) - if err != nil { - return err - } - } - } - - return nil -} - -func (irec *IngressReconciler) ingressToDomains(ctx context.Context, ingress *netv1.Ingress) ([]ingressv1alpha1.Domain, error) { - reservedDomains := make([]ingressv1alpha1.Domain, 0) - - if ingress == nil { - return reservedDomains, nil - } - - for _, rule := range ingress.Spec.Rules { - if rule.Host == "" { - continue - } - reservedDomains = append(reservedDomains, ingressv1alpha1.Domain{ - ObjectMeta: metav1.ObjectMeta{ - Name: strings.Replace(rule.Host, ".", "-", -1), - Namespace: ingress.Namespace, - }, - Spec: ingressv1alpha1.DomainSpec{ - Domain: rule.Host, - }, - }) - } - - return reservedDomains, nil -} - -// listIngressesForDomains returns a list of ingresses that reference the given domain. -func (irec *IngressReconciler) listIngressesForDomain(obj client.Object) []reconcile.Request { - irec.Log.Info("Listing ingresses for domain to determine if they need to be reconciled") - domain, ok := obj.(*ingressv1alpha1.Domain) - if !ok { - irec.Log.Error(nil, "failed to convert object to domain", "object", obj) - return []reconcile.Request{} - } - - ingresses := &netv1.IngressList{} - if err := irec.Client.List(context.Background(), ingresses); err != nil { - irec.Log.Error(err, "failed to list ingresses for domain", "domain", domain.Spec.Domain) - return []reconcile.Request{} - } - - recs := []reconcile.Request{} - - for _, ingress := range ingresses.Items { - for _, rule := range ingress.Spec.Rules { - if rule.Host == domain.Status.Domain { - recs = append(recs, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: ingress.GetName(), - Namespace: ingress.GetNamespace(), - }, - }) - break - } - } - } - - irec.Log.Info("Domain change triggered ingress reconciliation", "count", len(recs), "domain", domain.Spec.Domain) - return recs -} - -func ingressToTunnels(ingress *netv1.Ingress) []ingressv1alpha1.Tunnel { - tunnels := make([]ingressv1alpha1.Tunnel, 0) - - if ingress == nil || len(ingress.Spec.Rules) == 0 { - return tunnels - } - - // Tunnels should be unique on a service and port basis so if they are referenced more than once, we - // only create one tunnel per service and port. - tunnelMap := make(map[string]ingressv1alpha1.Tunnel) - for _, rule := range ingress.Spec.Rules { - if rule.Host == "" { - continue - } - - for _, path := range rule.HTTP.Paths { - serviceName := path.Backend.Service.Name - servicePort := path.Backend.Service.Port.Number - tunnelAddr := fmt.Sprintf("%s.%s.%s:%d", serviceName, ingress.Namespace, clusterDomain, servicePort) - tunnelName := fmt.Sprintf("%s-%d", serviceName, servicePort) - - tunnelMap[tunnelName] = ingressv1alpha1.Tunnel{ - ObjectMeta: metav1.ObjectMeta{ - Name: tunnelName, - Namespace: ingress.Namespace, - }, - Spec: ingressv1alpha1.TunnelSpec{ - ForwardsTo: tunnelAddr, - Labels: backendToLabelMap(path.Backend, ingress.Namespace), - }, - } - } - } - - for _, tunnel := range tunnelMap { - tunnels = append(tunnels, tunnel) - } - - return tunnels -} - -// Generates a labels map for matching ngrok Routes to Agent Tunnels -func backendToLabelMap(backend netv1.IngressBackend, namespace string) map[string]string { - return map[string]string{ - "k8s.ngrok.com/namespace": namespace, - "k8s.ngrok.com/service": backend.Service.Name, - "k8s.ngrok.com/port": strconv.Itoa(int(backend.Service.Port.Number)), - } -} diff --git a/internal/controllers/ingress_controller_test.go b/internal/controllers/ingress_controller_test.go index d03307b1..ac309236 100644 --- a/internal/controllers/ingress_controller_test.go +++ b/internal/controllers/ingress_controller_test.go @@ -1,15 +1,7 @@ package controllers import ( - "context" - "fmt" - "testing" - - ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" - "github.com/ngrok/kubernetes-ingress-controller/internal/annotations" - "github.com/stretchr/testify/assert" netv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func makeTestBackend(serviceName string, servicePort int32) netv1.IngressBackend { @@ -22,299 +14,3 @@ func makeTestBackend(serviceName string, servicePort int32) netv1.IngressBackend }, } } - -func makeTestTunnel(namespace, serviceName string, servicePort int) ingressv1alpha1.Tunnel { - return ingressv1alpha1.Tunnel{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", serviceName, servicePort), - Namespace: namespace, - }, - Spec: ingressv1alpha1.TunnelSpec{ - ForwardsTo: fmt.Sprintf("%s.%s.%s:%d", serviceName, namespace, clusterDomain, servicePort), - Labels: map[string]string{ - "k8s.ngrok.com/namespace": namespace, - "k8s.ngrok.com/service": serviceName, - "k8s.ngrok.com/port": fmt.Sprintf("%d", servicePort), - }, - }, - } -} - -func TestIngressReconcilerIngressToEdge(t *testing.T) { - prefix := netv1.PathTypePrefix - testCases := []struct { - testName string - ingress *netv1.Ingress - edge *ingressv1alpha1.HTTPSEdge - err error - }{ - { - testName: "Returns a nil edge when ingress is nil", - ingress: nil, - edge: nil, - }, - { - testName: "Returns a nil edge when ingress has no rules", - ingress: &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - }, - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{}, - }, - }, - edge: nil, - }, - { - testName: "", - ingress: &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - Annotations: map[string]string{ - "k8s.ngrok.com/https-compression": "false", - }, - }, - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{ - { - Host: "my-test-tunnel.ngrok.io", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - PathType: &prefix, - Backend: makeTestBackend("test-service", 8080), - }, - }, - }, - }, - }, - }, - }, - }, - edge: &ingressv1alpha1.HTTPSEdge{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - }, - Spec: ingressv1alpha1.HTTPSEdgeSpec{ - Hostports: []string{"my-test-tunnel.ngrok.io:443"}, - Routes: []ingressv1alpha1.HTTPSEdgeRouteSpec{ - { - Match: "/", - MatchType: "path_prefix", - Backend: ingressv1alpha1.TunnelGroupBackend{ - Labels: map[string]string{ - "k8s.ngrok.com/namespace": "test-namespace", - "k8s.ngrok.com/service": "test-service", - "k8s.ngrok.com/port": "8080", - }, - }, - Compression: &ingressv1alpha1.EndpointCompression{ - Enabled: false, - }, - }, - }, - }, - }, - }, - { - testName: "", - ingress: &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - Annotations: map[string]string{ - "k8s.ngrok.com/https-compression": "true", - "k8s.ngrok.com/ip-policies": "policy-1,policy-2", - "k8s.ngrok.com/tls-min-version": "1.3", - }, - }, - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{ - { - Host: "my-test-tunnel.ngrok.io", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - PathType: &prefix, - Backend: makeTestBackend("test-service", 8080), - }, - }, - }, - }, - }, - }, - }, - }, - edge: &ingressv1alpha1.HTTPSEdge{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - }, - Spec: ingressv1alpha1.HTTPSEdgeSpec{ - Hostports: []string{"my-test-tunnel.ngrok.io:443"}, - TLSTermination: &ingressv1alpha1.EndpointTLSTerminationAtEdge{ - MinVersion: "1.3", - }, - Routes: []ingressv1alpha1.HTTPSEdgeRouteSpec{ - { - Match: "/", - MatchType: "path_prefix", - Backend: ingressv1alpha1.TunnelGroupBackend{ - Labels: map[string]string{ - "k8s.ngrok.com/namespace": "test-namespace", - "k8s.ngrok.com/service": "test-service", - "k8s.ngrok.com/port": "8080", - }, - }, - Compression: &ingressv1alpha1.EndpointCompression{ - Enabled: true, - }, - IPRestriction: &ingressv1alpha1.EndpointIPPolicy{ - IPPolicies: []string{"policy-1", "policy-2"}, - }, - }, - }, - }, - }, - }, - } - - for _, testCase := range testCases { - irec := IngressReconciler{ - AnnotationsExtractor: annotations.NewAnnotationsExtractor(), - } - edge, err := irec.ingressToEdge(context.Background(), testCase.ingress) - - if testCase.err != nil { - assert.ErrorIs(t, err, testCase.err) - continue - } - assert.NoError(t, err) - - if testCase.edge == nil { - assert.Nil(t, edge) - continue - } - - assert.Equal(t, testCase.edge, edge, "Edge does not match expected value") - } -} - -func TestIngressToTunnels(t *testing.T) { - - testCases := []struct { - testName string - ingress *netv1.Ingress - tunnels []ingressv1alpha1.Tunnel - }{ - { - testName: "Returns empty list when ingress is nil", - ingress: nil, - tunnels: []ingressv1alpha1.Tunnel{}, - }, - { - testName: "Returns empty list when ingress has no rules", - ingress: &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - }, - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{}, - }, - }, - tunnels: []ingressv1alpha1.Tunnel{}, - }, - { - testName: "Converts an ingress to a tunnel", - ingress: &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - }, - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{ - { - Host: "my-test-tunnel.ngrok.io", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - Backend: makeTestBackend("test-service", 8080), - }, - }, - }, - }, - }, - }, - }, - }, - tunnels: []ingressv1alpha1.Tunnel{ - makeTestTunnel("test-namespace", "test-service", 8080), - }, - }, - { - testName: "Correctly converts an ingress with multiple paths that point to the same service", - ingress: &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "test-namespace", - }, - Spec: netv1.IngressSpec{ - Rules: []netv1.IngressRule{ - { - Host: "my-test-tunnel.ngrok.io", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - Backend: makeTestBackend("test-service", 8080), - }, - { - Path: "/api", - Backend: makeTestBackend("test-api", 80), - }, - }, - }, - }, - }, - { - Host: "my-other-test-tunnel.ngrok.io", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - Backend: makeTestBackend("test-service", 8080), - }, - { - Path: "/api", - Backend: makeTestBackend("test-api", 80), - }, - }, - }, - }, - }, - }, - }, - }, - tunnels: []ingressv1alpha1.Tunnel{ - makeTestTunnel("test-namespace", "test-service", 8080), - makeTestTunnel("test-namespace", "test-api", 80), - }, - }, - } - - for _, test := range testCases { - tunnels := ingressToTunnels(test.ingress) - assert.ElementsMatch(t, tunnels, test.tunnels) - } -} diff --git a/internal/controllers/utils.go b/internal/controllers/utils.go index 05997155..c9ec738d 100644 --- a/internal/controllers/utils.go +++ b/internal/controllers/utils.go @@ -13,7 +13,6 @@ import ( const ( finalizerName = "k8s.ngrok.com/finalizer" // TODO: We can technically figure this out by looking at things like our resolv.conf or we can just take this as a helm option - clusterDomain = "svc.cluster.local" ) func isDelete(meta metav1.ObjectMeta) bool { From 1c51cfa31e2df302b74a59758e17dd5e750aeef3 Mon Sep 17 00:00:00 2001 From: Alex Bezek Date: Thu, 9 Feb 2023 21:09:59 -0800 Subject: [PATCH 2/2] add docs --- docs/.gitkeep | 0 docs/ingress-to-edge-relationship.md | 206 +++++++++++++++++++++++++++ scripts/e2e.sh | 2 +- 3 files changed, 207 insertions(+), 1 deletion(-) delete mode 100644 docs/.gitkeep create mode 100644 docs/ingress-to-edge-relationship.md diff --git a/docs/.gitkeep b/docs/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/ingress-to-edge-relationship.md b/docs/ingress-to-edge-relationship.md new file mode 100644 index 00000000..5425ec71 --- /dev/null +++ b/docs/ingress-to-edge-relationship.md @@ -0,0 +1,206 @@ +# Ingress to Edge Relationship + +This ingress controller aims to take the [ingress spec](https://kubernetes.io/docs/concepts/services-networking/ingress/#the-ingress-resource) and implement each specified concept into ngrok edges. The concept of an ngrok Edge is documented more [here](https://ngrok.com/docs/cloud-edge/). This document aims to explain how multiple ingress objects with rules and hosts that overlap combine to form edges in the ngrok API. + +Overall +- a host correlates directly to an edge +- rules spread across ingress objects with matching hosts get merged into the same edge +- annotations on an ingress apply only to the routes in the rules in that ingress if possible + +## Types of Ingress + +[This Kubernetes "Types of Ingress" Doc](https://kubernetes.io/docs/concepts/services-networking/ingress/#types-of-ingress) breaks down a few common ingress examples. We'll use these examples to explain how the ingress controller will handle them and what the end result edge configurations are. + +### Single Service Ingress + +> There are existing Kubernetes concepts that allow you to expose a single Service (see alternatives). You can also do this with an Ingress by specifying a default backend with no rules. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-ingress +spec: + defaultBackend: + service: + name: test + port: + number: 80 +``` + +While not implemented yet, when completed, this style of ingress should allow exposing a service across all edges configured by the controller. Without any other ingress objects with hosts configured though, this ingress would do nothing as there is no edge and host to attach to. + +### Simple Fanout + +> A fanout configuration routes traffic from a single IP address to more than one Service, based on the HTTP URI being requested. An Ingress allows you to keep the number of load balancers down to a minimum. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: simple-fanout-example +spec: + rules: + - host: foo.bar.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + service: + name: service1 + port: + number: 4200 + - path: /bar + pathType: Prefix + backend: + service: + name: service2 + port: + number: 8080 +``` + +This configuration would produce a single edge with two routes. +- edge: `foo.bar.com` + - route: `/foo` -> `service1:4200` + - route: `/bar` -> `service2:8080` + +If another ingress object like this was created + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: simple-fanout-example-2 +spec: + rules: + - host: foo.bar.com + http: + paths: + - path: /baz + pathType: Prefix + backend: + service: + name: service3 + port: + number: 8080 +``` + +The edge would be updated to have three routes. +- edge: `foo.bar.com` + - route: `/foo` -> `service1:4200` + - route: `/bar` -> `service2:8080` + - route: `/baz` -> `service3:8080` + +Ingress rules with the same host are merged into the same edge. + +### Name-based Virtual Hosting + +> Name-based virtual hosting is a common way to implement virtual hosting on a single IP address. Each host name is associated with a distinct IP address, and the DNS resolver associates a given host name with its corresponding IP address(es). The Ingress resource does not require any special support for name-based virtual hosting, but it does require that the DNS resolver be configured to return the correct IP address for a given host name. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: name-virtual-host-ingress +spec: + rules: + - host: foo.bar.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: service1 + port: + number: 80 + - host: bar.foo.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: service2 + port: + number: 80 +``` + +This configuration would produce two edges with one route each. +- edge: `foo.bar.com` + - route: `/` -> `service1:80` +- edge: `bar.foo.com` + - route: `/` -> `service2:80` + +#### No Host + +> If you create an Ingress resource without any hosts defined in the rules, then any web traffic to the IP address of your Ingress controller can be matched without a name based virtual host being required. + +While not implemented yet, this would work the same as default backends where the rule is applied to all edges. +TODO: The example has 2 rules with hosts and then a third without a host, so when a request matches neither, it does match that one. I'm not sure really if thats the same "default apply to everything" or if there is a fallback problem + +### TLS + +TLS Edges are not yet implemented but should be implemented in the future. + +## Annotations + +Annotations are created and applied at the ingress object level. However, from the section above, multiple ingresses can combine and be shared to form multiple edges. When using annotations that apply specifically to routes, the annotations on the ingress apply to all routes, but routes for multiple edges across different ingresses don't have to have the same annotations or modules. + +So while annotations are limited to being applied to the whole ingress object, and we'd like to apply different route module annotations to different routes in 1 edge, we can leverage the fact that multiple ingresses can be combined to form an edge if the rules hosts match, but each annotation only applies to those routes in the ingress object they came from. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: example-header-add-1 + annotations: + k8s.ngrok.com/response-headers-add: | + { + "X-SEND-TO-CLIENT": "Value1" + } +spec: + rules: + - host: foo.bar.com + http: + paths: + - path: /foo + pathType: Prefix + backend: + service: + name: service1 + port: + number: 4200 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: example-header-add-2 + annotations: + k8s.ngrok.com/response-headers-add: | + { + "X-SEND-TO-CLIENT": "Value2" + } +spec: + rules: + - host: foo.bar.com + http: + paths: + - path: /bar + pathType: Prefix + backend: + service: + name: service2 + port: + number: 8080 +``` + +This configuration would produce a single edge with two routes. Each route has a different http response header module + +- edge: `foo.bar.com` + - route: `/foo` -> `service1:4200` + - module: `response-headers-add` -> `{"X-SEND-TO-CLIENT": "Value1"}` + - route: `/bar` -> `service2:8080` + - module: `response-headers-add` -> `{"X-SEND-TO-CLIENT": "Value2"}` + diff --git a/scripts/e2e.sh b/scripts/e2e.sh index a46e7a78..b9d72c93 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -39,7 +39,7 @@ for example in $(ls -d examples/*) do kubectl apply -k $example || true done -sleep 30 +sleep 60 # Run tests echo "--- Running e2e tests"