Initialize route backends after module updates

internal/controllers/httpsedge_controller.go

@@ -189,12 +189,10 @@ func (r *HTTPSEdgeReconciler) reconcileRoutes(ctx context.Context, edge *ingress
 		secretResolver:   secretResolver{r.Client},
 	}
 
+	edgeRoutes := r.NgrokClientset.HTTPSEdgeRoutes()
+
 	// TODO: clean this up. This is way too much nesting
 	for i, routeSpec := range edge.Spec.Routes {
-		backend, err := tunnelGroupReconciler.findOrCreate(ctx, routeSpec.Backend)
-		if err != nil {
-			return err
-		}
 
 		if routeSpec.IPRestriction != nil {
 			if err := routeModuleUpdater.ipPolicyResolver.validateIPPolicyNames(ctx, edge.Namespace, routeSpec.IPRestriction.IPPolicies); err != nil {
@@ -209,51 +207,74 @@ func (r *HTTPSEdgeReconciler) reconcileRoutes(ctx context.Context, edge *ingress
 
 		match := r.getMatchingRouteFromEdgeStatus(edge, routeSpec)
 		var route *ngrok.HTTPSEdgeRoute
+		// Now we go ahead and create the route if it doesn't exist, or find the existing route.
+		// It's important to note here that we are intentionally ommiting the `route.Backend` for new routes, and
+		//  removing it from existing routes. The success or failure of applying a route's modules is then strongly 
+		//  linked the state of its backend. Thus, any route with a backend is considered properly configured.
+		//  See https://github.com/ngrok/kubernetes-ingress-controller/issues/208 for additional context.
 		if match == nil {
-			r.Log.Info("Creating new route", "edgeID", edge.Status.ID, "match", routeSpec.Match, "matchType", routeSpec.MatchType, "backendID", backend.ID)
-			// This is a new route, so we need to create it
+			r.Log.Info("Creating new route", "edgeID", edge.Status.ID, "match", routeSpec.Match, "matchType", routeSpec.MatchType)
 			req := &ngrok.HTTPSEdgeRouteCreate{
 				EdgeID:    edge.Status.ID,
 				Match:     routeSpec.Match,
 				MatchType: routeSpec.MatchType,
-				Backend: &ngrok.EndpointBackendMutate{
-					BackendID: backend.ID,
-				},
 			}
-			route, err = r.NgrokClientset.HTTPSEdgeRoutes().Create(ctx, req)
+			route, err = edgeRoutes.Create(ctx, req)
 		} else {
-			r.Log.Info("Updating route", "edgeID", edge.Status.ID, "match", routeSpec.Match, "matchType", routeSpec.MatchType, "backendID", backend.ID)
-			// This is an existing route, so we need to update it
+			r.Log.Info("Route found, disabling backend to prepare for module updates", "edgeID", edge.Status.ID, "match", routeSpec.Match, "matchType", routeSpec.MatchType)
 			req := &ngrok.HTTPSEdgeRouteUpdate{
 				EdgeID:    edge.Status.ID,
 				ID:        match.ID,
 				Match:     routeSpec.Match,
 				MatchType: routeSpec.MatchType,
-				Backend: &ngrok.EndpointBackendMutate{
-					BackendID: backend.ID,
-				},
 			}
-			route, err = r.NgrokClientset.HTTPSEdgeRoutes().Update(ctx, req)
+			route, err = edgeRoutes.Update(ctx, req)
 		}
 		if err != nil {
 			return err
 		}
+
 		routeStatuses[i] = ingressv1alpha1.HTTPSEdgeRouteStatus{
 			ID:        route.ID,
 			URI:       route.URI,
 			Match:     route.Match,
 			MatchType: route.MatchType,
 		}
+
+		// With the route properly staged, we now attempt to apply its module updates
+		r.Log.Info("Updating route modules", "edgeID", edge.Status.ID, "match", routeSpec.Match, "matchType", routeSpec.MatchType)
+		if err := routeModuleUpdater.updateModulesForRoute(ctx, route, &routeSpec); err != nil {
+			r.Recorder.Event(edge, v1.EventTypeWarning, "RouteModuleUpdateFailed", err.Error())
+			return err
+		}
+
+		// The route modules were successfully applied, so now we update the route with its specified backend
+		backend, err := tunnelGroupReconciler.findOrCreate(ctx, routeSpec.Backend)
+		if err != nil {
+			return err
+		}
+		r.Log.Info("Setting route backend", "edgeID", edge.Status.ID, "match", routeSpec.Match, "matchType", routeSpec.MatchType, "backendID", backend.ID)
+
+		req := &ngrok.HTTPSEdgeRouteUpdate{
+			EdgeID:    edge.Status.ID,
+			ID:        match.ID,
+			Match:     routeSpec.Match,
+			MatchType: routeSpec.MatchType,
+			Backend: &ngrok.EndpointBackendMutate{
+				BackendID: backend.ID,
+			},
+		}
+		route, err = edgeRoutes.Update(ctx, req)
+		if err != nil {
+			return err
+		}
+
+		// With the route modules successfully applied and the edge updated, we now update the route's backend status
 		if route.Backend != nil {
 			routeStatuses[i].Backend = ingressv1alpha1.TunnelGroupBackendStatus{
 				ID: route.Backend.Backend.ID,
 			}
 		}
-
-		// Update all route modules for a given route
-		if err := routeModuleUpdater.updateModulesForRoute(ctx, route, &routeSpec); err != nil {
-			return err
-		}
 	}
 
 	// Delete any routes that are no longer in the spec
@@ -267,7 +288,7 @@ func (r *HTTPSEdgeReconciler) reconcileRoutes(ctx context.Context, edge *ingress
 		}
 		if !found {
 			r.Log.Info("Deleting route", "edgeID", edge.Status.ID, "routeID", remoteRoute.ID)
-			if err := r.NgrokClientset.HTTPSEdgeRoutes().Delete(ctx, &ngrok.EdgeRouteItem{EdgeID: edge.Status.ID, ID: remoteRoute.ID}); err != nil {
+			if err := edgeRoutes.Delete(ctx, &ngrok.EdgeRouteItem{EdgeID: edge.Status.ID, ID: remoteRoute.ID}); err != nil {
 				return err
 			}
 		}