Ignore reconcile errors that occur because a pod is being terminated (#…

…1233) * Ignore reconcile errors that occur because a pod is being terminated Signed-off-by: Kevin Earls <[email protected]> * Appease the all powerfull linter Signed-off-by: Kevin Earls <[email protected]> * Change behavior to end reconcile loop if pod has been terminated Signed-off-by: Kevin Earls <[email protected]> * Print a log message if we exit the reconciler loop Signed-off-by: Kevin Earls <[email protected]> * Look for NamespaceTerminatingCause Signed-off-by: Kevin Earls <[email protected]> * Appease the almighty linter Signed-off-by: Kevin Earls <[email protected]> * Fix log message Signed-off-by: Kevin Earls <[email protected]> * Skip flaky test Signed-off-by: Kevin Earls <[email protected]> Signed-off-by: Kevin Earls <[email protected]> Co-authored-by: Ben B <[email protected]>
open-telemetry · Dec 1, 2022 · ce30a35 · ce30a35
1 parent 61accfd
commit ce30a35
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/cmd/otel-allocator/allocation/least_weighted_test.go b/cmd/otel-allocator/allocation/least_weighted_test.go
@@ -181,6 +181,7 @@ func TestNoCollectorReassignment(t *testing.T) {
 }
 
 func TestSmartCollectorReassignment(t *testing.T) {
+	t.Skip("This test is flaky and fails frequently, see issue 1291")
 	s, _ := New("least-weighted", logger)
 
 	cols := makeNCollectors(4, 0)

diff --git a/controllers/opentelemetrycollector_controller.go b/controllers/opentelemetrycollector_controller.go
@@ -168,6 +168,11 @@ func (r *OpenTelemetryCollectorReconciler) Reconcile(ctx context.Context, req ct
 func (r *OpenTelemetryCollectorReconciler) RunTasks(ctx context.Context, params reconcile.Params) error {
 	for _, task := range r.tasks {
 		if err := task.Do(ctx, params); err != nil {
+			// If we get an error that occurs because a pod is being terminated, then exit this loop
+			if apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
+				r.log.V(2).Info("Exiting reconcile loop because namespace is being terminated", "namespace", params.Instance.Namespace)
+				return nil
+			}
 			r.log.Error(err, fmt.Sprintf("failed to reconcile %s", task.Name))
 			if task.BailOnError {
 				return err