stop: clarify the defer behavior of Stop, introduce EmergencyStop

Release note: None

pkg/cli/democluster/demo_cluster.go

@@ -802,6 +802,10 @@ func TestingForceRandomizeDemoPorts() func() {
 
 func (c *transientCluster) Close(ctx context.Context) {
 	if c.stopper != nil {
+		if r := recover(); r != nil {
+			c.stopper.EmergencyStop(ctx)
+			panic(r)
+		}
 		c.stopper.Stop(ctx)
 	}
 	if c.demoDir != "" {

pkg/util/stop/stopper.go

@@ -550,12 +550,7 @@ func (s *Stopper) Stop(ctx context.Context) {
 	// avoids stalls and helps some tests in `./cli` finish cleanly (where
 	// panics happen on purpose).
 	if r := recover(); r != nil {
-		go s.Quiesce(ctx)
-		s.mu.Lock()
-		for _, c := range s.mu.closers {
-			go c.Close()
-		}
-		s.mu.Unlock()
+		s.EmergencyStop(ctx)
 		panic(r)
 	}
 
@@ -569,6 +564,39 @@ func (s *Stopper) Stop(ctx context.Context) {
 	}
 }
 
+// EmergencyStop should be called instead of Stop() when the Stop()
+// call is intended to be called under `defer`, but it cannot be
+// called _directly_ via `defer`, and the caller intends the call to
+// handle panics through the return path.  For example, when
+//
+//	defer stopper.Stop(ctx)
+//
+// is not directly possible, such as when the Stop call is inside a
+// closure like this:
+//
+//	defer func() {
+//	   ... stopper.Stop(ctx) ...
+//	}()
+//
+// The Stop call will not catch panics properly. In that case,
+// use:
+//
+//	defer func() {
+//	     if r := recover(); r != nil {
+//	        stopper.EmergencyStop(ctx)
+//	        panic(r)
+//	     }
+//	     stopper.Stop(ctx)
+//	}()
+func (s *Stopper) EmergencyStop(ctx context.Context) {
+	go s.Quiesce(ctx)
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, c := range s.mu.closers {
+		go c.Close()
+	}
+}
+
 // ShouldQuiesce returns a channel which will be closed when Stop() has been
 // invoked and outstanding tasks should begin to quiesce.
 func (s *Stopper) ShouldQuiesce() <-chan struct{} {