storage: replace a few panics with log.Fatal

panic causes stacks to unwind and defers to be invoked (possibly
releasing locks) until eventually something either handles the
panic (unlikely) or we reach the runtime and it kills the process. While
the process is dying, other goroutines are still running which may
encounter invalid states due to the panic. These invalid states can
produce fatal errors of there own which seem impossible and are
difficult to debug. For fatal errors we should use log.Fatal{,f} instead
which will kill the process before any locks are released.

See cockroachdb#16479

Author: petermattis

pkg/storage/replica.go

@@ -1580,7 +1580,10 @@ func (r *Replica) assertStateLocked(ctx context.Context, reader engine.Reader) {
 		// TODO(dt): expose properly once #15892 is addressed.
 		log.Errorf(ctx, "on-disk and in-memory state diverged:\n%s", pretty.Diff(diskState, r.mu.state))
 		r.mu.state.Desc, diskState.Desc = nil, nil
-		panic(log.Safe{V: fmt.Sprintf("on-disk and in-memory state diverged: %s", pretty.Diff(diskState, r.mu.state))})
+		log.Fatal(ctx, log.Safe{
+			V: fmt.Sprintf("on-disk and in-memory state diverged: %s",
+				pretty.Diff(diskState, r.mu.state)),
+		})
 	}
 }
 
@@ -4396,7 +4399,7 @@ func (r *Replica) evaluateTxnWriteBatch(
 				err := roachpb.NewTransactionRetryError(roachpb.RETRY_REASON_UNKNOWN)
 				return nil, ms, nil, EvalResult{}, roachpb.NewError(err)
 			}
-			panic("unreachable")
+			log.Fatal(ctx, "unreachable")
 		}
 
 		log.VEventf(ctx, 2, "1PC execution failed, reverting to regular execution for batch")

pkg/storage/store.go

@@ -896,7 +896,7 @@ func NewStore(cfg StoreConfig, eng engine.Engine, nodeDesc *roachpb.NodeDescript
 	cfg.SetDefaults()
 
 	if !cfg.Valid() {
-		panic(fmt.Sprintf("invalid store configuration: %+v", &cfg))
+		log.Fatalf(context.Background(), "invalid store configuration: %+v", &cfg)
 	}
 	s := &Store{
 		cfg:      cfg,
@@ -1387,7 +1387,7 @@ func (s *Store) GossipStore(ctx context.Context) error {
 	select {
 	case <-s.cfg.Gossip.Connected:
 	default:
-		panic(fmt.Sprintf("%s: not connected to gossip", s))
+		log.Fatalf(ctx, "not connected to gossip")
 	}
 
 	storeDesc, err := s.Descriptor()
@@ -2854,7 +2854,7 @@ func (s *Store) HandleRaftRequest(
 ) *roachpb.Error {
 	if len(req.Heartbeats)+len(req.HeartbeatResps) > 0 {
 		if req.RangeID != 0 {
-			panic("coalesced heartbeats must have rangeID == 0")
+			log.Fatalf(ctx, "coalesced heartbeats must have rangeID == 0")
 		}
 		s.uncoalesceBeats(ctx, req.Heartbeats, req.FromReplica, req.ToReplica, raftpb.MsgHeartbeat, respStream)
 		s.uncoalesceBeats(ctx, req.HeartbeatResps, req.FromReplica, req.ToReplica, raftpb.MsgHeartbeatResp, respStream)
@@ -2915,7 +2915,7 @@ func (s *Store) processRaftRequest(
 
 	if req.Quiesce {
 		if req.Message.Type != raftpb.MsgHeartbeat {
-			panic(fmt.Sprintf("unexpected quiesce: %+v", req))
+			log.Fatalf(ctx, "unexpected quiesce: %+v", req)
 		}
 		status := r.RaftStatus()
 		if status != nil && status.Term == req.Message.Term && status.Commit == req.Message.Commit {
@@ -3150,7 +3150,7 @@ func (s *Store) processRaftRequest(
 
 	if _, err := r.handleRaftReadyRaftMuLocked(inSnap); err != nil {
 		// mimic the behavior in processRaft.
-		panic(err)
+		log.Fatal(ctx, err)
 	}
 	removePlaceholder = false
 	return nil
@@ -3450,7 +3450,7 @@ func (s *Store) processReady(rangeID roachpb.RangeID) {
 	if ok {
 		stats, err := r.handleRaftReady(IncomingSnapshot{})
 		if err != nil {
-			panic(err) // TODO(bdarnell)
+			log.Fatal(r.AnnotateCtx(context.Background()), err) // TODO(bdarnell)
 		}
 		elapsed := timeutil.Since(start)
 		s.metrics.RaftWorkingDurationNanos.Inc(elapsed.Nanoseconds())
@@ -3579,7 +3579,7 @@ func (s *Store) sendQueuedHeartbeatsToNode(
 	} else if len(beats) == 0 {
 		msgType = raftpb.MsgHeartbeatResp
 	} else {
-		panic("cannot coalesce both heartbeats and responses")
+		log.Fatal(s.AnnotateCtx(context.Background()), "cannot coalesce both heartbeats and responses")
 	}
 
 	chReq := &RaftMessageRequest{