Skip to content

Commit

Permalink
cli: fix indefinite retrying for cockroach quit when quorum is lost
Browse files Browse the repository at this point in the history
Fixes #14620

added a timeout of 1 minute inside runQuit, after which a hard shutdown is initiated
  • Loading branch information
xphoniex committed Apr 13, 2017
1 parent 04ee3b0 commit 93cc469
Showing 1 changed file with 18 additions and 8 deletions.
26 changes: 18 additions & 8 deletions pkg/cli/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,11 @@ func runStart(cmd *cobra.Command, args []string) error {
case err := <-errChan:
return err
case <-stopper.ShouldStop():
<-stopper.IsStopped()
const msgDone = "server drained and shutdown completed"
log.Infof(shutdownCtx, msgDone)
fmt.Fprintln(os.Stdout, msgDone)
return nil
case sig := <-signalCh:
log.Infof(shutdownCtx, "received signal '%s'", sig)
if sig == os.Interrupt {
Expand Down Expand Up @@ -729,17 +734,22 @@ func runQuit(_ *cobra.Command, _ []string) (err error) {
defer stopper.Stop()

ctx := stopperContext(stopper)

if err := doShutdown(ctx, c, onModes); err != nil {
if _, ok := err.(*errTryHardShutdown); ok {
fmt.Fprintf(
os.Stdout, "graceful shutdown failed: %s\nproceeding with hard shutdown\n", err,
)
} else {
errChan := make(chan error, 1)
go func() {
errChan <- doShutdown(ctx, c, onModes)
}()
select {
case err := <-errChan:
if err != nil {
if _, ok := err.(errTryHardShutdown); ok {
fmt.Printf("graceful shutdown failed: %s; proceeding with hard shutdown\n", err)
break
}
return err
}
} else {
return nil
case <-time.After(time.Minute):
fmt.Println("timed out; proceeding with hard shutdown")
}
// Not passing drain modes tells the server to not bother and go
// straight to shutdown.
Expand Down

0 comments on commit 93cc469

Please sign in to comment.