sentry: db.go:550: log.Fatal: write ×: no space left on device -- *errutil.leafError: log.Fatal: write ×: no space left on device (1) *secondary.withSecondaryError db.go:550: *withstack.withStack (top exception) (check the extra data payloads)

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2144732109/?referrer=webhooks_plugin

Panic message:

db.go:550: log.Fatal: write ×: no space left on device
--
*errutil.leafError: log.Fatal: write ×: no space left on device (1)
*secondary.withSecondaryError
db.go:550: *withstack.withStack (top exception)
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

https://github.com/cockroachdb/cockroach/blob/89dd2d14c29787378c434ed54937757ef5d9877c/vendor/github.com/cockroachdb/pebble/db.go#L549-L551 in github.com/cockroachdb/pebble.(*DB).Apply
https://github.com/cockroachdb/cockroach/blob/89dd2d14c29787378c434ed54937757ef5d9877c/vendor/github.com/cockroachdb/pebble/db.go#L429-L431 in github.com/cockroachdb/pebble.(*DB).Set

cockroach/pkg/storage/pebble.go

Lines 715 to 717 in 89dd2d1

	}
	return p.db.Set(EncodeKey(key), value, pebble.Sync)
	}

in pkg/storage.(*Pebble).Put

cockroach/pkg/storage/mvcc.go

Lines 1167 to 1169 in 89dd2d1

	}
	if err := writer.Put(key, bytes); err != nil {
	return 0, 0, err

in pkg/storage.(*putBuffer).putMeta

cockroach/pkg/storage/mvcc.go

Lines 1498 to 1500 in 89dd2d1

	buf.meta = enginepb.MVCCMetadata{RawBytes: value}
	metaKeySize, metaValSize, err = buf.putMeta(writer, metaKey, &buf.meta)
	}

in pkg/storage.mvccPutInternal

cockroach/pkg/storage/mvcc.go

Lines 1278 to 1280 in 89dd2d1

	err := mvccPutInternal(ctx, writer, iter, ms, key, timestamp, rawBytes,
	txn, buf, valueFn)

in pkg/storage.mvccPutUsingIter

cockroach/pkg/storage/mvcc.go

Lines 1206 to 1208 in 89dd2d1

	}
	return mvccPutUsingIter(ctx, rw, iter, ms, key, timestamp, value, txn, nil /* valueFn */)
	}

in pkg/storage.MVCCPut

cockroach/pkg/storage/mvcc.go

Lines 682 to 684 in 89dd2d1

	value.InitChecksum(key)
	return MVCCPut(ctx, rw, ms, key, timestamp, value, txn)
	}

in pkg/storage.MVCCPutProto

cockroach/pkg/kv/kvserver/store.go

Lines 2076 to 2078 in 89dd2d1

	ctx = s.AnnotateCtx(ctx)
	return storage.MVCCPutProto(
	ctx,

in pkg/kv/kvserver.(*Store).WriteLastUpTimestamp

cockroach/pkg/server/server.go

Lines 1553 to 1555 in 89dd2d1

	if err := s.node.stores.VisitStores(func(s *kvserver.Store) error {
	return s.WriteLastUpTimestamp(ctx, now)
	}); err != nil {

in pkg/server.(*Server).Start.func17.1

cockroach/pkg/kv/kvserver/stores.go

Lines 130 to 132 in 89dd2d1

	ls.storeMap.Range(func(k int64, v unsafe.Pointer) bool {
	err = visitor((*Store)(v))
	return err == nil

in pkg/kv/kvserver.(*Stores).VisitStores.func1

cockroach/pkg/util/syncutil/int_map.go

Lines 351 to 353 in 89dd2d1

	}
	if !f(k, v) {
	break

in pkg/util/syncutil.(*IntMap).Range

cockroach/pkg/kv/kvserver/stores.go

Lines 129 to 131 in 89dd2d1

	var err error
	ls.storeMap.Range(func(k int64, v unsafe.Pointer) bool {
	err = visitor((*Store)(v))

in pkg/kv/kvserver.(*Stores).VisitStores

cockroach/pkg/server/server.go

Lines 1552 to 1554 in 89dd2d1

	now := s.clock.Now()
	if err := s.node.stores.VisitStores(func(s *kvserver.Store) error {
	return s.WriteLastUpTimestamp(ctx, now)

in pkg/server.(*Server).Start.func17

cockroach/pkg/kv/kvserver/node_liveness.go

Lines 1209 to 1211 in 89dd2d1

	if cb != nil {
	cb(ctx)
	}

in pkg/kv/kvserver.(*NodeLiveness).updateLivenessAttempt

cockroach/pkg/kv/kvserver/node_liveness.go

Lines 1127 to 1129 in 89dd2d1

	}
	written, err := nl.updateLivenessAttempt(ctx, update, handleCondFailed)
	if err != nil {

in pkg/kv/kvserver.(*NodeLiveness).updateLiveness

cockroach/pkg/kv/kvserver/node_liveness.go

Lines 877 to 879 in 89dd2d1

	}
	written, err := nl.updateLiveness(ctx, update, func(actual LivenessRecord) error {
	// Update liveness to actual value on mismatch.

in pkg/kv/kvserver.(*NodeLiveness).heartbeatInternal

cockroach/pkg/kv/kvserver/node_liveness.go

Lines 735 to 737 in 89dd2d1

	func (nl *NodeLiveness) Heartbeat(ctx context.Context, liveness kvserverpb.Liveness) error {
	return nl.heartbeatInternal(ctx, liveness, false /* increment epoch */)
	}

in pkg/kv/kvserver.(*NodeLiveness).Heartbeat

cockroach/pkg/kv/kvserver/replica_range_lease.go

Lines 337 to 339 in 89dd2d1

	p.repl.store.StoreID() == nextLeaseHolder.StoreID {
	if err = p.repl.store.cfg.NodeLiveness.Heartbeat(ctx, status.Liveness); err != nil {
	log.Errorf(ctx, "%v", err)

in pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func2

cockroach/pkg/util/stop/stopper.go

Lines 346 to 348 in 89dd2d1

	f(ctx)
	}()

in pkg/util/stop.(*Stopper).RunAsyncTask.func1
/usr/local/go/src/runtime/asm_amd64.s#L1356-L1358 in runtime.goexit

vendor/github.com/cockroachdb/pebble/db.go in github.com/cockroachdb/pebble.(*DB).Apply at line 550
vendor/github.com/cockroachdb/pebble/db.go in github.com/cockroachdb/pebble.(*DB).Set at line 430
pkg/storage/pebble.go in pkg/storage.(*Pebble).Put at line 716
pkg/storage/mvcc.go in pkg/storage.(*putBuffer).putMeta at line 1168
pkg/storage/mvcc.go in pkg/storage.mvccPutInternal at line 1499
pkg/storage/mvcc.go in pkg/storage.mvccPutUsingIter at line 1279
pkg/storage/mvcc.go in pkg/storage.MVCCPut at line 1207
pkg/storage/mvcc.go in pkg/storage.MVCCPutProto at line 683
pkg/kv/kvserver/store.go in pkg/kv/kvserver.(*Store).WriteLastUpTimestamp at line 2077
pkg/server/server.go in pkg/server.(*Server).Start.func17.1 at line 1554
pkg/kv/kvserver/stores.go in pkg/kv/kvserver.(*Stores).VisitStores.func1 at line 131
pkg/util/syncutil/int_map.go in pkg/util/syncutil.(*IntMap).Range at line 352
pkg/kv/kvserver/stores.go in pkg/kv/kvserver.(*Stores).VisitStores at line 130
pkg/server/server.go in pkg/server.(*Server).Start.func17 at line 1553
pkg/kv/kvserver/node_liveness.go in pkg/kv/kvserver.(*NodeLiveness).updateLivenessAttempt at line 1210
pkg/kv/kvserver/node_liveness.go in pkg/kv/kvserver.(*NodeLiveness).updateLiveness at line 1128
pkg/kv/kvserver/node_liveness.go in pkg/kv/kvserver.(*NodeLiveness).heartbeatInternal at line 878
pkg/kv/kvserver/node_liveness.go in pkg/kv/kvserver.(*NodeLiveness).Heartbeat at line 736
pkg/kv/kvserver/replica_range_lease.go in pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func2 at line 338
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunAsyncTask.func1 at line 347
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1357

Tag	Value
Cockroach Release	v20.2.3
Cockroach SHA:	89dd2d1
Platform	darwin amd64
Distribution	CCL
Environment	v20.2.3
Command	start-single-node
Go Version	``
# of CPUs
# of Goroutines

[yuzefovich]

dup of #58281