Skip to content

Commit 5e945a2

Browse files
petermattispetermattis
authored
Merge pull request #13481 from petermattis/pmattis/sync-raft-log
storage: sync Raft log writes
2 parents 7b91fbc + 04be417 commit 5e945a2

13 files changed

+89
-66
lines changed

‎pkg/ccl/storageccl/writebatch.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ func evalWriteBatch(
7272
return storage.EvalResult{}, errors.New("WriteBatch can only be called on empty ranges")
7373
}
7474

75-
if err := batch.ApplyBatchRepr(args.Data); err != nil {
75+
if err := batch.ApplyBatchRepr(args.Data, false /* !sync */); err != nil {
7676
return storage.EvalResult{}, err
7777
}
7878
return storage.EvalResult{}, nil

‎pkg/storage/abort_cache.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ func (sc *AbortCache) ClearData(e engine.Engine) error {
9494
if err != nil {
9595
return err
9696
}
97-
return b.Commit()
97+
return b.Commit(false /* !sync */)
9898
}
9999

100100
// Get looks up an abort cache entry recorded for this transaction ID.

‎pkg/storage/engine/batch_test.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ func testBatchBasics(t *testing.T, writeOnly bool, commit func(e Engine, b Batch
129129
func TestBatchBasics(t *testing.T) {
130130
defer leaktest.AfterTest(t)()
131131
testBatchBasics(t, false /* writeOnly */, func(e Engine, b Batch) error {
132-
return b.Commit()
132+
return b.Commit(false /* !sync */)
133133
})
134134
}
135135

@@ -231,14 +231,14 @@ func TestBatchRepr(t *testing.T) {
231231
t.Fatalf("expected %v, but found %v", expOps, ops)
232232
}
233233

234-
return e.ApplyBatchRepr(repr)
234+
return e.ApplyBatchRepr(repr, false /* !sync */)
235235
})
236236
}
237237

238238
func TestWriteBatchBasics(t *testing.T) {
239239
defer leaktest.AfterTest(t)()
240240
testBatchBasics(t, true /* writeOnly */, func(e Engine, b Batch) error {
241-
return b.Commit()
241+
return b.Commit(false /* !sync */)
242242
})
243243
}
244244

@@ -264,7 +264,7 @@ func TestApplyBatchRepr(t *testing.T) {
264264

265265
b2 := e.NewBatch()
266266
defer b2.Close()
267-
if err := b2.ApplyBatchRepr(repr1); err != nil {
267+
if err := b2.ApplyBatchRepr(repr1, false /* !sync */); err != nil {
268268
t.Fatal(err)
269269
}
270270
repr2 := b2.Repr()
@@ -290,11 +290,11 @@ func TestApplyBatchRepr(t *testing.T) {
290290

291291
b4 := e.NewBatch()
292292
defer b4.Close()
293-
if err := b4.ApplyBatchRepr(repr); err != nil {
293+
if err := b4.ApplyBatchRepr(repr, false /* !sync */); err != nil {
294294
t.Fatal(err)
295295
}
296296
// Intentionally don't call Repr() because the expected user wouldn't.
297-
if err := b4.Commit(); err != nil {
297+
if err := b4.Commit(false /* !sync */); err != nil {
298298
t.Fatal(err)
299299
}
300300

@@ -456,7 +456,7 @@ func TestBatchProto(t *testing.T) {
456456
t.Fatalf("expected GetProto to fail ok=%t: %s", ok, err)
457457
}
458458
// Commit and verify the proto can be read directly from the engine.
459-
if err := b.Commit(); err != nil {
459+
if err := b.Commit(false /* !sync */); err != nil {
460460
t.Fatal(err)
461461
}
462462
if ok, _, _, err := e.GetProto(mvccKey("proto"), getVal); !ok || err != nil {
@@ -545,7 +545,7 @@ func TestBatchScan(t *testing.T) {
545545
}
546546

547547
// Now, commit batch and re-scan using engine direct to compare results.
548-
if err := b.Commit(); err != nil {
548+
if err := b.Commit(false /* !sync */); err != nil {
549549
t.Fatal(err)
550550
}
551551
for i, scan := range scans {
@@ -912,7 +912,7 @@ func TestBatchDistinctPanics(t *testing.T) {
912912
func() { _ = batch.Put(a, nil) },
913913
func() { _ = batch.Merge(a, nil) },
914914
func() { _ = batch.Clear(a) },
915-
func() { _ = batch.ApplyBatchRepr(nil) },
915+
func() { _ = batch.ApplyBatchRepr(nil, false) },
916916
func() { _, _ = batch.Get(a) },
917917
func() { _, _, _, _ = batch.GetProto(a, nil) },
918918
func() { _ = batch.Iterate(a, a, nil) },

‎pkg/storage/engine/bench_test.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func setupMVCCData(
124124
// sstables.
125125
if scaled := len(order) / 20; i > 0 && (i%scaled) == 0 {
126126
log.Infof(context.Background(), "committing (%d/~%d)", i/scaled, 20)
127-
if err := batch.Commit(); err != nil {
127+
if err := batch.Commit(false /* !sync */); err != nil {
128128
b.Fatal(err)
129129
}
130130
batch.Close()
@@ -143,7 +143,7 @@ func setupMVCCData(
143143
b.Fatal(err)
144144
}
145145
}
146-
if err := batch.Commit(); err != nil {
146+
if err := batch.Commit(false /* !sync */); err != nil {
147147
b.Fatal(err)
148148
}
149149
batch.Close()
@@ -352,7 +352,7 @@ func runMVCCBatchPut(emk engineMaker, valueSize, batchSize int, b *testing.B) {
352352
}
353353
}
354354

355-
if err := batch.Commit(); err != nil {
355+
if err := batch.Commit(false /* !sync */); err != nil {
356356
b.Fatal(err)
357357
}
358358

@@ -403,7 +403,7 @@ func runMVCCBatchTimeSeries(emk engineMaker, batchSize int, b *testing.B) {
403403
}
404404
}
405405

406-
if err := batch.Commit(); err != nil {
406+
if err := batch.Commit(false /* !sync */); err != nil {
407407
b.Fatal(err)
408408
}
409409
batch.Close()
@@ -561,7 +561,7 @@ func runBatchApplyBatchRepr(
561561
} else {
562562
batch = eng.NewBatch()
563563
}
564-
if err := batch.ApplyBatchRepr(repr); err != nil {
564+
if err := batch.ApplyBatchRepr(repr, false /* !sync */); err != nil {
565565
b.Fatal(err)
566566
}
567567
batch.Close()

‎pkg/storage/engine/engine.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,10 @@ type Reader interface {
125125
type Writer interface {
126126
// ApplyBatchRepr atomically applies a set of batched updates. Created by
127127
// calling Repr() on a batch. Using this method is equivalent to constructing
128-
// and committing a batch whose Repr() equals repr.
129-
ApplyBatchRepr(repr []byte) error
128+
// and committing a batch whose Repr() equals repr. If sync is true, the
129+
// batch is synchronously written to disk. It is an error to specify
130+
// sync=true if the Writer is a Batch.
131+
ApplyBatchRepr(repr []byte, sync bool) error
130132
// Clear removes the item from the db with the given key.
131133
// Note that clear actually removes entries from the storage
132134
// engine, rather than inserting tombstones.
@@ -202,8 +204,9 @@ type Engine interface {
202204
type Batch interface {
203205
ReadWriter
204206
// Commit atomically applies any batched updates to the underlying
205-
// engine. This is a noop unless the engine was created via NewBatch().
206-
Commit() error
207+
// engine. This is a noop unless the engine was created via NewBatch(). If
208+
// sync is true, the batch is synchronously committed to disk.
209+
Commit(sync bool) error
207210
// Distinct returns a view of the existing batch which only sees writes that
208211
// were performed before the Distinct batch was created. That is, the
209212
// returned batch will not read its own writes, but it will read writes to

‎pkg/storage/engine/engine_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func TestEngineBatchCommit(t *testing.T) {
115115
t.Fatal(err)
116116
}
117117
}
118-
if err := batch.Commit(); err != nil {
118+
if err := batch.Commit(false /* !sync */); err != nil {
119119
t.Fatal(err)
120120
}
121121
close(writesDone)
@@ -320,7 +320,7 @@ func TestEngineBatch(t *testing.T) {
320320
}
321321
iter.Close()
322322
// Commit the batch and try getting the value from the engine.
323-
if err := b.Commit(); err != nil {
323+
if err := b.Commit(false /* !sync */); err != nil {
324324
t.Errorf("%d: %v", i, err)
325325
continue
326326
}

‎pkg/storage/engine/rocksdb.go

+10-10
Original file line numberDiff line numberDiff line change
@@ -444,8 +444,8 @@ func (r *RocksDB) Merge(key MVCCKey, value []byte) error {
444444
// ApplyBatchRepr atomically applies a set of batched updates. Created by
445445
// calling Repr() on a batch. Using this method is equivalent to constructing
446446
// and committing a batch whose Repr() equals repr.
447-
func (r *RocksDB) ApplyBatchRepr(repr []byte) error {
448-
return dbApplyBatchRepr(r.rdb, repr)
447+
func (r *RocksDB) ApplyBatchRepr(repr []byte, sync bool) error {
448+
return dbApplyBatchRepr(r.rdb, repr, sync)
449449
}
450450

451451
// Get returns the value for the given key.
@@ -968,13 +968,13 @@ func (r *rocksDBBatch) Merge(key MVCCKey, value []byte) error {
968968

969969
// ApplyBatchRepr atomically applies a set of batched updates to the current
970970
// batch (the receiver).
971-
func (r *rocksDBBatch) ApplyBatchRepr(repr []byte) error {
971+
func (r *rocksDBBatch) ApplyBatchRepr(repr []byte, sync bool) error {
972972
if r.distinctOpen {
973973
panic("distinct batch open")
974974
}
975975
r.flushMutations()
976976
r.flushes++ // make sure that Repr() doesn't take a shortcut
977-
return dbApplyBatchRepr(r.batch, repr)
977+
return dbApplyBatchRepr(r.batch, repr, sync)
978978
}
979979

980980
func (r *rocksDBBatch) Get(key MVCCKey) ([]byte, error) {
@@ -1068,7 +1068,7 @@ func (r *rocksDBBatch) NewIterator(prefix bool) Iterator {
10681068
return iter
10691069
}
10701070

1071-
func (r *rocksDBBatch) Commit() error {
1071+
func (r *rocksDBBatch) Commit(sync bool) error {
10721072
if r.closed() {
10731073
panic("this batch was already committed")
10741074
}
@@ -1081,7 +1081,7 @@ func (r *rocksDBBatch) Commit() error {
10811081
// We've previously flushed mutations to the C++ batch, so we have to flush
10821082
// any remaining mutations as well and then commit the batch.
10831083
r.flushMutations()
1084-
if err := statusToError(C.DBCommitAndCloseBatch(r.batch)); err != nil {
1084+
if err := statusToError(C.DBCommitAndCloseBatch(r.batch, C.bool(sync))); err != nil {
10851085
return err
10861086
}
10871087
r.batch = nil
@@ -1091,7 +1091,7 @@ func (r *rocksDBBatch) Commit() error {
10911091

10921092
// Fast-path which avoids flushing mutations to the C++ batch. Instead, we
10931093
// directly apply the mutations to the database.
1094-
if err := r.parent.ApplyBatchRepr(r.builder.Finish()); err != nil {
1094+
if err := r.parent.ApplyBatchRepr(r.builder.Finish(), sync); err != nil {
10951095
return err
10961096
}
10971097
C.DBClose(r.batch)
@@ -1135,7 +1135,7 @@ func (r *rocksDBBatch) flushMutations() {
11351135
r.flushes++
11361136
r.flushedCount += r.builder.count
11371137
r.flushedSize += len(r.builder.repr)
1138-
if err := r.ApplyBatchRepr(r.builder.Finish()); err != nil {
1138+
if err := r.ApplyBatchRepr(r.builder.Finish(), false); err != nil {
11391139
panic(err)
11401140
}
11411141
// Force a seek of the underlying iterator on the next Seek/ReverseSeek.
@@ -1474,8 +1474,8 @@ func dbMerge(rdb *C.DBEngine, key MVCCKey, value []byte) error {
14741474
return statusToError(C.DBMerge(rdb, goToCKey(key), goToCSlice(value)))
14751475
}
14761476

1477-
func dbApplyBatchRepr(rdb *C.DBEngine, repr []byte) error {
1478-
return statusToError(C.DBApplyBatchRepr(rdb, goToCSlice(repr)))
1477+
func dbApplyBatchRepr(rdb *C.DBEngine, repr []byte, sync bool) error {
1478+
return statusToError(C.DBApplyBatchRepr(rdb, goToCSlice(repr), C.bool(sync)))
14791479
}
14801480

14811481
// dbGet returns the value for the given key.

0 commit comments

Comments
 (0)