Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip] ethdb: increase ideal batch size by 10x #20197

Closed
wants to merge 3 commits into from
Closed

[wip] ethdb: increase ideal batch size by 10x #20197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4e835e5
ethdb: increase ideal batch size by 10x
holiman Oct 23, 2019
1fb58b2
leveldb: add gauge for batch usage
holiman Oct 23, 2019
7f39561
leveldb: fix nil deref
holiman Oct 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ethdb/batch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package ethdb

// IdealBatchSize defines the size of the data batches should ideally add in one
// write.
const IdealBatchSize = 100 * 1024
const IdealBatchSize = 1000 * 1024

// Batch is a write-only database that commits changes to its host database
// when Write is called. A batch cannot be used concurrently.
Expand Down
17 changes: 12 additions & 5 deletions ethdb/leveldb/leveldb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ type Database struct {
level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0
nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level
seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt
batchWriteGauge metrics.Gauge // Gauge for tracking the number of batched writes we do

quitLock sync.Mutex // Mutex protecting the quit channel access
quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
Expand Down Expand Up @@ -127,6 +128,7 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro
ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil)
ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil)
ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil)
ldb.batchWriteGauge = metrics.NewRegisteredGauge(namespace+"batch/write/counter", nil)

// Start up the metrics gathering and return
go ldb.meter(metricsGatheringInterval)
Expand Down Expand Up @@ -178,8 +180,9 @@ func (db *Database) Delete(key []byte) error {
// database until a final write is called.
func (db *Database) NewBatch() ethdb.Batch {
return &batch{
db: db.db,
b: new(leveldb.Batch),
db: db.db,
b: new(leveldb.Batch),
writeGauge: db.batchWriteGauge,
}
}

Expand Down Expand Up @@ -425,9 +428,10 @@ func (db *Database) meter(refresh time.Duration) {
// batch is a write-only leveldb batch that commits changes to its host database
// when Write is called. A batch cannot be used concurrently.
type batch struct {
db *leveldb.DB
b *leveldb.Batch
size int
db *leveldb.DB
b *leveldb.Batch
size int
writeGauge metrics.Gauge
}

// Put inserts the given value into the batch for later committing.
Expand All @@ -451,6 +455,9 @@ func (b *batch) ValueSize() int {

// Write flushes any accumulated data to disk.
func (b *batch) Write() error {
if b.writeGauge != nil {
b.writeGauge.Inc(1)
}
return b.db.Write(b.b, nil)
}

Expand Down