Skip to content

Commit

Permalink
ethdb/leveldb: disable seek compaction (ethereum#20130)
Browse files Browse the repository at this point in the history
* vendor: update leveldb

* ethdb/leveldb: disable seek compaction and add metrics

* vendor: udpate latest levledb

* ethdb/leveldb: fix typo
  • Loading branch information
rjl493456442 authored and enriquefynn committed Feb 15, 2021
1 parent a5652ab commit 5bd205f
Show file tree
Hide file tree
Showing 15 changed files with 197 additions and 70 deletions.
48 changes: 40 additions & 8 deletions ethdb/leveldb/leveldb.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,18 @@ type Database struct {
fn string // filename for reporting
db *leveldb.DB // LevelDB instance

compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction
compReadMeter metrics.Meter // Meter for measuring the data read during compaction
compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction
diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database
diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read
diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written
compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction
compReadMeter metrics.Meter // Meter for measuring the data read during compaction
compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction
diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database
diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read
diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written
memCompGauge metrics.Gauge // Gauge for tracking the number of memory compaction
level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0
nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level
seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt

quitLock sync.Mutex // Mutex protecting the quit channel access
quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
Expand All @@ -96,6 +100,7 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro
BlockCacheCapacity: cache / 2 * opt.MiB,
WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally
Filter: filter.NewBloomFilter(10),
DisableSeeksCompaction: true,
})
if _, corrupted := err.(*errors.ErrCorrupted); corrupted {
db, err = leveldb.RecoverFile(file, nil)
Expand All @@ -118,6 +123,10 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro
ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil)
ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil)
ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil)
ldb.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil)
ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil)
ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil)
ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil)

// Start up the metrics gathering and return
go ldb.meter(metricsGatheringInterval)
Expand Down Expand Up @@ -375,6 +384,29 @@ func (db *Database) meter(refresh time.Duration) {
}
iostats[0], iostats[1] = nRead, nWrite

compCount, err := db.db.GetProperty("leveldb.compcount")
if err != nil {
db.log.Error("Failed to read database iostats", "err", err)
merr = err
continue
}

var (
memComp uint32
level0Comp uint32
nonLevel0Comp uint32
seekComp uint32
)
if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil {
db.log.Error("Compaction count statistic not found")
merr = err
continue
}
db.memCompGauge.Update(int64(memComp))
db.level0CompGauge.Update(int64(level0Comp))
db.nonlevel0CompGauge.Update(int64(nonLevel0Comp))
db.seekCompGauge.Update(int64(seekComp))

// Sleep a bit, then repeat the stats collection
select {
case errc = <-db.quitChan:
Expand Down
5 changes: 5 additions & 0 deletions vendor/github.com/syndtr/goleveldb/leveldb/batch.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 18 additions & 4 deletions vendor/github.com/syndtr/goleveldb/leveldb/db.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

43 changes: 26 additions & 17 deletions vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5bd205f

Please sign in to comment.