use batch (#1)

 use batch for faster insert

go.mod

@@ -2,7 +2,11 @@ module level2pebble
 
 go 1.18
 
-require github.com/tendermint/tm-db v0.6.7
+require (
+	github.com/cockroachdb/pebble v0.0.0-20220726134658-7b78c71e4055
+	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
+	github.com/tendermint/tm-db v0.6.7
+)
 
 require (
 	github.com/DataDog/zstd v1.4.5 // indirect
@@ -11,7 +15,6 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cockroachdb/errors v1.8.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
-	github.com/cockroachdb/pebble v0.0.0-20220726134658-7b78c71e4055 // indirect
 	github.com/cockroachdb/redact v1.0.8 // indirect
 	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/cosmos/gorocksdb v1.2.0 // indirect
@@ -30,7 +33,6 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.opencensus.io v0.22.5 // indirect
 	golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
@@ -39,4 +41,4 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 )
 
-replace github.com/tendermint/tm-db => github.com/baabeetaa/tm-db v0.6.7-0.20220728101401-7c0a33fd88c8
+replace github.com/tendermint/tm-db => github.com/baabeetaa/tm-db v0.6.7-0.20220827003937-31989c12be6f

go.sum

@@ -18,8 +18,8 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
-github.com/baabeetaa/tm-db v0.6.7-0.20220728101401-7c0a33fd88c8 h1:X+t2DFByU+pjr9mQnGMdoLpbDZ9W4CdUcIMcLqb//lY=
-github.com/baabeetaa/tm-db v0.6.7-0.20220728101401-7c0a33fd88c8/go.mod h1:89wa64apdojIUNiEO2D6WmfDEPduTVpsbstMwxCY2Ks=
+github.com/baabeetaa/tm-db v0.6.7-0.20220827003937-31989c12be6f h1:+pTUthDVIt3W61cJk3xGdgb/5AEmK8afrTPWWIXJmso=
+github.com/baabeetaa/tm-db v0.6.7-0.20220827003937-31989c12be6f/go.mod h1:89wa64apdojIUNiEO2D6WmfDEPduTVpsbstMwxCY2Ks=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

main.go

@@ -1,7 +1,13 @@
 package main
 
 import (
+	"encoding/hex"
 	"fmt"
+	"github.com/cockroachdb/pebble"
+	"math"
+
+	levelopt "github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
 	tmdb "github.com/tendermint/tm-db"
 	"os"
 	"path/filepath"
@@ -20,12 +26,28 @@ func main() {
 	dbDirSource := filepath.Dir(os.Args[1])
 	dbDirTarget := os.Args[2]
 
-	dbLev, errLev := tmdb.NewGoLevelDB(dbName, dbDirSource)
+	// options to disable compaction for goleveldb
+	levelOptions := levelopt.Options{
+		ReadOnly: true,
+		//CompactionTableSizeMultiplier: 2.0,
+	}
+	dbLev, errLev := tmdb.NewGoLevelDBWithOpts(dbName, dbDirSource, &levelOptions)
 	if errLev != nil {
 		panic(errLev)
 	}
 
-	dbPeb, errPeb := tmdb.NewPebbleDB(dbName, dbDirTarget)
+	// options to disable compaction for pebbledb
+	pebbleOptions := &pebble.Options{
+		L0CompactionFileThreshold:   math.MaxInt32,
+		L0CompactionThreshold:       math.MaxInt32,
+		L0StopWritesThreshold:       math.MaxInt32,
+		MaxConcurrentCompactions:    1,
+		DisableAutomaticCompactions: true,
+		MaxOpenFiles:                100,
+	}
+	pebbleOptions.Experimental.ReadCompactionRate = math.MaxInt32
+	pebbleOptions.EnsureDefaults()
+	dbPeb, errPeb := tmdb.NewPebbleDBWithOpts(dbName, dbDirTarget, pebbleOptions)
 
 	if errPeb != nil {
 		panic(errPeb)
@@ -36,30 +58,72 @@ func main() {
 		dbLev.Close()
 	}()
 
-	itr, itrErr := dbLev.Iterator(nil, nil)
-
-	if itrErr != nil {
-		panic(itrErr)
+	readOptions := levelopt.ReadOptions{
+		DontFillCache: true,
+		Strict:        levelopt.DefaultStrict,
 	}
 
+	itr := dbLev.DB().NewIterator(&util.Range{Start: nil, Limit: nil}, &readOptions)
+
+	defer func() {
+		// itr.Close()
+		itr.Release()
+	}()
+
 	offset := 0
 
-	for ; itr.Valid(); itr.Next() {
-		key := itr.Key()
-		value := itr.Value()
+	rawDBPebble := dbPeb.DB()
+	bat := rawDBPebble.NewBatch()
+
+	for itr.First(); itr.Valid(); itr.Next() {
+		offset++
+
+		key := cp(itr.Key())
+		value := cp(itr.Value())
 
-		errSet := dbPeb.Set(key, value)
+		errSet := bat.Set(key, value, pebble.Sync)
 		if errSet != nil {
 			panic(errSet)
 		}
 
-		offset++
+		if bat.Len() >= 107374182 { // 100 MB
+			str_hex_key := hex.EncodeToString(key)
+			fmt.Printf("processing %s: %d, key=%s\n", dbName, offset, str_hex_key)
+
+			errComit := bat.Commit(pebble.Sync)
+			if errComit != nil {
+				panic(errComit)
+			}
+
+			errFlush := rawDBPebble.Flush()
+			if errFlush != nil {
+				panic(errFlush)
+			}
+
+			bat.Reset()
 
-		if offset%1000000 == 0 {
-			fmt.Printf("processing %s: %d\n", dbName, offset)
 			runtime.GC() // Force GC
 		}
 	}
 
-	itr.Close()
+	// write the last batch
+	errComit := bat.Commit(pebble.Sync)
+	if errComit != nil {
+		panic(errComit)
+	}
+
+	errFlush := rawDBPebble.Flush()
+	if errFlush != nil {
+		panic(errFlush)
+	}
+
+	bat.Close()
+
+	fmt.Printf("Done!")
+}
+
+func cp(bz []byte) (ret []byte) {
+	ret = make([]byte, len(bz))
+	copy(ret, bz)
+	return ret
 }