Skip to content

Commit

Permalink
all: add read-only option to database (ethereum#22407)
Browse files Browse the repository at this point in the history 
* all: add read-only option to database

* all: fixes tests

* cmd/geth: migrate flags

* cmd/geth: fix the compact

* cmd/geth: fix the format

* cmd/geth: fix log

* cmd: add chain-readonly

* core: add readonly notion to freezer

* core/rawdb: add log

* core/rawdb: fix freezer close

* cmd: fix

* cmd, core: construct db

* core: update tests
  • Loading branch information
@rjl493456442 @atif-konasl
rjl493456442 authored and atif-konasl committed Oct 15, 2021
1 parent e47b3b0 commit 9a362e9
Show file tree
Hide file tree
Showing 25 changed files with 264 additions and 189 deletions.
33 changes: 20 additions & 13 deletions cmd/geth/chaincmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
Expand Down Expand Up @@ -191,7 +192,7 @@ func initGenesis(ctx *cli.Context) error {
defer stack.Close()

for _, name := range []string{"chaindata", "lightchaindata"} {
chaindb, err := stack.OpenDatabase(name, 0, 0, "")
chaindb, err := stack.OpenDatabase(name, 0, 0, "", false)
if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}
Expand Down Expand Up @@ -229,7 +230,7 @@ func importChain(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, db := utils.MakeChain(ctx, stack, false)
chain, db := utils.MakeChain(ctx, stack)
defer db.Close()

// Start periodically gathering memory profiles
Expand Down Expand Up @@ -304,7 +305,7 @@ func exportChain(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, _ := utils.MakeChain(ctx, stack, true)
chain, _ := utils.MakeChain(ctx, stack)
start := time.Now()

var err error
Expand Down Expand Up @@ -340,7 +341,7 @@ func importPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, false)
start := time.Now()

if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
Expand All @@ -359,7 +360,7 @@ func exportPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, true)
start := time.Now()

if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
Expand All @@ -373,21 +374,27 @@ func dump(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
db := utils.MakeChainDatabase(ctx, stack, true)
for _, arg := range ctx.Args() {
var block *types.Block
var header *types.Header
if hashish(arg) {
block = chain.GetBlockByHash(common.HexToHash(arg))
hash := common.HexToHash(arg)
number := rawdb.ReadHeaderNumber(db, hash)
if number != nil {
header = rawdb.ReadHeader(db, hash, *number)
}
} else {
num, _ := strconv.Atoi(arg)
block = chain.GetBlockByNumber(uint64(num))
number, _ := strconv.Atoi(arg)
hash := rawdb.ReadCanonicalHash(db, uint64(number))
if hash != (common.Hash{}) {
header = rawdb.ReadHeader(db, hash, uint64(number))
}
}
if block == nil {
if header == nil {
fmt.Println("{}")
utils.Fatalf("block not found")
} else {
state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil)
state, err := state.New(header.Root, state.NewDatabase(db), nil)
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/geth/dao_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
}
// Retrieve the DAO config flag from the database
path := filepath.Join(datadir, "geth", "chaindata")
db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "")
db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false)
if err != nil {
t.Fatalf("test %d: failed to open test database: %v", test, err)
}
Expand Down
151 changes: 96 additions & 55 deletions cmd/geth/dbcmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@ import (
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/log"
"github.com/syndtr/goleveldb/leveldb/opt"
"gopkg.in/urfave/cli.v1"
)

Expand Down Expand Up @@ -65,43 +63,98 @@ Remove blockchain and state databases`,
Action: utils.MigrateFlags(inspect),
Name: "inspect",
ArgsUsage: "<prefix> <start>",

Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Usage: "Inspect the storage size for each type of data in the database",
Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
}
dbStatCmd = cli.Command{
Action: dbStats,
Action: utils.MigrateFlags(dbStats),
Name: "stats",
Usage: "Print leveldb statistics",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
}
dbCompactCmd = cli.Command{
Action: dbCompact,
Action: utils.MigrateFlags(dbCompact),
Name: "compact",
Usage: "Compact leveldb database. WARNING: May take a very long time",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
},
Description: `This command performs a database compaction.
WARNING: This operation may take a very long time to finish, and may cause database
corruption if it is aborted during execution'!`,
}
dbGetCmd = cli.Command{
Action: dbGet,
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Action: utils.MigrateFlags(dbGet),
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: "This command looks up the specified database key from the database.",
}
dbDeleteCmd = cli.Command{
Action: dbDelete,
Action: utils.MigrateFlags(dbDelete),
Name: "delete",
Usage: "Delete a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: `This command deletes the specified database key from the database.
WARNING: This is a low-level operation which may cause database corruption!`,
}
dbPutCmd = cli.Command{
Action: dbPut,
Action: utils.MigrateFlags(dbPut),
Name: "put",
Usage: "Set the value of a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key> <hex-encoded value>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: `This command sets a given database key to the given value.
WARNING: This is a low-level operation which may cause database corruption!`,
}
Expand Down Expand Up @@ -192,10 +245,10 @@ func inspect(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

_, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

return rawdb.InspectDatabase(chainDb, prefix, start)
return rawdb.InspectDatabase(db, prefix, start)
}

func showLeveldbStats(db ethdb.Stater) {
Expand All @@ -214,48 +267,32 @@ func showLeveldbStats(db ethdb.Stater) {
func dbStats(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.ReadOnly = true
})
if err != nil {
return err
}

db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

showLeveldbStats(db)
err = db.Close()
if err != nil {
log.Info("Close err", "error", err)
}
return nil
}

func dbCompact(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles()
options.BlockCacheCapacity = cache / 2 * opt.MiB
options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
})
if err != nil {
return err
}

db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()

log.Info("Stats before compaction")
showLeveldbStats(db)

log.Info("Triggering compaction")
err = db.Compact(nil, nil)
if err != nil {
if err := db.Compact(nil, nil); err != nil {
log.Info("Compact err", "error", err)
return err
}
log.Info("Stats after compaction")
showLeveldbStats(db)
log.Info("Closing db")
err = db.Close()
if err != nil {
log.Info("Close err", "error", err)
}
log.Info("Exiting")
return err
return nil
}

// dbGet shows the value of a given database key
Expand All @@ -265,14 +302,10 @@ func dbGet(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.ReadOnly = true
})
if err != nil {
return err
}

db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

key, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the key", "error", err)
Expand All @@ -283,7 +316,7 @@ func dbGet(ctx *cli.Context) error {
log.Info("Get operation failed", "error", err)
return err
}
fmt.Printf("key %#x:\n\t%#x\n", key, data)
fmt.Printf("key %#x: %#x\n", key, data)
return nil
}

Expand All @@ -294,13 +327,19 @@ func dbDelete(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)

db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()

key, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the key", "error", err)
return err
}
data, err := db.Get(key)
if err == nil {
fmt.Printf("Previous value: %#x\n", data)
}
if err = db.Delete(key); err != nil {
log.Info("Delete operation returned an error", "error", err)
return err
Expand All @@ -315,8 +354,10 @@ func dbPut(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)

db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()

var (
key []byte
value []byte
Expand All @@ -335,7 +376,7 @@ func dbPut(ctx *cli.Context) error {
}
data, err = db.Get(key)
if err == nil {
fmt.Printf("Previous value:\n%#x\n", data)
fmt.Printf("Previous value: %#x\n", data)
}
return db.Put(key, value)
}
Loading

0 comments on commit 9a362e9

Please sign in to comment.