Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

all: add read-only option to database #22407

Merged
merged 13 commits into from
Mar 22, 2021
Merged

all: add read-only option to database #22407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
d1b3257
all: add read-only option to database
rjl493456442 Mar 2, 2021
65f4d8f
all: fixes tests
rjl493456442 Mar 2, 2021
f9693e4
cmd/geth: migrate flags
rjl493456442 Mar 2, 2021
3b4b100
cmd/geth: fix the compact
rjl493456442 Mar 2, 2021
9208d5b
cmd/geth: fix the format
rjl493456442 Mar 2, 2021
ac6ab0d
cmd/geth: fix log
rjl493456442 Mar 9, 2021
5d1137b
cmd: add chain-readonly
rjl493456442 Mar 19, 2021
e3cb0cc
core: add readonly notion to freezer
rjl493456442 Mar 19, 2021
a9b4752
core/rawdb: add log
rjl493456442 Mar 19, 2021
987adaa
core/rawdb: fix freezer close
rjl493456442 Mar 19, 2021
c686af6
cmd: fix
rjl493456442 Mar 22, 2021
eb9747a
cmd, core: construct db
rjl493456442 Mar 22, 2021
1b5d4c8
core: update tests
rjl493456442 Mar 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 20 additions & 13 deletions cmd/geth/chaincmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
Expand Down Expand Up @@ -191,7 +192,7 @@ func initGenesis(ctx *cli.Context) error {
defer stack.Close()

for _, name := range []string{"chaindata", "lightchaindata"} {
chaindb, err := stack.OpenDatabase(name, 0, 0, "")
chaindb, err := stack.OpenDatabase(name, 0, 0, "", false)
if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}
Expand Down Expand Up @@ -229,7 +230,7 @@ func importChain(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, db := utils.MakeChain(ctx, stack, false)
chain, db := utils.MakeChain(ctx, stack)
defer db.Close()

// Start periodically gathering memory profiles
Expand Down Expand Up @@ -304,7 +305,7 @@ func exportChain(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, _ := utils.MakeChain(ctx, stack, true)
chain, _ := utils.MakeChain(ctx, stack)
start := time.Now()

var err error
Expand Down Expand Up @@ -340,7 +341,7 @@ func importPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, false)
start := time.Now()

if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
Expand All @@ -359,7 +360,7 @@ func exportPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, true)
start := time.Now()

if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
Expand All @@ -373,21 +374,27 @@ func dump(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
db := utils.MakeChainDatabase(ctx, stack, true)
for _, arg := range ctx.Args() {
var block *types.Block
var header *types.Header
if hashish(arg) {
block = chain.GetBlockByHash(common.HexToHash(arg))
hash := common.HexToHash(arg)
number := rawdb.ReadHeaderNumber(db, hash)
if number != nil {
header = rawdb.ReadHeader(db, hash, *number)
}
} else {
num, _ := strconv.Atoi(arg)
block = chain.GetBlockByNumber(uint64(num))
number, _ := strconv.Atoi(arg)
hash := rawdb.ReadCanonicalHash(db, uint64(number))
if hash != (common.Hash{}) {
header = rawdb.ReadHeader(db, hash, uint64(number))
}
}
if block == nil {
if header == nil {
fmt.Println("{}")
utils.Fatalf("block not found")
} else {
state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil)
state, err := state.New(header.Root, state.NewDatabase(db), nil)
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/geth/dao_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
}
// Retrieve the DAO config flag from the database
path := filepath.Join(datadir, "geth", "chaindata")
db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "")
db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false)
if err != nil {
t.Fatalf("test %d: failed to open test database: %v", test, err)
}
Expand Down
151 changes: 96 additions & 55 deletions cmd/geth/dbcmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@ import (
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/log"
"github.com/syndtr/goleveldb/leveldb/opt"
"gopkg.in/urfave/cli.v1"
)

Expand Down Expand Up @@ -65,43 +63,98 @@ Remove blockchain and state databases`,
Action: utils.MigrateFlags(inspect),
Name: "inspect",
ArgsUsage: "<prefix> <start>",

Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Usage: "Inspect the storage size for each type of data in the database",
Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
}
dbStatCmd = cli.Command{
Action: dbStats,
Action: utils.MigrateFlags(dbStats),
Name: "stats",
Usage: "Print leveldb statistics",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
}
dbCompactCmd = cli.Command{
Action: dbCompact,
Action: utils.MigrateFlags(dbCompact),
Name: "compact",
Usage: "Compact leveldb database. WARNING: May take a very long time",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
},
Description: `This command performs a database compaction.
WARNING: This operation may take a very long time to finish, and may cause database
corruption if it is aborted during execution'!`,
}
dbGetCmd = cli.Command{
Action: dbGet,
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Action: utils.MigrateFlags(dbGet),
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: "This command looks up the specified database key from the database.",
}
dbDeleteCmd = cli.Command{
Action: dbDelete,
Action: utils.MigrateFlags(dbDelete),
Name: "delete",
Usage: "Delete a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: `This command deletes the specified database key from the database.
WARNING: This is a low-level operation which may cause database corruption!`,
}
dbPutCmd = cli.Command{
Action: dbPut,
Action: utils.MigrateFlags(dbPut),
Name: "put",
Usage: "Set the value of a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key> <hex-encoded value>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: `This command sets a given database key to the given value.
WARNING: This is a low-level operation which may cause database corruption!`,
}
Expand Down Expand Up @@ -192,10 +245,10 @@ func inspect(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

_, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

return rawdb.InspectDatabase(chainDb, prefix, start)
return rawdb.InspectDatabase(db, prefix, start)
}

func showLeveldbStats(db ethdb.Stater) {
Expand All @@ -214,48 +267,32 @@ func showLeveldbStats(db ethdb.Stater) {
func dbStats(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.ReadOnly = true
})
if err != nil {
return err
}

db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

showLeveldbStats(db)
err = db.Close()
if err != nil {
log.Info("Close err", "error", err)
}
return nil
}

func dbCompact(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles()
options.BlockCacheCapacity = cache / 2 * opt.MiB
options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
})
if err != nil {
return err
}

db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()

log.Info("Stats before compaction")
showLeveldbStats(db)

log.Info("Triggering compaction")
err = db.Compact(nil, nil)
if err != nil {
if err := db.Compact(nil, nil); err != nil {
log.Info("Compact err", "error", err)
return err
}
log.Info("Stats after compaction")
showLeveldbStats(db)
log.Info("Closing db")
err = db.Close()
if err != nil {
log.Info("Close err", "error", err)
}
log.Info("Exiting")
return err
return nil
}

// dbGet shows the value of a given database key
Expand All @@ -265,14 +302,10 @@ func dbGet(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.ReadOnly = true
})
if err != nil {
return err
}

db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

key, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the key", "error", err)
Expand All @@ -283,7 +316,7 @@ func dbGet(ctx *cli.Context) error {
log.Info("Get operation failed", "error", err)
return err
}
fmt.Printf("key %#x:\n\t%#x\n", key, data)
fmt.Printf("key %#x: %#x\n", key, data)
return nil
}

Expand All @@ -294,13 +327,19 @@ func dbDelete(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)

db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()

key, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the key", "error", err)
return err
}
data, err := db.Get(key)
if err == nil {
fmt.Printf("Previous value: %#x\n", data)
}
if err = db.Delete(key); err != nil {
log.Info("Delete operation returned an error", "error", err)
return err
Expand All @@ -315,8 +354,10 @@ func dbPut(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)

db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()

var (
key []byte
value []byte
Expand All @@ -335,7 +376,7 @@ func dbPut(ctx *cli.Context) error {
}
data, err = db.Get(key)
if err == nil {
fmt.Printf("Previous value:\n%#x\n", data)
fmt.Printf("Previous value: %#x\n", data)
}
return db.Put(key, value)
}
Loading