Split header prefix bucket to headers, td and canonical (#1556)

* split headers prefix * migration * fix downloader bug * test for migration * fix lint * uncomment t.Prallel * fix postprocessing test
erigontech · Mar 19, 2021 · c03da8f · c03da8f
1 parent c594f83
commit c03da8f
Show file tree

Hide file tree

Showing 29 changed files with 401 additions and 202 deletions.
diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
@@ -1574,16 +1574,13 @@ func mint(chaindata string, block uint64) error {
 	blockEncoded := dbutils.EncodeBlockNumber(block)
 	canonical := make(map[common.Hash]struct{})
 	if err1 := db.KV().View(context.Background(), func(tx ethdb.Tx) error {
-		c := tx.Cursor(dbutils.HeaderPrefix)
+		c := tx.Cursor(dbutils.HeaderCanonicalBucket)
 		// This is a mapping of contractAddress + incarnation => CodeHash
 		for k, v, err := c.Seek(blockEncoded); k != nil; k, v, err = c.Next() {
 			if err != nil {
 				return err
 			}
 			// Skip non relevant records
-			if !dbutils.CheckCanonicalKey(k) {
-				continue
-			}
 			canonical[common.BytesToHash(v)] = struct{}{}
 			if len(canonical)%100_000 == 0 {
 				log.Info("Read canonical hashes", "count", len(canonical))

diff --git a/cmd/integration/commands/snapshot_check.go b/cmd/integration/commands/snapshot_check.go
@@ -82,7 +82,7 @@ var cmdSnapshotCheck = &cobra.Command{
 
 		kv := ethdb.NewSnapshot2KV().
 			DB(tmpDb).
-			SnapshotDB([]string{dbutils.HeaderPrefix, dbutils.BlockBodyPrefix, dbutils.Senders, dbutils.HeadBlockKey, dbutils.HeaderNumberPrefix}, mainDB.KV()).
+			SnapshotDB([]string{dbutils.HeadersBucket, dbutils.HeaderCanonicalBucket, dbutils.HeaderTDBucket, dbutils.BlockBodyPrefix, dbutils.Senders, dbutils.HeadBlockKey, dbutils.HeaderNumberBucket}, mainDB.KV()).
 			SnapshotDB([]string{dbutils.PlainStateBucket, dbutils.CodeBucket, dbutils.PlainContractCodeBucket}, stateSnapshot).
 			MustOpen()
 

diff --git a/cmd/pics/state.go b/cmd/pics/state.go
@@ -120,9 +120,11 @@ var bucketLabels = map[string]string{
 	dbutils.Log:                         "Event Logs",
 	dbutils.AccountsHistoryBucket:       "History Of Accounts",
 	dbutils.StorageHistoryBucket:        "History Of Storage",
-	dbutils.HeaderPrefix:                "Headers",
+	dbutils.HeadersBucket:               "Headers",
+	dbutils.HeaderCanonicalBucket:       "Canonical headers",
+	dbutils.HeaderTDBucket:              "Headers TD",
 	dbutils.BlockBodyPrefix:             "Block Bodies",
-	dbutils.HeaderNumberPrefix:          "Header Numbers",
+	dbutils.HeaderNumberBucket:          "Header Numbers",
 	dbutils.TxLookupPrefix:              "Transaction Index",
 	dbutils.CodeBucket:                  "Code Of Contracts",
 	dbutils.Senders:                     "Senders",

diff --git a/cmd/snapshots/generator/commands/copy_from_state.go b/cmd/snapshots/generator/commands/copy_from_state.go
@@ -3,13 +3,14 @@ package commands
 import (
 	"context"
 	"fmt"
+	"os"
+	"time"
+
 	"github.com/ledgerwatch/turbo-geth/common/dbutils"
 	"github.com/ledgerwatch/turbo-geth/ethdb"
 	"github.com/ledgerwatch/turbo-geth/log"
 	"github.com/ledgerwatch/turbo-geth/turbo/snapshotsync"
 	"github.com/spf13/cobra"
-	"os"
-	"time"
 )
 
 func init() {

diff --git a/cmd/snapshots/generator/commands/generate_header_snapshot.go b/cmd/snapshots/generator/commands/generate_header_snapshot.go
@@ -61,8 +61,7 @@ func HeaderSnapshot(ctx context.Context, dbPath, snapshotPath string, toBlock ui
 	}
 	snKV := ethdb.NewLMDB().WithBucketsConfig(func(defaultBuckets dbutils.BucketsCfg) dbutils.BucketsCfg {
 		return dbutils.BucketsCfg{
-			dbutils.HeaderPrefix:              dbutils.BucketConfigItem{},
-			dbutils.HeadersSnapshotInfoBucket: dbutils.BucketConfigItem{},
+			dbutils.HeadersBucket: dbutils.BucketConfigItem{},
 		}
 	}).Path(snapshotPath).MustOpen()
 
@@ -87,7 +86,7 @@ func HeaderSnapshot(ctx context.Context, dbPath, snapshotPath string, toBlock ui
 		if len(header) == 0 {
 			return fmt.Errorf("empty header: %v", i)
 		}
-		tuples = append(tuples, []byte(dbutils.HeaderPrefix), dbutils.HeaderKey(i, hash), header)
+		tuples = append(tuples, []byte(dbutils.HeadersBucket), dbutils.HeaderKey(i, hash), header)
 		if len(tuples) >= chunkFile {
 			log.Info("Committed", "block", i)
 			_, err = snDB.MultiPut(tuples...)

diff --git a/cmd/snapshots/generator/commands/generate_state_snapshot.go b/cmd/snapshots/generator/commands/generate_state_snapshot.go
@@ -4,6 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"time"
+
 	"github.com/ledgerwatch/turbo-geth/common"
 	"github.com/ledgerwatch/turbo-geth/common/dbutils"
 	"github.com/ledgerwatch/turbo-geth/core/state"
@@ -12,8 +15,6 @@ import (
 	"github.com/ledgerwatch/turbo-geth/turbo/snapshotsync"
 	"github.com/ledgerwatch/turbo-geth/turbo/trie"
 	"github.com/spf13/cobra"
-	"os"
-	"time"
 )
 
 func init() {

diff --git a/cmd/snapshots/generator/commands/verify_state_snapshot.go b/cmd/snapshots/generator/commands/verify_state_snapshot.go
@@ -3,16 +3,17 @@ package commands
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
 	"github.com/ledgerwatch/lmdb-go/lmdb"
 	"github.com/ledgerwatch/turbo-geth/common/dbutils"
 	"github.com/ledgerwatch/turbo-geth/core/rawdb"
 	"github.com/ledgerwatch/turbo-geth/eth/stagedsync"
 	"github.com/ledgerwatch/turbo-geth/ethdb"
 	"github.com/ledgerwatch/turbo-geth/turbo/snapshotsync"
 	"github.com/spf13/cobra"
-	"io/ioutil"
-	"os"
-	"time"
 )
 
 func init() {

diff --git a/cmd/state/generate/regenerate_tx_lookup.go b/cmd/state/generate/regenerate_tx_lookup.go
@@ -33,7 +33,7 @@ func RegenerateTxLookup(chaindata string) error {
 		log.Error("Cant get last executed block", "err", err)
 	}
 	log.Info("TxLookup generation started", "start time", startTime)
-	err = stagedsync.TxLookupTransform("txlookup", db, dbutils.HeaderHashKey(0), dbutils.HeaderHashKey(lastExecutedBlock), quitCh, os.TempDir())
+	err = stagedsync.TxLookupTransform("txlookup", db, dbutils.EncodeBlockNumber(0), dbutils.EncodeBlockNumber(lastExecutedBlock+1), quitCh, os.TempDir())
 	if err != nil {
 		return err
 	}

diff --git a/cmd/state/stateless/state.go b/cmd/state/stateless/state.go
@@ -491,11 +491,8 @@ func (r *GasLimitReporter) GasLimits(ctx context.Context) {
 	var blockNum uint64 = 0
 
 	if err := r.remoteDB.View(ctx, func(tx ethdb.Tx) error {
-		c := tx.Cursor(dbutils.HeaderPrefix)
+		c := tx.Cursor(dbutils.HeadersBucket)
 		if err := ethdb.ForEach(c, func(k, v []byte) (bool, error) {
-			if len(k) != 40 {
-				return true, nil
-			}
 			header := new(types.Header)
 			if err := rlp.Decode(bytes.NewReader(v), header); err != nil {
 				return false, err

diff --git a/cmd/state/verify/verify_headers_snapshot.go b/cmd/state/verify/verify_headers_snapshot.go
@@ -15,13 +15,13 @@ import (
 func HeadersSnapshot(snapshotPath string) error {
 	snKV := ethdb.NewLMDB().Path(snapshotPath).Flags(func(flags uint) uint { return flags | lmdb.Readonly }).WithBucketsConfig(func(defaultBuckets dbutils.BucketsCfg) dbutils.BucketsCfg {
 		return dbutils.BucketsCfg{
-			dbutils.HeaderPrefix:              dbutils.BucketConfigItem{},
+			dbutils.HeadersBucket:             dbutils.BucketConfigItem{},
 			dbutils.HeadersSnapshotInfoBucket: dbutils.BucketConfigItem{},
 		}
 	}).MustOpen()
 	var prevHeader *types.Header
 	err := snKV.View(context.Background(), func(tx ethdb.Tx) error {
-		c := tx.Cursor(dbutils.HeaderPrefix)
+		c := tx.Cursor(dbutils.HeadersBucket)
 		k, v, innerErr := c.First()
 		for {
 			if len(k) == 0 && len(v) == 0 {

diff --git a/common/dbutils/bucket.go b/common/dbutils/bucket.go
@@ -147,10 +147,12 @@ var (
 	DatabaseVerisionKey = "DatabaseVersion"
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
-	HeaderPrefix       = "h"         // block_num_u64 + hash -> header
-	HeaderTDSuffix     = []byte("t") // block_num_u64 + hash + headerTDSuffix -> td
-	HeaderHashSuffix   = []byte("n") // block_num_u64 + headerHashSuffix -> hash
-	HeaderNumberPrefix = "H"         // headerNumberPrefix + hash -> num (uint64 big endian)
+	HeaderPrefixOld    = "h" // block_num_u64 + hash -> header
+	HeaderNumberBucket = "H" // headerNumberPrefix + hash -> num (uint64 big endian)
+
+	HeaderCanonicalBucket = "canonical_headers" // block_num_u64 -> header hash
+	HeadersBucket         = "headers"           // block_num_u64 + hash -> header
+	HeaderTDBucket        = "header_to_td"      // block_num_u64 + hash + headerTDSuffix -> td
 
 	BlockBodyPrefix     = "b"      // block_num_u64 + hash -> block body
 	EthTx               = "eth_tx" // tbl_sequence_u64 -> rlp(tx)
@@ -254,8 +256,7 @@ var Buckets = []string{
 	CodeBucket,
 	ContractCodeBucket,
 	DatabaseVerisionKey,
-	HeaderPrefix,
-	HeaderNumberPrefix,
+	HeaderNumberBucket,
 	BlockBodyPrefix,
 	BlockReceiptsPrefix,
 	TxLookupPrefix,
@@ -294,6 +295,10 @@ var Buckets = []string{
 	HashedAccountsBucket,
 	HashedStorageBucket,
 	IntermediateTrieHashBucketOld2,
+
+	HeaderCanonicalBucket,
+	HeadersBucket,
+	HeaderTDBucket,
 }
 
 // DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration
@@ -303,6 +308,7 @@ var DeprecatedBuckets = []string{
 	CurrentStateBucketOld1,
 	PlainStateBucketOld1,
 	IntermediateTrieHashBucketOld1,
+	HeaderPrefixOld,
 }
 
 type CustomComparator string

diff --git a/common/dbutils/composite_keys.go b/common/dbutils/composite_keys.go
@@ -1,7 +1,6 @@
 package dbutils
 
 import (
-	"bytes"
 	"encoding/binary"
 
 	"github.com/ledgerwatch/turbo-geth/common"
@@ -19,38 +18,6 @@ func HeaderKey(number uint64, hash common.Hash) []byte {
 	return append(EncodeBlockNumber(number), hash.Bytes()...)
 }
 
-func IsHeaderKey(k []byte) bool {
-	l := common.BlockNumberLength + common.HashLength
-	if len(k) != l {
-		return false
-	}
-
-	return !IsHeaderHashKey(k) && !IsHeaderTDKey(k)
-}
-
-// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix
-func HeaderTDKey(number uint64, hash common.Hash) []byte {
-	return append(HeaderKey(number, hash), HeaderTDSuffix...)
-}
-
-func IsHeaderTDKey(k []byte) bool {
-	l := common.BlockNumberLength + common.HashLength + 1
-	return len(k) == l && bytes.Equal(k[l-1:], HeaderTDSuffix)
-}
-
-// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix
-func HeaderHashKey(number uint64) []byte {
-	return append(EncodeBlockNumber(number), HeaderHashSuffix...)
-}
-
-func CheckCanonicalKey(k []byte) bool {
-	return len(k) == 8+len(HeaderHashSuffix) && bytes.Equal(k[8:], HeaderHashSuffix)
-}
-
-func IsHeaderHashKey(k []byte) bool {
-	l := common.BlockNumberLength + 1
-	return len(k) == l && bytes.Equal(k[l-1:], HeaderHashSuffix)
-}
 
 // blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash
 func BlockBodyKey(number uint64, hash common.Hash) []byte {

diff --git a/common/dbutils/composite_keys_test.go b/common/dbutils/composite_keys_test.go
@@ -7,41 +7,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestHeaderTypeDetection(t *testing.T) {
-
-	// good input
-	headerHashKey := common.Hex2Bytes("00000000000000006e")
-	assert.False(t, IsHeaderKey(headerHashKey))
-	assert.False(t, IsHeaderTDKey(headerHashKey))
-	assert.True(t, IsHeaderHashKey(headerHashKey))
-
-	headerKey := common.Hex2Bytes("0000000000004321ed7240d411782ae438adfd85f7edad373cea722318c6e7f5f5b30f9abc9b36fd")
-	assert.True(t, IsHeaderKey(headerKey))
-	assert.False(t, IsHeaderTDKey(headerKey))
-	assert.False(t, IsHeaderHashKey(headerKey))
-
-	headerTdKey := common.Hex2Bytes("0000000000004321ed7240d411782ae438adfd85f7edad373cea722318c6e7f5f5b30f9abc9b36fd74")
-	assert.False(t, IsHeaderKey(headerTdKey))
-	assert.True(t, IsHeaderTDKey(headerTdKey))
-	assert.False(t, IsHeaderHashKey(headerTdKey))
-
-	// bad input
-	emptyKey := common.Hex2Bytes("")
-	assert.False(t, IsHeaderKey(emptyKey))
-	assert.False(t, IsHeaderTDKey(emptyKey))
-	assert.False(t, IsHeaderHashKey(emptyKey))
-
-	tooLongKey := common.Hex2Bytes("0000000000004321ed7240d411782ae438adfd85f7edad373cea722318c6e7f5f5b30f9abc9b36fd0000000000004321ed7240d411782ae438adfd85f7edad373cea722318c6e7f5f5b30f9abc9b36fd0000000000004321ed7240d411782ae438adfd85f7edad373cea722318c6e7f5f5b30f9abc9b36fd0000000000004321ed7240d411782ae438adfd85f7edad373cea722318c6e7f5f5b30f9abc9b36fd")
-	assert.False(t, IsHeaderKey(tooLongKey))
-	assert.False(t, IsHeaderTDKey(tooLongKey))
-	assert.False(t, IsHeaderHashKey(tooLongKey))
-
-	notRelatedInput := common.Hex2Bytes("alex")
-	assert.False(t, IsHeaderKey(notRelatedInput))
-	assert.False(t, IsHeaderTDKey(notRelatedInput))
-	assert.False(t, IsHeaderHashKey(notRelatedInput))
 
-}
 
 func TestPlainParseStoragePrefix(t *testing.T) {
 	expectedAddr := common.HexToAddress("0x5A0b54D5dc17e0AadC383d2db43B0a0D3E029c4c")