blockchain: Migrate to new block index and use it.

This adds code to migrate the existing block index in ffldb to the new
format managed by the blockchain package and updates the code to use the
new infrastructure.

blockchain/accept.go

@@ -142,6 +142,21 @@ func (b *BlockChain) maybeAcceptBlock(block *dcrutil.Block, flags BehaviorFlags)
 		return false, err
 	}
 
+	// Prune stake nodes which are no longer needed before creating a new
+	// node.
+	if !dryRun {
+		b.pruner.pruneChainIfNeeded()
+	}
+
+	// Create a new block node for the block and add it to the block index.
+	// The block could either be on a side chain or the main chain, but it
+	// starts off as a side chain regardless.
+	blockHeader := &block.MsgBlock().Header
+	newNode := newBlockNode(blockHeader, prevNode)
+	newNode.populateTicketInfo(stake.FindSpentTicketsInBlock(block.MsgBlock()))
+	newNode.status = statusDataStored
+	b.index.AddNode(newNode)
+
 	// Insert the block into the database if it's not already there.  Even
 	// though it is possible the block will ultimately fail to connect, it
 	// has already passed all proof-of-work and validity tests which means
@@ -151,27 +166,26 @@ func (b *BlockChain) maybeAcceptBlock(block *dcrutil.Block, flags BehaviorFlags)
 	// expensive connection logic.  It also has some other nice properties
 	// such as making blocks that never become part of the main chain or
 	// blocks that fail to connect available for further analysis.
+	//
+	// Also, store the associated block index entry when not running in dry
+	// run mode.
 	err = b.db.Update(func(dbTx database.Tx) error {
-		return dbMaybeStoreBlock(dbTx, block)
+		if err := dbMaybeStoreBlock(dbTx, block); err != nil {
+			return err
+		}
+
+		if !dryRun {
+			if err := dbPutBlockNode(dbTx, newNode); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	})
 	if err != nil {
 		return false, err
 	}
 
-	// Prune stake nodes which are no longer needed before creating a new
-	// node.
-	if !dryRun {
-		b.pruner.pruneChainIfNeeded()
-	}
-
-	// Create a new block node for the block and add it to the in-memory
-	// block chain (could be either a side chain or the main chain).
-	blockHeader := &block.MsgBlock().Header
-	newNode := newBlockNode(blockHeader, prevNode)
-	newNode.populateTicketInfo(stake.FindSpentTicketsInBlock(block.MsgBlock()))
-	newNode.status = statusDataStored
-	b.index.AddNode(newNode)
-
 	// Remove the node from the block index and disconnect it from the
 	// parent node when running in dry run mode.
 	if dryRun {

blockchain/blockindex.go

@@ -314,17 +314,18 @@ func (bi *blockIndex) HaveBlock(hash *chainhash.Hash) bool {
 // This function MUST be called with the block index lock held (for writes).
 // The database transaction may be read-only.
 func (bi *blockIndex) loadBlockNode(dbTx database.Tx, hash *chainhash.Hash) (*blockNode, error) {
-	// Load the block from the db.
-	block, err := dbFetchBlockByHash(dbTx, hash)
+	// Try to look up the height for passed block hash in the main chain.
+	height, err := dbFetchHeightByHash(dbTx, hash)
 	if err != nil {
 		return nil, err
 	}
 
-	// Create the new block node for the block.
-	blockHeader := block.MsgBlock().Header
-	node := newBlockNode(&blockHeader, nil)
-	node.populateTicketInfo(stake.FindSpentTicketsInBlock(block.MsgBlock()))
-	node.status = statusDataStored | statusValid
+	// Load the block node for the provided hash and height from the
+	// database.
+	node, err := dbFetchBlockNode(dbTx, hash, uint32(height))
+	if err != nil {
+		return nil, err
+	}
 	node.inMainChain = true
 
 	// Add the node to the chain.
@@ -333,7 +334,7 @@ func (bi *blockIndex) loadBlockNode(dbTx database.Tx, hash *chainhash.Hash) (*bl
 	//  2) This node is the parent of one or more nodes
 	//  3) Neither 1 or 2 is true which implies it's an orphan block and
 	//     therefore is an error to insert into the chain
-	prevHash := &blockHeader.PrevBlock
+	prevHash := &node.parentHash
 	if parentNode, ok := bi.index[*prevHash]; ok {
 		// Case 1 -- This node is a child of an existing block node.
 		// Update the node's work sum with the sum of the parent node's

blockchain/chain.go

@@ -745,7 +745,7 @@ func (b *BlockChain) getReorganizeNodes(node *blockNode) (*list.List, *list.List
 		return detachNodes, attachNodes, nil
 	}
 
-	// Don't allow a reorganize to a descendant of an known invalid block.
+	// Don't allow a reorganize to a descendant of a known invalid block.
 	if b.index.NodeStatus(node.parent).KnownInvalid() {
 		b.index.SetStatusFlags(node, statusInvalidAncestor)
 		return detachNodes, attachNodes, nil
@@ -871,6 +871,16 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block,
 			return err
 		}
 
+		// Add the block to the block index.  Ultimately the block index
+		// should track modified nodes and persist all of them prior
+		// this point as opposed to unconditionally peristing the node
+		// again.  However, this is needed for now in lieu of that to
+		// ensure the updated status is written to the database.
+		err = dbPutBlockNode(dbTx, node)
+		if err != nil {
+			return err
+		}
+
 		// Add the block hash and height to the main chain index.
 		err = dbPutMainChainIndex(dbTx, block.Hash(), node.height)
 		if err != nil {
@@ -1994,8 +2004,9 @@ func New(config *Config) (*BlockChain, error) {
 	b.subsidyCache = NewSubsidyCache(b.bestNode.height, b.chainParams)
 	b.pruner = newChainPruner(&b)
 
-	log.Infof("Blockchain database version %v loaded",
-		b.dbInfo.version)
+	log.Infof("Blockchain database version info: chain: %d, compression: "+
+		"%d, block index: %d", b.dbInfo.version, b.dbInfo.compVer,
+		b.dbInfo.bidxVer)
 
 	log.Infof("Chain state: height %d, hash %v, total transactions %d, "+
 		"work %v, stake version %v", b.bestNode.height, b.bestNode.hash,

blockchain/chainio.go

@@ -31,7 +31,11 @@ var (
 const (
 	// currentDatabaseVersion indicates what the current database
 	// version is.
-	currentDatabaseVersion = 2
+	currentDatabaseVersion = 3
+
+	// currentBlockIndexVersion indicates what the current block index
+	// database version.
+	currentBlockIndexVersion = 2
 
 	// blockHdrSize is the size of a block header.  This is simply the
 	// constant from wire and is only provided here for convenience since
@@ -1319,13 +1323,15 @@ func dbFetchHashByHeight(dbTx database.Tx, height int64) (*chainhash.Hash, error
 //   Key        Value    Size      Description
 //   version    uint32   4 bytes   The version of the database
 //   compver    uint32   4 bytes   The script compression version of the database
+//   bidxver    uint32   4 bytes   The block index version of the database
 //   created    uint64   8 bytes   The date of the creation of the database
 // -----------------------------------------------------------------------------
 
 // databaseInfo is the structure for a database.
 type databaseInfo struct {
 	version uint32
 	compVer uint32
+	bidxVer uint32
 	created time.Time
 }
 
@@ -1364,6 +1370,13 @@ func dbPutDatabaseInfo(dbTx database.Tx, dbi *databaseInfo) error {
 		return err
 	}
 
+	// Store the block index version.
+	err = bucket.Put(dbnamespace.BCDBInfoBlockIndexVersionKeyName,
+		uint32Bytes(dbi.bidxVer))
+	if err != nil {
+		return err
+	}
+
 	// Store the database creation date.
 	return bucket.Put(dbnamespace.BCDBInfoCreatedKeyName,
 		uint64Bytes(uint64(dbi.created.Unix())))
@@ -1394,6 +1407,13 @@ func dbFetchDatabaseInfo(dbTx database.Tx) (*databaseInfo, error) {
 		compVer = dbnamespace.ByteOrder.Uint32(compVerBytes)
 	}
 
+	// Load the database block index version.
+	var bidxVer uint32
+	bidxVerBytes := bucket.Get(dbnamespace.BCDBInfoBlockIndexVersionKeyName)
+	if bidxVerBytes != nil {
+		bidxVer = dbnamespace.ByteOrder.Uint32(bidxVerBytes)
+	}
+
 	// Load the database creation date.
 	var created time.Time
 	createdBytes := bucket.Get(dbnamespace.BCDBInfoCreatedKeyName)
@@ -1405,6 +1425,7 @@ func dbFetchDatabaseInfo(dbTx database.Tx) (*databaseInfo, error) {
 	return &databaseInfo{
 		version: version,
 		compVer: compVer,
+		bidxVer: bidxVer,
 		created: created,
 	}, nil
 }
@@ -1560,13 +1581,20 @@ func (b *BlockChain) createChainState() error {
 		b.dbInfo = &databaseInfo{
 			version: currentDatabaseVersion,
 			compVer: currentCompressionVersion,
+			bidxVer: currentBlockIndexVersion,
 			created: time.Now(),
 		}
 		err = dbPutDatabaseInfo(dbTx, b.dbInfo)
 		if err != nil {
 			return err
 		}
 
+		// Create the bucket that houses the block index data.
+		_, err = meta.CreateBucket(dbnamespace.BlockIndexBucketName)
+		if err != nil {
+			return err
+		}
+
 		// Create the bucket that houses the chain block hash to height
 		// index.
 		_, err = meta.CreateBucket(dbnamespace.HashIndexBucketName)
@@ -1595,6 +1623,12 @@ func (b *BlockChain) createChainState() error {
 			return err
 		}
 
+		// Add the genesis block to the block index.
+		err = dbPutBlockNode(dbTx, node)
+		if err != nil {
+			return err
+		}
+
 		// Add the genesis block hash to height and height to hash
 		// mappings to the index.
 		err = dbPutMainChainIndex(dbTx, &node.hash, node.height)
@@ -1691,8 +1725,16 @@ func (b *BlockChain) initChainState(interrupt <-chan struct{}) error {
 				"version of the software (%d > %d)",
 				dbInfo.compVer, currentCompressionVersion)
 		}
-		b.dbInfo = dbInfo
 
+		// Don't allow downgrades of the block index.
+		if dbInfo.bidxVer > currentBlockIndexVersion {
+			return fmt.Errorf("the current database block index "+
+				"version is no longer compatible with this "+
+				"version of the software (%d > %d)",
+				dbInfo.bidxVer, currentBlockIndexVersion)
+		}
+
+		b.dbInfo = dbInfo
 		isStateInitialized = true
 		return nil
 	})

blockchain/internal/dbnamespace/dbnamespace.go

@@ -26,6 +26,11 @@ var (
 	// the BCDBInfoBucketName bucket.
 	BCDBInfoCompressionVersionKeyName = []byte("compver")
 
+	// BCDBInfoBlockIndexVersionKeyName is the name of the database key
+	// used to house the database block index version.  It is itself under
+	// the BCDBInfoBucketName bucket.
+	BCDBInfoBlockIndexVersionKeyName = []byte("bidxver")
+
 	// BCDBInfoCreatedKeyName is the name of the database key used to house
 	// date the database was created.  It is itself under the
 	// BCDBInfoBucketName bucket.