keep im state root in db (#3427)

db/migrations/state/0017.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+ALTER TABLE state.receipt
+    ADD COLUMN IF NOT EXISTS im_state_root BYTEA;
+
+UPDATE state.receipt SET im_state_root = post_state WHERE block_num >= (SELECT MIN(block_num) FROM state.l2block WHERE batch_num >= (SELECT from_batch_num FROM state.fork_id WHERE fork_id = 7));
+
+-- +migrate Down
+ALTER TABLE state.receipt
+    DROP COLUMN IF EXISTS im_state_root;

proto/src/proto/executor/v1/executor.proto

@@ -872,4 +872,6 @@ enum ExecutorError {
     EXECUTOR_ERROR_INVALID_DATA_STREAM = 115;
     // EXECUTOR_ERROR_INVALID_UPDATE_MERKLE_TREE indicates that the provided update merkle tree is invalid, e.g. because the executor is configured not to write to database
     EXECUTOR_ERROR_INVALID_UPDATE_MERKLE_TREE = 116;
+    // EXECUTOR_ERROR_SM_MAIN_INVALID_TX_STATUS_ERROR indicates that a TX has an invalid status-error combination
+    EXECUTOR_ERROR_SM_MAIN_INVALID_TX_STATUS_ERROR = 117;
 }

state/datastream.go

@@ -118,6 +118,7 @@ func (b DSL2BlockStart) Decode(data []byte) DSL2BlockStart {
 // DSL2Transaction represents a data stream L2 transaction
 type DSL2Transaction struct {
 	L2BlockNumber               uint64      // Not included in the encoded data
+	ImStateRoot                 common.Hash // Not included in the encoded data
 	EffectiveGasPricePercentage uint8       // 1 byte
 	IsValid                     uint8       // 1 byte
 	StateRoot                   common.Hash // 32 bytes
@@ -553,6 +554,9 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St
 					}
 
 					for _, tx := range l2Block.Txs {
+						// < ETROG => IM State root is retrieved from the system SC (using cache is available)
+						// = ETROG => IM State root is retrieved from the receipt.post_state => Do nothing
+						// > ETROG => IM State root is retrieved from the receipt.im_state_root
 						if l2Block.ForkID < FORKID_ETROG {
 							// Populate intermediate state root with information from the system SC (or cache if available)
 							if imStateRoots == nil || (*imStateRoots)[blockStart.L2BlockNumber] == nil {
@@ -565,6 +569,8 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St
 							} else {
 								tx.StateRoot = common.BytesToHash((*imStateRoots)[blockStart.L2BlockNumber])
 							}
+						} else if l2Block.ForkID > FORKID_ETROG {
+							tx.StateRoot = tx.ImStateRoot
 						}
 
 						_, err = streamServer.AddStreamEntry(EntryTypeL2Tx, tx.Encode())

state/genesis.go

@@ -204,7 +204,7 @@ func (s *State) SetGenesis(ctx context.Context, block Block, genesis Genesis, m
 	storeTxsEGPData := []StoreTxEGPData{}
 	txsL2Hash := []common.Hash{}
 
-	err = s.AddL2Block(ctx, batch.BatchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, dbTx)
+	err = s.AddL2Block(ctx, batch.BatchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, []common.Hash{}, dbTx)
 	if err != nil {
 		return common.Hash{}, err
 	}

state/helper.go

@@ -279,7 +279,7 @@ func DecodeTx(encodedTx string) (*types.Transaction, error) {
 }
 
 // GenerateReceipt generates a receipt from a processed transaction
-func GenerateReceipt(blockNumber *big.Int, processedTx *ProcessTransactionResponse, txIndex uint, forkID uint64) *types.Receipt {
+func GenerateReceipt(blockNumber *big.Int, processedTx *ProcessTransactionResponse, txIndex uint, forkID uint64) (*types.Receipt, common.Hash) {
 	receipt := &types.Receipt{
 		Type:             uint8(processedTx.Type),
 		BlockNumber:      blockNumber,
@@ -323,7 +323,7 @@ func GenerateReceipt(blockNumber *big.Int, processedTx *ProcessTransactionRespon
 		receipt.Status = uint64(processedTx.Status)
 	}
 
-	return receipt
+	return receipt, processedTx.StateRoot
 }
 
 // IsPreEIP155Tx checks if the tx is a tx that has a chainID as zero and

state/interfaces.go

@@ -72,7 +72,7 @@ type storage interface {
 	GetL2BlockTransactionCountByHash(ctx context.Context, blockHash common.Hash, dbTx pgx.Tx) (uint64, error)
 	GetL2BlockTransactionCountByNumber(ctx context.Context, blockNumber uint64, dbTx pgx.Tx) (uint64, error)
 	GetTransactionEGPLogByHash(ctx context.Context, transactionHash common.Hash, dbTx pgx.Tx) (*EffectiveGasPriceLog, error)
-	AddL2Block(ctx context.Context, batchNumber uint64, l2Block *L2Block, receipts []*types.Receipt, txsL2Hash []common.Hash, txsEGPData []StoreTxEGPData, dbTx pgx.Tx) error
+	AddL2Block(ctx context.Context, batchNumber uint64, l2Block *L2Block, receipts []*types.Receipt, txsL2Hash []common.Hash, txsEGPData []StoreTxEGPData, imStateRoots []common.Hash, dbTx pgx.Tx) error
 	GetLastVirtualizedL2BlockNumber(ctx context.Context, dbTx pgx.Tx) (uint64, error)
 	GetLastConsolidatedL2BlockNumber(ctx context.Context, dbTx pgx.Tx) (uint64, error)
 	GetLastVerifiedL2BlockNumberUntilL1Block(ctx context.Context, l1FinalizedBlockNumber uint64, dbTx pgx.Tx) (uint64, error)
@@ -93,7 +93,7 @@ type storage interface {
 	IsL2BlockConsolidated(ctx context.Context, blockNumber uint64, dbTx pgx.Tx) (bool, error)
 	IsL2BlockVirtualized(ctx context.Context, blockNumber uint64, dbTx pgx.Tx) (bool, error)
 	GetLogs(ctx context.Context, fromBlock uint64, toBlock uint64, addresses []common.Address, topics [][]common.Hash, blockHash *common.Hash, since *time.Time, dbTx pgx.Tx) ([]*types.Log, error)
-	AddReceipt(ctx context.Context, receipt *types.Receipt, dbTx pgx.Tx) error
+	AddReceipt(ctx context.Context, receipt *types.Receipt, imStateRoot common.Hash, dbTx pgx.Tx) error
 	AddLog(ctx context.Context, l *types.Log, dbTx pgx.Tx) error
 	GetExitRootByGlobalExitRoot(ctx context.Context, ger common.Hash, dbTx pgx.Tx) (*GlobalExitRoot, error)
 	AddSequence(ctx context.Context, sequence Sequence, dbTx pgx.Tx) error

state/pgstatestorage/datastream.go

@@ -91,7 +91,7 @@ func scanL2Block(row pgx.Row) (*state.DSL2Block, error) {
 
 // GetDSL2Transactions returns the L2 transactions
 func (p *PostgresStorage) GetDSL2Transactions(ctx context.Context, firstL2Block, lastL2Block uint64, dbTx pgx.Tx) ([]*state.DSL2Transaction, error) {
-	const l2TxSQL = `SELECT l2_block_num, t.effective_percentage, t.encoded, r.post_state
+	const l2TxSQL = `SELECT l2_block_num, t.effective_percentage, t.encoded, r.post_state, r.im_state_root
 					 FROM state.transaction t, state.receipt r
 					 WHERE l2_block_num BETWEEN $1 AND $2 AND r.tx_hash = t.hash
 					 ORDER BY t.l2_block_num ASC, r.tx_index ASC`
@@ -120,11 +120,13 @@ func scanDSL2Transaction(row pgx.Row) (*state.DSL2Transaction, error) {
 	l2Transaction := state.DSL2Transaction{}
 	encoded := []byte{}
 	postState := []byte{}
+	imStateRoot := []byte{}
 	if err := row.Scan(
 		&l2Transaction.L2BlockNumber,
 		&l2Transaction.EffectiveGasPricePercentage,
 		&encoded,
 		&postState,
+		&imStateRoot,
 	); err != nil {
 		return nil, err
 	}
@@ -142,6 +144,7 @@ func scanDSL2Transaction(row pgx.Row) (*state.DSL2Transaction, error) {
 	l2Transaction.EncodedLength = uint32(len(l2Transaction.Encoded))
 	l2Transaction.IsValid = 1
 	l2Transaction.StateRoot = common.BytesToHash(postState)
+	l2Transaction.ImStateRoot = common.BytesToHash(imStateRoot)
 	return &l2Transaction, nil
 }
 

state/pgstatestorage/l2block.go

@@ -169,8 +169,8 @@ func (p *PostgresStorage) GetL2BlockTransactionCountByNumber(ctx context.Context
 }
 
 // AddL2Block adds a new L2 block to the State Store
-func (p *PostgresStorage) AddL2Block(ctx context.Context, batchNumber uint64, l2Block *state.L2Block, receipts []*types.Receipt, txsL2Hash []common.Hash, txsEGPData []state.StoreTxEGPData, dbTx pgx.Tx) error {
-	//TODO: Optmize this function using only one SQL (with several values) to insert all the txs, receips and logs
+func (p *PostgresStorage) AddL2Block(ctx context.Context, batchNumber uint64, l2Block *state.L2Block, receipts []*types.Receipt, txsL2Hash []common.Hash, txsEGPData []state.StoreTxEGPData, imStateRoots []common.Hash, dbTx pgx.Tx) error {
+	// TODO: Optimize this function using only one SQL (with several values) to insert all the txs, receipts and logs
 	log.Debugf("[AddL2Block] adding L2 block %d", l2Block.NumberU64())
 	start := time.Now()
 
@@ -255,7 +255,7 @@ func (p *PostgresStorage) AddL2Block(ctx context.Context, batchNumber uint64, l2
 	}
 
 	if len(receipts) > 0 {
-		p.AddReceipts(ctx, receipts, dbTx)
+		p.AddReceipts(ctx, receipts, imStateRoots, dbTx)
 
 		var logs []*types.Log
 		for _, receipt := range receipts {

state/pgstatestorage/pgstatestorage_test.go

@@ -188,6 +188,7 @@ func TestGetBatchByL2BlockNumber(t *testing.T) {
 	transactions := []*types.Transaction{tx}
 
 	receipts := []*types.Receipt{receipt}
+	imStateRoots := []common.Hash{state.ZeroHash}
 
 	// Create block to be able to calculate its hash
 	st := trie.NewStackTrie(nil)
@@ -202,7 +203,7 @@ func TestGetBatchByL2BlockNumber(t *testing.T) {
 		txsL2Hash[i] = common.HexToHash(fmt.Sprintf("0x%d", i))
 	}
 
-	err = pgStateStorage.AddL2Block(ctx, batchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, dbTx)
+	err = pgStateStorage.AddL2Block(ctx, batchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, imStateRoots, dbTx)
 	require.NoError(t, err)
 	result, err := pgStateStorage.BatchNumberByL2BlockNumber(ctx, l2Block.Number().Uint64(), dbTx)
 	require.NoError(t, err)
@@ -724,7 +725,7 @@ func TestGetLastVerifiedL2BlockNumberUntilL1Block(t *testing.T) {
 			txsL2Hash[i] = common.HexToHash(fmt.Sprintf("0x%d", i))
 		}
 
-		err = testState.AddL2Block(ctx, batchNumber, l2Block, []*types.Receipt{}, txsL2Hash, storeTxsEGPData, dbTx)
+		err = testState.AddL2Block(ctx, batchNumber, l2Block, []*types.Receipt{}, txsL2Hash, storeTxsEGPData, []common.Hash{}, dbTx)
 		require.NoError(t, err)
 
 		virtualBatch := state.VirtualBatch{BlockNumber: blockNumber, BatchNumber: batchNumber, Coinbase: addr, SequencerAddr: addr, TxHash: hash}
@@ -923,6 +924,7 @@ func TestGetLogs(t *testing.T) {
 
 		transactions := []*types.Transaction{tx}
 		receipts := []*types.Receipt{receipt}
+		stateRoots := []common.Hash{state.ZeroHash}
 
 		header := state.NewL2Header(&types.Header{
 			Number:     big.NewInt(int64(i) + 1),
@@ -948,7 +950,7 @@ func TestGetLogs(t *testing.T) {
 			txsL2Hash[i] = common.HexToHash(fmt.Sprintf("0x%d", i))
 		}
 
-		err = testState.AddL2Block(ctx, batchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, dbTx)
+		err = testState.AddL2Block(ctx, batchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, stateRoots, dbTx)
 		require.NoError(t, err)
 	}
 
@@ -1054,6 +1056,7 @@ func TestGetNativeBlockHashesInRange(t *testing.T) {
 
 		transactions := []*types.Transaction{tx}
 		receipts := []*types.Receipt{receipt}
+		stateRoots := []common.Hash{state.ZeroHash}
 
 		header := state.NewL2Header(&types.Header{
 			Number:     big.NewInt(int64(i) + 1),
@@ -1079,7 +1082,7 @@ func TestGetNativeBlockHashesInRange(t *testing.T) {
 			txsL2Hash[i] = common.HexToHash(fmt.Sprintf("0x%d", i))
 		}
 
-		err = testState.AddL2Block(ctx, batchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, dbTx)
+		err = testState.AddL2Block(ctx, batchNumber, l2Block, receipts, txsL2Hash, storeTxsEGPData, stateRoots, dbTx)
 		require.NoError(t, err)
 
 		nativeBlockHashes = append(nativeBlockHashes, l2Block.Header().Root)