Add bookmarks to streamer

.gitignore

@@ -11,7 +11,9 @@
 /test/contracts/bin/**/*.abi
 
 /tools/datastreamer/*.bin
+/test/datastreamer/*.db/*
 /test/datastream.bin
+/test/*.db/**/*
 
 **/.DS_Store
 .vscode

go.mod

@@ -3,7 +3,7 @@ module github.com/0xPolygonHermez/zkevm-node
 go 1.19
 
 require (
-	github.com/0xPolygonHermez/zkevm-data-streamer v0.0.8
+	github.com/0xPolygonHermez/zkevm-data-streamer v0.0.9
 	github.com/didip/tollbooth/v6 v6.1.2
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127
 	github.com/ethereum/go-ethereum v1.13.2
@@ -58,7 +58,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
@@ -112,7 +112,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect

go.sum

@@ -42,8 +42,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/0xPolygonHermez/zkevm-data-streamer v0.0.8 h1:hOByFEvUC8hJnfbINMFzXxBru07AQLEhN50afow6Eu8=
-github.com/0xPolygonHermez/zkevm-data-streamer v0.0.8/go.mod h1:UqLxA+/R20fm63Mp+J7wYMfh6WoE+6vBj6rOmFGuRm4=
+github.com/0xPolygonHermez/zkevm-data-streamer v0.0.9 h1:kn5V5qTyTSyCCI+AK8Z8yno/XdQTN5SLQJwx6VQNtU0=
+github.com/0xPolygonHermez/zkevm-data-streamer v0.0.9/go.mod h1:gAHakobPs4k6Ku0VeAhbGo4T0OaNswXBkPp/2ff4rXo=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -145,8 +145,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
 github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
@@ -599,8 +600,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
 github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=

sequencer/dbmanager.go

@@ -175,6 +175,17 @@ func (d *dbManager) sendDataToStreamer() {
 				continue
 			}
 
+			bookMark := state.DSBookMark{
+				Type:          state.BookMarkTypeL2Block,
+				L2BlockNumber: l2Block.L2BlockNumber,
+			}
+
+			_, err = d.streamServer.AddStreamBookmark(bookMark.Encode())
+			if err != nil {
+				log.Errorf("failed to add stream bookmark for l2block %v: %v", l2Block.L2BlockNumber, err)
+				continue
+			}
+
 			blockStart := state.DSL2BlockStart{
 				BatchNumber:    l2Block.BatchNumber,
 				L2BlockNumber:  l2Block.L2BlockNumber,

state/datastream.go

@@ -16,6 +16,8 @@ const (
 	EntryTypeL2Tx datastreamer.EntryType = 2
 	// EntryTypeL2BlockEnd represents a L2 block end
 	EntryTypeL2BlockEnd datastreamer.EntryType = 3
+	// BookMarkTypeL2Block represents a L2 block bookmark
+	BookMarkTypeL2Block byte = 0
 )
 
 // DSL2FullBlock represents a data stream L2 full block and its transactions
@@ -87,7 +89,21 @@ type DSL2BlockEnd struct {
 func (b DSL2BlockEnd) Encode() []byte {
 	bytes := make([]byte, 0)
 	bytes = binary.LittleEndian.AppendUint64(bytes, b.L2BlockNumber)
-	bytes = append(bytes, b.BlockHash.Bytes()...)
-	bytes = append(bytes, b.StateRoot.Bytes()...)
+	bytes = append(bytes, b.BlockHash[:]...)
+	bytes = append(bytes, b.StateRoot[:]...)
+	return bytes
+}
+
+// DSBookMark represents a data stream bookmark
+type DSBookMark struct {
+	Type          byte
+	L2BlockNumber uint64
+}
+
+// Encode returns the encoded DSBookMark as a byte slice
+func (b DSBookMark) Encode() []byte {
+	bytes := make([]byte, 0)
+	bytes = append(bytes, b.Type)
+	bytes = binary.LittleEndian.AppendUint64(bytes, b.L2BlockNumber)
 	return bytes
 }

tools/datastreamer/Makefile

@@ -6,18 +6,28 @@ check-go:
 
 # Targets that require the checks
 generate-file: check-go
+rebuild: check-go
+decode-entry: check-go
+adecode-l2block: check-go
 
 arguments := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
 
 .PHONY: generate-file
 generate-file: ## Runs the tool to populate the binary file
 	go run main.go generate -cfg config/tool.config.toml
 
-rebuild: ## Runs the tool to rebuild state roots from a given block
-	go run main.go rebuild -cfg config/tool.config.toml
+.PHONY: decode-l2block
+decode-l2block: ## Runs the tool to decode a given l2 block number
+	go run main.go decode-l2block -cfg config/tool.config.toml -l2block $(arguments)
 
+.PHONY: decode-entry
 decode-entry: ## Runs the tool to decode a given entry number
-	go run main.go decode -cfg config/tool.config.toml -entry $(arguments)
+	go run main.go decode-entry -cfg config/tool.config.toml -entry $(arguments)
+
+.PHONY: rebuild
+rebuild: ## Runs the tool to rebuild state roots from a given block
+	go run main.go rebuild -cfg config/tool.config.toml -l2block $(arguments)
+
 
 ## Help display.
 ## Pulls comments from beside commands and prints a nicely formatted

tools/datastreamer/config/tool.config.toml

@@ -13,7 +13,7 @@ Password = "state_password"
 Name = "state_db"
 Host = "localhost"
 Port = "5432"
-EnableLog = false	
+EnableLog = false
 MaxConns = 200
 
 [Executor]

tools/datastreamer/main.go

@@ -31,6 +31,13 @@ var (
 		Usage:    "Entry `NUMBER`",
 		Required: false,
 	}
+
+	l2blockFlag = cli.Uint64Flag{
+		Name:     "l2block",
+		Aliases:  []string{"b"},
+		Usage:    "L2Block `NUMBER`",
+		Required: false,
+	}
 )
 
 func main() {
@@ -40,6 +47,7 @@ func main() {
 	flags := []cli.Flag{
 		&configFileFlag,
 		&entryFlag,
+		&l2blockFlag,
 	}
 
 	app.Commands = []*cli.Command{
@@ -58,10 +66,17 @@ func main() {
 			Flags:   flags,
 		},
 		{
-			Name:    "decode",
+			Name:    "decode-entry",
 			Aliases: []string{},
 			Usage:   "Decodes an entry",
-			Action:  decode,
+			Action:  decodeEntry,
+			Flags:   flags,
+		},
+		{
+			Name:    "decode-l2block",
+			Aliases: []string{},
+			Usage:   "Decodes a l2 block",
+			Action:  decodeL2Block,
 			Flags:   flags,
 		},
 	}
@@ -150,6 +165,16 @@ func generate(cliCtx *cli.Context) error {
 			log.Fatal(err)
 		}
 
+		bookMark := state.DSBookMark{
+			Type:          state.BookMarkTypeL2Block,
+			L2BlockNumber: genesisL2Block.L2BlockNumber,
+		}
+
+		_, err = streamServer.AddStreamBookmark(bookMark.Encode())
+		if err != nil {
+			log.Fatal(err)
+		}
+
 		genesisBlock := state.DSL2BlockStart{
 			BatchNumber:    genesisL2Block.BatchNumber,
 			L2BlockNumber:  genesisL2Block.L2BlockNumber,
@@ -261,6 +286,16 @@ func generate(cliCtx *cli.Context) error {
 				ForkID:         l2block.ForkID,
 			}
 
+			bookMark := state.DSBookMark{
+				Type:          state.BookMarkTypeL2Block,
+				L2BlockNumber: blockStart.L2BlockNumber,
+			}
+
+			_, err = streamServer.AddStreamBookmark(bookMark.Encode())
+			if err != nil {
+				log.Fatal(err)
+			}
+
 			_, err = streamServer.AddStreamEntry(state.EntryTypeL2BlockStart, blockStart.Encode())
 			if err != nil {
 				log.Fatal(err)
@@ -438,7 +473,7 @@ func rebuild(cliCtx *cli.Context) error {
 	return nil
 }
 
-func decode(cliCtx *cli.Context) error {
+func decodeEntry(cliCtx *cli.Context) error {
 	c, err := config.Load(cliCtx)
 	if err != nil {
 		log.Fatal(err)
@@ -457,27 +492,85 @@ func decode(cliCtx *cli.Context) error {
 
 	log.Infof("Selected entry: %+v", entry)
 
+	printEntry(entry)
+
+	return nil
+}
+
+func decodeL2Block(cliCtx *cli.Context) error {
+	c, err := config.Load(cliCtx)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Infof("Loaded configuration: %+v", c)
+
+	streamServer, err := initializeStreamServer(c)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	bookMark := state.DSBookMark{
+		Type:          state.BookMarkTypeL2Block,
+		L2BlockNumber: cliCtx.Uint64("l2block"),
+	}
+
+	startEntry, err := streamServer.GetFirstEventAfterBookmark(bookMark.Encode())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	txEntry, err := streamServer.GetEntry(startEntry.EntryNum + 1)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	endEntry, err := streamServer.GetEntry(startEntry.EntryNum + 2)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	printEntry(startEntry)
+	printEntry(txEntry)
+	printEntry(endEntry)
+
+	return nil
+}
+
+func printEntry(entry datastreamer.FileEntry) {
 	switch entry.EntryType {
 	case state.EntryTypeL2BlockStart:
-		log.Info("Entry type is L2BlockStart")
+		log.Infof("Entry %d: L2BlockStart", entry.EntryNum)
 		batchNumber := binary.LittleEndian.Uint64(entry.Data[0:8])
 		log.Infof("Batch number: %d", batchNumber)
 		l2BlockNumber := binary.LittleEndian.Uint64(entry.Data[8:16])
 		log.Infof("L2 block number: %d", l2BlockNumber)
+		timestamp := binary.LittleEndian.Uint64(entry.Data[16:24])
+		log.Infof("Timestamp: %d", timestamp)
+		globalExitRoot := "0x" + common.Bytes2Hex(entry.Data[24:56])
+		log.Infof("Global exit root: %s", globalExitRoot)
+		coinbase := "0x" + common.Bytes2Hex(entry.Data[56:76])
+		log.Infof("Coinbase: %s", coinbase)
+		forkID := binary.LittleEndian.Uint16(entry.Data[76:78])
+		log.Infof("Fork ID: %d", forkID)
 	case state.EntryTypeL2Tx:
-		log.Info("Entry type is L2Tx")
-		log.Infof("Data: %s", string(entry.Data[6:]))
+		log.Infof("Entry %d: L2Tx", entry.EntryNum)
+		effectiveGasPricePercentage := entry.Data[0]
+		log.Infof("Effective gas price percentage: %d", effectiveGasPricePercentage)
+		isValid := entry.Data[1] == 1
+		log.Infof("Is valid: %t", isValid)
+		encodeLength := binary.LittleEndian.Uint16(entry.Data[2:6])
+		log.Infof("Encode length: %d", encodeLength)
+		encode := entry.Data[6:]
+		log.Infof("Encode: %s", "0x"+common.Bytes2Hex(encode))
 		tx, err := state.DecodeTx(string(entry.Data[6:]))
 		if err != nil {
 			log.Fatal(err)
 		}
-		log.Infof("Transaction: %+v", tx)
+		log.Infof("Decoded: %+v", tx)
 	case state.EntryTypeL2BlockEnd:
-		log.Info("Entry type is L2BlockEnd")
+		log.Infof("Entry %d: L2BlockEnd", entry.EntryNum)
 		log.Infof("L2 Block Number: %d", binary.LittleEndian.Uint64(entry.Data[0:8]))
 		log.Infof("Block Hash: %s", "0x"+common.Bytes2Hex(entry.Data[8:40]))
 		log.Infof("State root: %s", "0x"+common.Bytes2Hex(entry.Data[40:72]))
 	}
-
-	return nil
 }