save work (#327)

.gitignore

@@ -15,4 +15,6 @@ bin/*
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
-main
+main
+.vscode
+*.log

cmd/server/server.go

@@ -2,13 +2,18 @@ package server
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
 	"time"
 
 	"github.com/iotexproject/iotex-analytics/config"
+	"github.com/iotexproject/iotex-analytics/indexer"
 	"github.com/iotexproject/iotex-analytics/indexservice"
 	"github.com/iotexproject/iotex-analytics/sql"
+	"github.com/iotexproject/iotex-core/action/protocol"
+	"github.com/iotexproject/iotex-core/blockchain/blockdao"
+	"github.com/iotexproject/iotex-core/blockchain/genesis"
 	"github.com/iotexproject/iotex-core/pkg/log"
 	"github.com/iotexproject/iotex-proto/golang/iotexapi"
 	"github.com/urfave/cli/v2"
@@ -44,13 +49,17 @@ func runServer(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	if err := log.InitLoggers(cfg.Log, cfg.SubLogs); err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: Failed to init logger: %v\n", err)
+		os.Exit(1)
+	}
 
 	store := sql.CreateMySQLStore(
-		cfg.Database.UserName,
-		cfg.Database.Password,
-		cfg.Database.Host,
-		cfg.Database.Port,
-		cfg.Database.DBName,
+		cfg.Mysql.UserName,
+		cfg.Mysql.Password,
+		cfg.Mysql.Host,
+		cfg.Mysql.Port,
+		cfg.Mysql.DBName,
 	)
 
 	ctx := sql.WithStore(c.Context, store)
@@ -62,12 +71,31 @@ func runServer(c *cli.Context) error {
 		log.L().Error("Failed to connect to chain's API server.")
 	}
 	chainClient := iotexapi.NewAPIServiceClient(conn1)
-	ids := indexservice.NewIndexService(chainClient, 1, nil, nil)
 
-	if err := ids.Start(ctx); err != nil {
-		log.L().Fatal("Failed to start the indexer", zap.Error(err))
+	var indexers []blockdao.BlockIndexer
+	var dao blockdao.BlockDAO
+	if false {
+		dao = blockdao.NewBlockDAOInMemForTest(indexers)
+	} else {
+		dao = blockdao.NewBlockDAO(indexers, cfg.BlockDB)
 	}
-
+	var tip protocol.TipInfo
+	ctx = protocol.WithBlockchainCtx(
+		ctx,
+		protocol.BlockchainCtx{
+			Genesis: genesis.Default,
+			Tip:     tip,
+		},
+	)
+	var asyncindexers []indexer.AsyncIndexer
+	iht := &indexer.IndexHeightTable{}
+	asyncindexers = append(asyncindexers, indexer.NewBlockIndexer(store, iht))
+	ids := indexservice.NewIndexService(chainClient, 1, dao, asyncindexers)
+	go func() {
+		if err := ids.Start(ctx); err != nil {
+			log.L().Fatal("Failed to start the indexer", zap.Error(err))
+		}
+	}()
 	handleShutdown(ids)
 
 	return nil

config/config.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path/filepath"
 
+	coreconfig "github.com/iotexproject/iotex-core/config"
+	"github.com/iotexproject/iotex-core/pkg/log"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
@@ -17,29 +19,43 @@ var (
 			Host: "127.0.0.1",
 			Port: "8113",
 		},
+		BlockDB: coreconfig.DB{
+			NumRetries:            3,
+			MaxCacheSize:          64,
+			BlockStoreBatchSize:   16,
+			V2BlocksToSplitDB:     1000000,
+			Compressor:            "Snappy",
+			CompressLegacy:        false,
+			SplitDBSizeMB:         0,
+			SplitDBHeight:         900000,
+			HistoryStateRetention: 2000,
+		},
+		SubLogs: make(map[string]log.GlobalConfig),
 	}
 )
 
 type (
 	Server struct {
-		Host string `yaml:"host" json:"host"`
-		Port string `yaml:"port" json:"port"`
+		Host string `yaml:"host"`
+		Port string `yaml:"port"`
 	}
-	Database struct {
-		DBName   string `yaml:"dbname" json:"dbname"`
-		UserName string `yaml:"username" json:"username"`
-		Password string `yaml:"password" json:"password"`
-		Host     string `yaml:"host" json:"host"`
-		Port     string `yaml:"port" json:"port"`
+	Mysql struct {
+		DBName   string `yaml:"dbname"`
+		UserName string `yaml:"username"`
+		Password string `yaml:"password"`
+		Host     string `yaml:"host"`
+		Port     string `yaml:"port"`
 	}
 	Iotex struct {
-		ChainEndPoint string `yaml:"chainEndPoint" json:"chainEndPoint"`
+		ChainEndPoint string `yaml:"chainEndPoint"`
 	}
 	Config struct {
-		Server   Server   `yaml:"server" json:"server"`
-		Database Database `yaml:"database" json:"database"`
-		Iotex    Iotex    `yaml:"iotex" json:"iotex"`
-		//Index    indexservice.Config `yaml:"indexer" json:"indexer"`
+		Server  Server                      `yaml:"server"`
+		Mysql   Mysql                       `yaml:"mysql"`
+		Iotex   Iotex                       `yaml:"iotex"`
+		BlockDB coreconfig.DB               `yaml:"blockDB"`
+		Log     log.GlobalConfig            `yaml:"log"`
+		SubLogs map[string]log.GlobalConfig `yaml:"subLogs" json:"subLogs"`
 	}
 )
 

config_new.yaml

@@ -1,10 +1,23 @@
 server:
   port: 8883
-database:
+mysql:
   host: 127.0.0.1
   port: 3306
   username: root
   password: root
   dbname: iotex_analytics
 iotex:
   chainEndPoint: api.testnet.iotex.one:80
+blockDB:
+  dbPath: chain.db
+log:
+  zap:
+    development: true
+    level: debug
+    encoding: console
+    disableCaller: true
+    disableStacktrace: false
+    outputPaths: ["stdout"]
+    errorOutputPaths: ["stderr"]
+  stderrRedirectFile: error.log
+  stdLogRedirect: false

indexer/blockindexer.go

@@ -5,6 +5,8 @@ import (
 
 	s "github.com/iotexproject/iotex-analytics/sql"
 	"github.com/iotexproject/iotex-core/blockchain/block"
+	"github.com/iotexproject/iotex-core/pkg/log"
+	"go.uber.org/zap"
 )
 
 /**
@@ -32,27 +34,34 @@ CREATE TABLE `block_history` (
 
 type blockIndexer struct {
 	iht *IndexHeightTable
+	log *zap.Logger
 }
 
 // NewBlockIndexer returns the
 func NewBlockIndexer(store s.Store, iht *IndexHeightTable) AsyncIndexer {
 	return &blockIndexer{
 		iht: iht,
+		log: log.Logger("blockindexer"),
 	}
 }
 
-func (bi *blockIndexer) Start(context.Context) error {
+func (bi *blockIndexer) Start(ctx context.Context) error {
+	bi.log.Debug("blockindex start")
 	return nil
 }
 
-func (bi *blockIndexer) Stop(context.Context) error {
+func (bi *blockIndexer) Stop(ctx context.Context) error {
+	bi.log.Debug("blockindex stop")
+
 	return nil
 }
 
-func (bi *blockIndexer) NextHeight(context.Context) (uint64, error) {
+func (bi *blockIndexer) NextHeight(ctx context.Context) (uint64, error) {
+	log.L().Info("NextHeight")
 	return 0, nil
 }
 
-func (bi *blockIndexer) PutBlock(context.Context, *block.Block) error {
+func (bi *blockIndexer) PutBlock(ctx context.Context, blk *block.Block) error {
+	bi.log.Debug("blockindexer putblock", zap.Any("block", blk))
 	return nil
 }

indexer/indexerheighttable.go

@@ -7,6 +7,13 @@ import (
 	s "github.com/iotexproject/iotex-analytics/sql"
 )
 
+/*
+CREATE TABLE `index_heights` (
+  `name` varchar(128) NOT NULL,
+  `height` bigint(20) unsigned NOT NULL DEFAULT '0',
+  PRIMARY KEY (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+*/
 type IndexHeightTable struct {
 }
 

indexservice/indexservice.go

@@ -58,7 +58,6 @@ func (is *IndexService) startIndex(ctx context.Context, i int) error {
 			select {
 			case <-is.terminate:
 				is.wg.Done()
-				is.terminate <- true
 				return
 			case tipHeight := <-is.subscribers[i]:
 				{
@@ -138,7 +137,6 @@ func (is *IndexService) Start(ctx context.Context) error {
 		for {
 			select {
 			case <-is.terminate:
-				is.terminate <- true
 				return
 			default:
 				res, err := is.chainClient.GetChainMeta(ctx, &iotexapi.GetChainMetaRequest{})
@@ -158,9 +156,8 @@ func (is *IndexService) Start(ctx context.Context) error {
 
 // Stop stops the index service
 func (is *IndexService) Stop(ctx context.Context) error {
-	is.terminate <- true
+	close(is.terminate)
 	is.wg.Wait()
-
 	return is.dao.Stop(ctx)
 }
 
@@ -169,6 +166,7 @@ func (is *IndexService) fetchAndBuild(ctx context.Context, tipHeight uint64) err
 	if err != nil {
 		return errors.Wrap(err, "failed to get tip height from block dao")
 	}
+	log.L().Debug("fetch dao height", zap.Uint64("height", lastHeight))
 	chainClient := is.chainClient
 	for startHeight := lastHeight + 1; startHeight <= tipHeight; {
 		count := is.batchSize