From 064623a64ba4971ab9b03233cc101f4277b7047e Mon Sep 17 00:00:00 2001 From: Nattharat Kondanna Wiriyakulnan Date: Thu, 23 Jul 2020 20:04:13 +0700 Subject: [PATCH] Merge pull request #2302 from bandprotocol/syncable-flusher flusher: Add offset check before sync flusher --- CHANGELOG_UNRELEASED.md | 1 + chain/.gitignore | 1 - chain/docker-config/run.sh | 4 +- .../single-validator/generate-genesis.sh | 41 ------------------- chain/docker-config/single-validator/run.sh | 16 -------- chain/scripts/start_bandd.sh | 2 +- chain/scripts/start_bandd_with_emitter.sh | 3 +- flusher/flusher/cli.py | 1 - flusher/flusher/sync.py | 12 +++++- 9 files changed, 16 insertions(+), 65 deletions(-) delete mode 100755 chain/docker-config/single-validator/generate-genesis.sh delete mode 100755 chain/docker-config/single-validator/run.sh diff --git a/CHANGELOG_UNRELEASED.md b/CHANGELOG_UNRELEASED.md index ef8d34f5dd..f000668e5a 100644 --- a/CHANGELOG_UNRELEASED.md +++ b/CHANGELOG_UNRELEASED.md @@ -22,6 +22,7 @@ ### Emitter & Flusher +- (impv) [\#2302](https://github.com/bandprotocol/bandchain/pull/2302) Add offset check before sync flusher. - (bugs) [\#2298](https://github.com/bandprotocol/bandchain/pull/2298) Fix bug `accumulated_commission` in `emitSetValidator`. - (bugs) [\#2295](https://github.com/bandprotocol/bandchain/pull/2295) Truncate `accumulated_commission` precision. - (impv) [\#2275](https://github.com/bandprotocol/bandchain/pull/2275) Add field `accumulated_commission` in `validators` table. diff --git a/chain/.gitignore b/chain/.gitignore index b8781dced0..da9dffbc29 100644 --- a/chain/.gitignore +++ b/chain/.gitignore @@ -13,4 +13,3 @@ vendor **/target/ /pkg/owasm/res/*.wasm /docker-config/genesis.json -/docker-config/single-validator/genesis.json diff --git a/chain/docker-config/run.sh b/chain/docker-config/run.sh index d53fdfa2d0..c9938a392e 100755 --- a/chain/docker-config/run.sh +++ b/chain/docker-config/run.sh @@ -10,8 +10,8 @@ cp -r /chain/docker-config/files ~/.bandd if [ "$1" == "emitter" ];then sleep 60 bandd start --with-emitter test@172.18.0.31:9092 \ - --p2p.persistent_peers 11392b605378063b1c505c0ab123f04bd710d7d7@172.18.0.11:26656,0851086afcd835d5a6fb0ffbf96fcdf74fec742e@172.18.0.12:26656,63808bd64f2ec19acb2a494c8ce8467c595f6fba@172.18.0.14:26656,7b58b086dd915a79836eb8bfa956aeb9488d13b0@172.18.0.13:26656 --pruning nothing + --p2p.persistent_peers 11392b605378063b1c505c0ab123f04bd710d7d7@172.18.0.11:26656,0851086afcd835d5a6fb0ffbf96fcdf74fec742e@172.18.0.12:26656,63808bd64f2ec19acb2a494c8ce8467c595f6fba@172.18.0.14:26656,7b58b086dd915a79836eb8bfa956aeb9488d13b0@172.18.0.13:26656 else bandd start --rpc.laddr tcp://0.0.0.0:26657 \ - --p2p.persistent_peers 11392b605378063b1c505c0ab123f04bd710d7d7@172.18.0.11:26656,0851086afcd835d5a6fb0ffbf96fcdf74fec742e@172.18.0.12:26656,63808bd64f2ec19acb2a494c8ce8467c595f6fba@172.18.0.14:26656,7b58b086dd915a79836eb8bfa956aeb9488d13b0@172.18.0.13:26656 --pruning nothing + --p2p.persistent_peers 11392b605378063b1c505c0ab123f04bd710d7d7@172.18.0.11:26656,0851086afcd835d5a6fb0ffbf96fcdf74fec742e@172.18.0.12:26656,63808bd64f2ec19acb2a494c8ce8467c595f6fba@172.18.0.14:26656,7b58b086dd915a79836eb8bfa956aeb9488d13b0@172.18.0.13:26656 fi diff --git a/chain/docker-config/single-validator/generate-genesis.sh b/chain/docker-config/single-validator/generate-genesis.sh deleted file mode 100755 index 8da19c92af..0000000000 --- a/chain/docker-config/single-validator/generate-genesis.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -DIR=`dirname "$0"` - -# remove old genesis -rm -rf ~/.band* - -# initial new node -bandd init validator --chain-id bandchain --oracle band1m5lq9u533qaya4q3nfyl6ulzqkpkhge9q8tpzs - -# create acccounts -echo "lock nasty suffer dirt dream fine fall deal curtain plate husband sound tower mom crew crawl guard rack snake before fragile course bacon range" \ - | bandcli keys add validator --recover --keyring-backend test - -echo "smile stem oven genius cave resource better lunar nasty moon company ridge brass rather supply used horn three panic put venue analyst leader comic" \ - | bandcli keys add requester --recover --keyring-backend test - -# add accounts to genesis -bandd add-genesis-account validator 10000000000000uband --keyring-backend test -bandd add-genesis-account requester 10000000000000uband --keyring-backend test - -# genesis configurations -bandcli config chain-id bandchain -bandcli config output json -bandcli config indent true -bandcli config trust-node true - -# register initial validators -bandd gentx \ - --amount 100000000uband \ - --node-id 11392b605378063b1c505c0ab123f04bd710d7d7 \ - --pubkey bandvalconspub1addwnpepq06h7wvh5n5pmrejr6t3pyn7ytpwd5c0kmv0wjdfujs847em8dusjl96sxg \ - --name validator \ - --ip 172.18.0.15 \ - --keyring-backend test - -# collect genesis transactions -bandd collect-gentxs - -# copy genesis to the proper location! -cp ~/.bandd/config/genesis.json $DIR/genesis.json diff --git a/chain/docker-config/single-validator/run.sh b/chain/docker-config/single-validator/run.sh deleted file mode 100755 index 10397f4a4c..0000000000 --- a/chain/docker-config/single-validator/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -bandd init validator --chain-id=bandchain - -cp /oracle/docker-config/single-validator/priv_validator_key.json ~/.bandd/config/priv_validator_key.json -cp /oracle/docker-config/single-validator/node_key.json ~/.bandd/config/node_key.json -cp /oracle/docker-config/single-validator/genesis.json ~/.bandd/config/genesis.json - -# add cors in config.toml -cd ~/.bandd/config/ -sed 's/cors_allowed_origins = \[\]/cors_allowed_origins = \["\*"\]/g' config.toml > config_tmp.toml -mv config_tmp.toml config.toml -cd /oracle/ - -mkdir ~/.banddb -bandd start --rpc.laddr tcp://0.0.0.0:26657 --with-db "postgres: host=172.18.0.88 port=5432 user=postgres dbname=postgres password=postgrespassword sslmode=disable" --pruning nothing diff --git a/chain/scripts/start_bandd.sh b/chain/scripts/start_bandd.sh index 0309f4af84..80c95bb1dc 100755 --- a/chain/scripts/start_bandd.sh +++ b/chain/scripts/start_bandd.sh @@ -5,4 +5,4 @@ dropdb my_db createdb my_db # start bandchain -bandd start --rpc.laddr tcp://0.0.0.0:26657 --pruning=nothing +bandd start --rpc.laddr tcp://0.0.0.0:26657 diff --git a/chain/scripts/start_bandd_with_emitter.sh b/chain/scripts/start_bandd_with_emitter.sh index 42b6c08d92..2e351b6068 100755 --- a/chain/scripts/start_bandd_with_emitter.sh +++ b/chain/scripts/start_bandd_with_emitter.sh @@ -17,5 +17,4 @@ python ../flusher/main.py init bandchain test --db localhost:5432/my_db # start bandchain -bandd start --with-emitter test@localhost:9092 \ - --rpc.laddr tcp://0.0.0.0:26657 --pruning=nothing +bandd start --with-emitter test@localhost:9092 --rpc.laddr tcp://0.0.0.0:26657 diff --git a/flusher/flusher/cli.py b/flusher/flusher/cli.py index 7946674b69..b44aee485b 100644 --- a/flusher/flusher/cli.py +++ b/flusher/flusher/cli.py @@ -5,4 +5,3 @@ def cli(): """BandChain's flusher utility program.""" pass - diff --git a/flusher/flusher/sync.py b/flusher/flusher/sync.py index 5cdc25055f..693a8d4e62 100644 --- a/flusher/flusher/sync.py +++ b/flusher/flusher/sync.py @@ -1,6 +1,8 @@ import json import click import sys +import time + from kafka import KafkaConsumer, TopicPartition from loguru import logger from sqlalchemy import create_engine @@ -39,7 +41,15 @@ def sync(commit_interval, db, servers, echo_sqlalchemy): partitions = consumer.partitions_for_topic(topic) if len(partitions) != 1: raise Exception("Only exact 1 partition is supported.") - consumer.seek(TopicPartition(topic, partitions.pop()), tracking_info.kafka_offset + 1) + tp = TopicPartition(topic, partitions.pop()) + while True: + consumer.seek_to_end(tp) + last_offset = consumer.position(tp) + if tracking_info.kafka_offset < last_offset: + break + logger.info("Waiting emitter sync current emitter offset is {}", last_offset) + time.sleep(5) + consumer.seek(tp, tracking_info.kafka_offset + 1) consumer_iter = iter(consumer) # Main loop while True: