From 34390af9971cea4881b0da639ab4ed2473133ffa Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 9 Mar 2022 12:56:49 +0000 Subject: [PATCH 1/9] updated history module --- database/history.go | 3 +- modules/history/utils_prices.go | 12 +++-- modules/pricefeed/expected_modules.go | 1 + .../pricefeed/handle_periodic_operations.go | 48 +++++++++++++++++-- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/database/history.go b/database/history.go index 10f7bc787..8aff0f7c4 100644 --- a/database/history.go +++ b/database/history.go @@ -2,6 +2,7 @@ package database import ( "fmt" + "time" "github.com/forbole/bdjuno/v2/types" ) @@ -18,7 +19,7 @@ func (db *Db) SaveTokenPricesHistory(prices []types.TokenPrice) error { for i, ticker := range prices { vi := i * 4 query += fmt.Sprintf("($%d,$%d,$%d,$%d),", vi+1, vi+2, vi+3, vi+4) - param = append(param, ticker.UnitName, ticker.Price, ticker.MarketCap, ticker.Timestamp) + param = append(param, ticker.UnitName, ticker.Price, ticker.MarketCap, time.Now()) } query = query[:len(query)-1] // Remove trailing "," diff --git a/modules/history/utils_prices.go b/modules/history/utils_prices.go index 492782821..f942c0cc0 100644 --- a/modules/history/utils_prices.go +++ b/modules/history/utils_prices.go @@ -4,11 +4,15 @@ import ( "github.com/forbole/bdjuno/v2/types" ) -// UpdatePricesHistory stores the given prices inside the price history table -func (m *Module) UpdatePricesHistory(prices []types.TokenPrice) error { - if !m.cfg.IsModuleEnabled(moduleName) { - return nil +// IsHistoryModuleEnabled checks if history module is enabled inside config.yaml file +func (m *Module) IsHistoryModuleEnabled() bool { + if m.cfg.IsModuleEnabled(moduleName) { + return true } + return false +} +// UpdatePricesHistory stores the given prices inside the price history table +func (m *Module) UpdatePricesHistory(prices []types.TokenPrice) error { return m.db.SaveTokenPricesHistory(prices) } diff --git a/modules/pricefeed/expected_modules.go b/modules/pricefeed/expected_modules.go index 19ac16113..3b2cdf857 100644 --- a/modules/pricefeed/expected_modules.go +++ b/modules/pricefeed/expected_modules.go @@ -3,5 +3,6 @@ package pricefeed import "github.com/forbole/bdjuno/v2/types" type HistoryModule interface { + IsHistoryModuleEnabled() bool UpdatePricesHistory([]types.TokenPrice) error } diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index 4cd5d023d..9773811a4 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -14,13 +14,20 @@ import ( func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { log.Debug().Str("module", "pricefeed").Msg("setting up periodic tasks") - // Fetch total supply of token in 30 seconds each - if _, err := scheduler.Every(30).Second().Do(func() { + // Fetch total supply of tokens every 2 mins + if _, err := scheduler.Every(2).Minutes().Do(func() { utils.WatchMethod(m.updatePrice) }); err != nil { return fmt.Errorf("error while setting up pricefeed period operations: %s", err) } + // Fetch total supply of tokens every 1hr to store historical price data + if _, err := scheduler.Every(1).Hour().Do(func() { + utils.WatchMethod(m.updatePricesHistory) + }); err != nil { + return fmt.Errorf("error while setting up historic pricefeed period operations: %s", err) + } + return nil } @@ -54,5 +61,40 @@ func (m *Module) updatePrice() error { return fmt.Errorf("error while saving token prices: %s", err) } - return m.historyModule.UpdatePricesHistory(prices) + return nil +} + +// updatePricesHistory fetch total amount of coins in the system from RPC +// and store historical perice data inside the database +func (m *Module) updatePricesHistory() error { + historyEnabled := m.historyModule.IsHistoryModuleEnabled() + + if historyEnabled { + log.Debug(). + Str("module", "pricefeed"). + Str("operation", "pricefeed"). + Msg("getting hourly token price and market cap") + + // Get the list of tokens price id + ids, err := m.db.GetTokensPriceID() + if err != nil { + return fmt.Errorf("error while getting tokens price id: %s", err) + } + + if len(ids) == 0 { + log.Debug().Str("module", "pricefeed").Msg("no traded tokens price id found") + return nil + } + + // Get the tokens prices + prices, err := coingecko.GetTokensPrices(ids) + if err != nil { + return fmt.Errorf("error while getting tokens prices: %s", err) + } + + return m.historyModule.UpdatePricesHistory(prices) + } + + return nil + } From 29520fde8b0c1999765a0a6773a5e6515cdb9ba6 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 9 Mar 2022 13:33:21 +0000 Subject: [PATCH 2/9] add periodic ops to history module --- go.mod | 2 - modules/history/handle_periodic_operations.go | 54 +++++++++++++++++++ modules/history/module.go | 3 +- modules/history/utils_prices.go | 8 --- modules/pricefeed/expected_modules.go | 8 --- .../pricefeed/handle_additional_operations.go | 2 +- .../pricefeed/handle_periodic_operations.go | 41 -------------- modules/pricefeed/module.go | 16 +++--- modules/registrar.go | 2 +- 9 files changed, 65 insertions(+), 71 deletions(-) create mode 100644 modules/history/handle_periodic_operations.go delete mode 100644 modules/pricefeed/expected_modules.go diff --git a/go.mod b/go.mod index 136115e22..d295f7781 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,9 @@ require ( github.com/proullon/ramsql v0.0.0-20181213202341-817cee58a244 github.com/rs/zerolog v1.26.1 github.com/spf13/cobra v1.3.0 - github.com/spf13/viper v1.10.0 github.com/stretchr/testify v1.7.0 github.com/tendermint/tendermint v0.34.14 google.golang.org/grpc v1.42.0 - gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/modules/history/handle_periodic_operations.go b/modules/history/handle_periodic_operations.go new file mode 100644 index 000000000..9d67e2ab0 --- /dev/null +++ b/modules/history/handle_periodic_operations.go @@ -0,0 +1,54 @@ +package history + +import ( + "fmt" + + "github.com/go-co-op/gocron" + "github.com/rs/zerolog/log" + + "github.com/forbole/bdjuno/v2/modules/pricefeed/coingecko" + "github.com/forbole/bdjuno/v2/modules/utils" +) + +// RegisterPeriodicOperations implements modules.PeriodicOperationsModule +func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { + log.Debug().Str("module", "history").Msg("setting up periodic tasks") + + // Fetch total supply of tokens every 1hr to store historical price data + if _, err := scheduler.Every(1).Hour().Do(func() { + utils.WatchMethod(m.updatePricesHistory) + }); err != nil { + return fmt.Errorf("error while setting up history period operations: %s", err) + } + + return nil +} + +// updatePricesHistory fetch total amount of coins in the system from RPC +// and store historical perice data inside the database +func (m *Module) updatePricesHistory() error { + log.Debug(). + Str("module", "history"). + Str("operation", "history"). + Msg("getting token price and market cap history") + + // Get the list of tokens price id + ids, err := m.db.GetTokensPriceID() + if err != nil { + return fmt.Errorf("error while getting tokens price id: %s", err) + } + + if len(ids) == 0 { + log.Debug().Str("module", "history").Msg("no traded tokens price id found") + return nil + } + + // Get the tokens prices + prices, err := coingecko.GetTokensPrices(ids) + if err != nil { + return fmt.Errorf("error while getting tokens prices history: %s", err) + } + + return m.UpdatePricesHistory(prices) + +} diff --git a/modules/history/module.go b/modules/history/module.go index c3b0ff37e..021dde559 100644 --- a/modules/history/module.go +++ b/modules/history/module.go @@ -14,7 +14,8 @@ const ( ) var ( - _ modules.Module = &Module{} + _ modules.Module = &Module{} + _ modules.PeriodicOperationsModule = &Module{} ) // Module represents the module that allows to store historic information diff --git a/modules/history/utils_prices.go b/modules/history/utils_prices.go index f942c0cc0..d6bbcc0f0 100644 --- a/modules/history/utils_prices.go +++ b/modules/history/utils_prices.go @@ -4,14 +4,6 @@ import ( "github.com/forbole/bdjuno/v2/types" ) -// IsHistoryModuleEnabled checks if history module is enabled inside config.yaml file -func (m *Module) IsHistoryModuleEnabled() bool { - if m.cfg.IsModuleEnabled(moduleName) { - return true - } - return false -} - // UpdatePricesHistory stores the given prices inside the price history table func (m *Module) UpdatePricesHistory(prices []types.TokenPrice) error { return m.db.SaveTokenPricesHistory(prices) diff --git a/modules/pricefeed/expected_modules.go b/modules/pricefeed/expected_modules.go deleted file mode 100644 index 3b2cdf857..000000000 --- a/modules/pricefeed/expected_modules.go +++ /dev/null @@ -1,8 +0,0 @@ -package pricefeed - -import "github.com/forbole/bdjuno/v2/types" - -type HistoryModule interface { - IsHistoryModuleEnabled() bool - UpdatePricesHistory([]types.TokenPrice) error -} diff --git a/modules/pricefeed/handle_additional_operations.go b/modules/pricefeed/handle_additional_operations.go index 3ed557792..e5bd62459 100644 --- a/modules/pricefeed/handle_additional_operations.go +++ b/modules/pricefeed/handle_additional_operations.go @@ -56,5 +56,5 @@ func (m *Module) storeTokens() error { return fmt.Errorf("error while storing token prices: %s", err) } - return m.historyModule.UpdatePricesHistory(prices) + return nil } diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index 9773811a4..508d00c0f 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -21,13 +21,6 @@ func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { return fmt.Errorf("error while setting up pricefeed period operations: %s", err) } - // Fetch total supply of tokens every 1hr to store historical price data - if _, err := scheduler.Every(1).Hour().Do(func() { - utils.WatchMethod(m.updatePricesHistory) - }); err != nil { - return fmt.Errorf("error while setting up historic pricefeed period operations: %s", err) - } - return nil } @@ -62,39 +55,5 @@ func (m *Module) updatePrice() error { } return nil -} - -// updatePricesHistory fetch total amount of coins in the system from RPC -// and store historical perice data inside the database -func (m *Module) updatePricesHistory() error { - historyEnabled := m.historyModule.IsHistoryModuleEnabled() - - if historyEnabled { - log.Debug(). - Str("module", "pricefeed"). - Str("operation", "pricefeed"). - Msg("getting hourly token price and market cap") - // Get the list of tokens price id - ids, err := m.db.GetTokensPriceID() - if err != nil { - return fmt.Errorf("error while getting tokens price id: %s", err) - } - - if len(ids) == 0 { - log.Debug().Str("module", "pricefeed").Msg("no traded tokens price id found") - return nil - } - - // Get the tokens prices - prices, err := coingecko.GetTokensPrices(ids) - if err != nil { - return fmt.Errorf("error while getting tokens prices: %s", err) - } - - return m.historyModule.UpdatePricesHistory(prices) - } - - return nil - } diff --git a/modules/pricefeed/module.go b/modules/pricefeed/module.go index d9bacd046..f327bd268 100644 --- a/modules/pricefeed/module.go +++ b/modules/pricefeed/module.go @@ -17,24 +17,22 @@ var ( // Module represents the module that allows to get the token prices type Module struct { - cfg *Config - cdc codec.Marshaler - db *database.Db - historyModule HistoryModule + cfg *Config + cdc codec.Marshaler + db *database.Db } // NewModule returns a new Module instance -func NewModule(cfg config.Config, historyModule HistoryModule, cdc codec.Marshaler, db *database.Db) *Module { +func NewModule(cfg config.Config, cdc codec.Marshaler, db *database.Db) *Module { pricefeedCfg, err := ParseConfig(cfg.GetBytes()) if err != nil { panic(err) } return &Module{ - cfg: pricefeedCfg, - cdc: cdc, - db: db, - historyModule: historyModule, + cfg: pricefeedCfg, + cdc: cdc, + db: db, } } diff --git a/modules/registrar.go b/modules/registrar.go index 7b896f45c..0d4131890 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -129,7 +129,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { historyModule, mintModule, modules.NewModule(ctx.JunoConfig.Chain, db), - pricefeed.NewModule(ctx.JunoConfig, historyModule, cdc, db), + pricefeed.NewModule(ctx.JunoConfig, cdc, db), slashingModule, stakingModule, } From 3d5ff55eb8bef1194f8969726b80ddfe5318aea1 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 10 Mar 2022 11:29:40 +0000 Subject: [PATCH 3/9] rm history module --- cmd/parse-genesis/parseGenesis.go | 2 +- database/history.go | 37 ------------- database/pricefeed.go | 29 ++++++++++ modules/history/handle_periodic_operations.go | 54 ------------------- modules/history/module.go | 43 --------------- modules/history/utils_prices.go | 10 ---- .../pricefeed/handle_periodic_operations.go | 40 ++++++++++++++ modules/registrar.go | 3 -- 8 files changed, 70 insertions(+), 148 deletions(-) delete mode 100644 database/history.go delete mode 100644 modules/history/handle_periodic_operations.go delete mode 100644 modules/history/module.go delete mode 100644 modules/history/utils_prices.go diff --git a/cmd/parse-genesis/parseGenesis.go b/cmd/parse-genesis/parseGenesis.go index 2f08d7cc8..a5b7b50c6 100644 --- a/cmd/parse-genesis/parseGenesis.go +++ b/cmd/parse-genesis/parseGenesis.go @@ -15,7 +15,7 @@ func NewParseGenesisCmd(parseCfg *parse.Config) *cobra.Command { return &cobra.Command{ Use: "parse-genesis [[module names]]", Short: "Parse genesis file. To parse specific modules, input module names as arguments", - Example: "bdjuno parse-genesis auth bank consensus gov history staking", + Example: "bdjuno parse-genesis auth bank consensus gov staking", PreRunE: parse.ReadConfig(parseCfg), RunE: func(cmd *cobra.Command, args []string) error { parseCtx, err := parse.GetParsingContext(parseCfg) diff --git a/database/history.go b/database/history.go deleted file mode 100644 index 8aff0f7c4..000000000 --- a/database/history.go +++ /dev/null @@ -1,37 +0,0 @@ -package database - -import ( - "fmt" - "time" - - "github.com/forbole/bdjuno/v2/types" -) - -// SaveTokenPricesHistory stores the given prices as historic ones -func (db *Db) SaveTokenPricesHistory(prices []types.TokenPrice) error { - if len(prices) == 0 { - return nil - } - - query := `INSERT INTO token_price_history (unit_name, price, market_cap, timestamp) VALUES` - var param []interface{} - - for i, ticker := range prices { - vi := i * 4 - query += fmt.Sprintf("($%d,$%d,$%d,$%d),", vi+1, vi+2, vi+3, vi+4) - param = append(param, ticker.UnitName, ticker.Price, ticker.MarketCap, time.Now()) - } - - query = query[:len(query)-1] // Remove trailing "," - query += ` -ON CONFLICT ON CONSTRAINT unique_price_for_timestamp DO UPDATE - SET price = excluded.price, - market_cap = excluded.market_cap` - - _, err := db.Sql.Exec(query, param...) - if err != nil { - return fmt.Errorf("error while storing tokens price history: %s", err) - } - - return nil -} diff --git a/database/pricefeed.go b/database/pricefeed.go index 318d9b658..4a4f1554a 100644 --- a/database/pricefeed.go +++ b/database/pricefeed.go @@ -92,3 +92,32 @@ WHERE token_price.timestamp <= excluded.timestamp` return nil } + +// SaveTokenPricesHistory stores the given prices as historic ones +func (db *Db) SaveTokenPricesHistory(prices []types.TokenPrice) error { + if len(prices) == 0 { + return nil + } + + query := `INSERT INTO token_price_history (unit_name, price, market_cap, timestamp) VALUES` + var param []interface{} + + for i, ticker := range prices { + vi := i * 4 + query += fmt.Sprintf("($%d,$%d,$%d,$%d),", vi+1, vi+2, vi+3, vi+4) + param = append(param, ticker.UnitName, ticker.Price, ticker.MarketCap, ticker.Timestamp) + } + + query = query[:len(query)-1] // Remove trailing "," + query += ` +ON CONFLICT ON CONSTRAINT unique_price_for_timestamp DO UPDATE + SET price = excluded.price, + market_cap = excluded.market_cap` + + _, err := db.Sql.Exec(query, param...) + if err != nil { + return fmt.Errorf("error while storing tokens price history: %s", err) + } + + return nil +} diff --git a/modules/history/handle_periodic_operations.go b/modules/history/handle_periodic_operations.go deleted file mode 100644 index 9d67e2ab0..000000000 --- a/modules/history/handle_periodic_operations.go +++ /dev/null @@ -1,54 +0,0 @@ -package history - -import ( - "fmt" - - "github.com/go-co-op/gocron" - "github.com/rs/zerolog/log" - - "github.com/forbole/bdjuno/v2/modules/pricefeed/coingecko" - "github.com/forbole/bdjuno/v2/modules/utils" -) - -// RegisterPeriodicOperations implements modules.PeriodicOperationsModule -func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { - log.Debug().Str("module", "history").Msg("setting up periodic tasks") - - // Fetch total supply of tokens every 1hr to store historical price data - if _, err := scheduler.Every(1).Hour().Do(func() { - utils.WatchMethod(m.updatePricesHistory) - }); err != nil { - return fmt.Errorf("error while setting up history period operations: %s", err) - } - - return nil -} - -// updatePricesHistory fetch total amount of coins in the system from RPC -// and store historical perice data inside the database -func (m *Module) updatePricesHistory() error { - log.Debug(). - Str("module", "history"). - Str("operation", "history"). - Msg("getting token price and market cap history") - - // Get the list of tokens price id - ids, err := m.db.GetTokensPriceID() - if err != nil { - return fmt.Errorf("error while getting tokens price id: %s", err) - } - - if len(ids) == 0 { - log.Debug().Str("module", "history").Msg("no traded tokens price id found") - return nil - } - - // Get the tokens prices - prices, err := coingecko.GetTokensPrices(ids) - if err != nil { - return fmt.Errorf("error while getting tokens prices history: %s", err) - } - - return m.UpdatePricesHistory(prices) - -} diff --git a/modules/history/module.go b/modules/history/module.go deleted file mode 100644 index 021dde559..000000000 --- a/modules/history/module.go +++ /dev/null @@ -1,43 +0,0 @@ -package history - -import ( - "github.com/cosmos/cosmos-sdk/codec" - "github.com/forbole/juno/v2/modules" - "github.com/forbole/juno/v2/modules/messages" - "github.com/forbole/juno/v2/types/config" - - "github.com/forbole/bdjuno/v2/database" -) - -const ( - moduleName = "history" -) - -var ( - _ modules.Module = &Module{} - _ modules.PeriodicOperationsModule = &Module{} -) - -// Module represents the module that allows to store historic information -type Module struct { - cfg config.ChainConfig - cdc codec.Marshaler - db *database.Db - - getAddresses messages.MessageAddressesParser -} - -// NewModule allows to build a new Module instance -func NewModule(cfg config.ChainConfig, messagesParser messages.MessageAddressesParser, cdc codec.Marshaler, db *database.Db) *Module { - return &Module{ - cfg: cfg, - cdc: cdc, - db: db, - getAddresses: messagesParser, - } -} - -// Name implements modules.Module -func (m *Module) Name() string { - return moduleName -} diff --git a/modules/history/utils_prices.go b/modules/history/utils_prices.go deleted file mode 100644 index d6bbcc0f0..000000000 --- a/modules/history/utils_prices.go +++ /dev/null @@ -1,10 +0,0 @@ -package history - -import ( - "github.com/forbole/bdjuno/v2/types" -) - -// UpdatePricesHistory stores the given prices inside the price history table -func (m *Module) UpdatePricesHistory(prices []types.TokenPrice) error { - return m.db.SaveTokenPricesHistory(prices) -} diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index 508d00c0f..0592bdb89 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -2,6 +2,7 @@ package pricefeed import ( "fmt" + "time" "github.com/go-co-op/gocron" "github.com/rs/zerolog/log" @@ -21,6 +22,13 @@ func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { return fmt.Errorf("error while setting up pricefeed period operations: %s", err) } + // Fetch total supply of tokens every 1hr to store historical price data + if _, err := scheduler.Every(1).Hour().Do(func() { + utils.WatchMethod(m.updatePricesHistory) + }); err != nil { + return fmt.Errorf("error while setting up history period operations: %s", err) + } + return nil } @@ -57,3 +65,35 @@ func (m *Module) updatePrice() error { return nil } + +// updatePricesHistory fetches total amount of coins in the system from RPC +// and stores historical perice data inside the database +func (m *Module) updatePricesHistory() error { + log.Debug(). + Str("module", "pricefeed"). + Str("operation", "pricefeed"). + Msg("getting token price and market cap history") + + // Get the list of tokens price id + ids, err := m.db.GetTokensPriceID() + if err != nil { + return fmt.Errorf("error while getting tokens price id: %s", err) + } + + if len(ids) == 0 { + log.Debug().Str("module", "pricefeed").Msg("no traded tokens price id found") + return nil + } + + // Get the tokens prices + prices, err := coingecko.GetTokensPrices(ids) + if err != nil { + return fmt.Errorf("error while getting tokens prices history: %s", err) + } + for _, price := range prices { + price.Timestamp = time.Now() + } + + return m.db.SaveTokenPricesHistory(prices) + +} diff --git a/modules/registrar.go b/modules/registrar.go index 0d4131890..6c426a203 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -13,7 +13,6 @@ import ( "github.com/cosmos/cosmos-sdk/simapp/params" "github.com/forbole/juno/v2/node/remote" - "github.com/forbole/bdjuno/v2/modules/history" "github.com/forbole/bdjuno/v2/modules/slashing" "github.com/cosmos/cosmos-sdk/codec" @@ -110,7 +109,6 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) consensusModule := consensus.NewModule(db) distrModule := distribution.NewModule(sources.DistrSource, cdc, db) - historyModule := history.NewModule(ctx.JunoConfig.Chain, r.parser, cdc, db) mintModule := mint.NewModule(sources.MintSource, cdc, db) slashingModule := slashing.NewModule(sources.SlashingSource, cdc, db) stakingModule := staking.NewModule(sources.StakingSource, slashingModule, cdc, db) @@ -126,7 +124,6 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { consensusModule, distrModule, govModule, - historyModule, mintModule, modules.NewModule(ctx.JunoConfig.Chain, db), pricefeed.NewModule(ctx.JunoConfig, cdc, db), From 77df6a86cae1e86377171ad7d21c114235aad4c7 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 10 Mar 2022 12:00:51 +0000 Subject: [PATCH 4/9] add tests --- database/pricefeed_test.go | 142 +++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/database/pricefeed_test.go b/database/pricefeed_test.go index 0aa12fe37..ee07c6444 100644 --- a/database/pricefeed_test.go +++ b/database/pricefeed_test.go @@ -123,3 +123,145 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPrice() { suite.Require().True(expected[i].Equals(row)) } } + +func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPriceHistory() { + suite.insertToken("desmos") + suite.insertToken("atom") + + + // Save data + tickers := []types.TokenPrice{ + types.NewTokenPrice( + "desmos", + 100.01, + 10, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + types.NewTokenPrice( + "desmos", + 200.01, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + types.NewTokenPrice( + "atom", + 1, + 20, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + types.NewTokenPrice( + "atom", + 1, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + } + err := suite.database.SaveTokenPricesHistory(tickers) + suite.Require().NoError(err) + + + // Verify data + expected := []dbtypes.TokenPriceRow{ + dbtypes.NewTokenPriceRow( + "desmos", + 100.01, + 10, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + dbtypes.NewTokenPriceRow( + "desmos", + 200.01, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + dbtypes.NewTokenPriceRow( + "atom", + 1, + 20, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + dbtypes.NewTokenPriceRow( + "atom", + 1, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + } + + + var rows []dbtypes.TokenPriceRow + err = suite.database.Sqlx.Select(&rows, `SELECT * FROM token_price_history`) + suite.Require().NoError(err) + + + for i, row := range rows { + suite.Require().True(expected[i].Equals(row)) + } + + // Update data + tickers = []types.TokenPrice{ + types.NewTokenPrice( + "desmos", + 100.01, + 10, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + types.NewTokenPrice( + "desmos", + 300.01, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + types.NewTokenPrice( + "atom", + 1, + 20, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + types.NewTokenPrice( + "atom", + 10, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + } + err = suite.database.SaveTokenPricesHistory(tickers) + suite.Require().NoError(err) + + // Verify data + expected = []dbtypes.TokenPriceRow{ + dbtypes.NewTokenPriceRow( + "desmos", + 100.01, + 10, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + dbtypes.NewTokenPriceRow( + "atom", + 1, + 20, + time.Date(2020, 10, 10, 15, 00, 00, 000, time.UTC), + ), + dbtypes.NewTokenPriceRow( + "desmos", + 300.01, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + + dbtypes.NewTokenPriceRow( + "atom", + 10, + 20, + time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), + ), + } + + rows = []dbtypes.TokenPriceRow{} + err = suite.database.Sqlx.Select(&rows, `SELECT * FROM token_price_history ORDER BY timestamp`) + suite.Require().NoError(err) + + for i, row := range rows { + suite.Require().True(expected[i].Equals(row)) + } +} From cb97db595d48de204e410284208413a721b8f59c Mon Sep 17 00:00:00 2001 From: Riccardo Montagnin Date: Thu, 10 Mar 2022 13:53:43 +0100 Subject: [PATCH 5/9] Update modules/pricefeed/handle_periodic_operations.go --- modules/pricefeed/handle_periodic_operations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index 0592bdb89..abe754bf6 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -15,7 +15,7 @@ import ( func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { log.Debug().Str("module", "pricefeed").Msg("setting up periodic tasks") - // Fetch total supply of tokens every 2 mins + // Fetch the token prices every 2 mins if _, err := scheduler.Every(2).Minutes().Do(func() { utils.WatchMethod(m.updatePrice) }); err != nil { From e351393f85c8bcaa1ce27a3c4869de5265dcd82f Mon Sep 17 00:00:00 2001 From: Riccardo Montagnin Date: Thu, 10 Mar 2022 13:53:47 +0100 Subject: [PATCH 6/9] Update modules/pricefeed/handle_periodic_operations.go --- modules/pricefeed/handle_periodic_operations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index abe754bf6..f66c6202e 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -22,7 +22,7 @@ func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { return fmt.Errorf("error while setting up pricefeed period operations: %s", err) } - // Fetch total supply of tokens every 1hr to store historical price data + // Update the historical token prices every 1 hour if _, err := scheduler.Every(1).Hour().Do(func() { utils.WatchMethod(m.updatePricesHistory) }); err != nil { From 6fad9832a08590725f9885354855034612abaf5d Mon Sep 17 00:00:00 2001 From: Riccardo Montagnin Date: Thu, 10 Mar 2022 13:58:41 +0100 Subject: [PATCH 7/9] made the code more DRY --- .../pricefeed/handle_periodic_operations.go | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index 0592bdb89..af11890aa 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -2,8 +2,7 @@ package pricefeed import ( "fmt" - "time" - + "github.com/forbole/bdjuno/v2/types" "github.com/go-co-op/gocron" "github.com/rs/zerolog/log" @@ -32,28 +31,38 @@ func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { return nil } -// updatePrice fetch total amount of coins in the system from RPC and store it into database -func (m *Module) updatePrice() error { - log.Debug(). - Str("module", "pricefeed"). - Str("operation", "pricefeed"). - Msg("getting token price and market cap") - +// getTokenPrices allows to get the most up-to-date token prices +func (m *Module) getTokenPrices() ([]types.TokenPrice, error) { // Get the list of tokens price id ids, err := m.db.GetTokensPriceID() if err != nil { - return fmt.Errorf("error while getting tokens price id: %s", err) + return nil, fmt.Errorf("error while getting tokens price id: %s", err) } if len(ids) == 0 { log.Debug().Str("module", "pricefeed").Msg("no traded tokens price id found") - return nil + return nil, nil } // Get the tokens prices prices, err := coingecko.GetTokensPrices(ids) if err != nil { - return fmt.Errorf("error while getting tokens prices: %s", err) + return nil, fmt.Errorf("error while getting tokens prices: %s", err) + } + + return prices, nil +} + +// updatePrice fetch total amount of coins in the system from RPC and store it into database +func (m *Module) updatePrice() error { + log.Debug(). + Str("module", "pricefeed"). + Str("operation", "pricefeed"). + Msg("updating token price and market cap") + + prices, err := m.getTokenPrices() + if err != nil { + return err } // Save the token prices @@ -72,28 +81,17 @@ func (m *Module) updatePricesHistory() error { log.Debug(). Str("module", "pricefeed"). Str("operation", "pricefeed"). - Msg("getting token price and market cap history") + Msg("updating token price and market cap history") - // Get the list of tokens price id - ids, err := m.db.GetTokensPriceID() + prices, err := m.getTokenPrices() if err != nil { - return fmt.Errorf("error while getting tokens price id: %s", err) - } - - if len(ids) == 0 { - log.Debug().Str("module", "pricefeed").Msg("no traded tokens price id found") - return nil + return err } - // Get the tokens prices - prices, err := coingecko.GetTokensPrices(ids) + err = m.db.SaveTokenPricesHistory(prices) if err != nil { - return fmt.Errorf("error while getting tokens prices history: %s", err) + return fmt.Errorf("error while saving token prices history: %s", err) } - for _, price := range prices { - price.Timestamp = time.Now() - } - - return m.db.SaveTokenPricesHistory(prices) + return nil } From 9827a53af939b192a0fc586be32a25a5c5b23a17 Mon Sep 17 00:00:00 2001 From: Riccardo Montagnin Date: Thu, 10 Mar 2022 13:59:13 +0100 Subject: [PATCH 8/9] fun make format --- database/pricefeed_test.go | 8 ++------ modules/pricefeed/handle_periodic_operations.go | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/database/pricefeed_test.go b/database/pricefeed_test.go index ee07c6444..d48e006fe 100644 --- a/database/pricefeed_test.go +++ b/database/pricefeed_test.go @@ -128,7 +128,6 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPriceHistory() { suite.insertToken("desmos") suite.insertToken("atom") - // Save data tickers := []types.TokenPrice{ types.NewTokenPrice( @@ -159,7 +158,6 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPriceHistory() { err := suite.database.SaveTokenPricesHistory(tickers) suite.Require().NoError(err) - // Verify data expected := []dbtypes.TokenPriceRow{ dbtypes.NewTokenPriceRow( @@ -188,12 +186,10 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPriceHistory() { ), } - var rows []dbtypes.TokenPriceRow err = suite.database.Sqlx.Select(&rows, `SELECT * FROM token_price_history`) suite.Require().NoError(err) - for i, row := range rows { suite.Require().True(expected[i].Equals(row)) } @@ -230,7 +226,7 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPriceHistory() { // Verify data expected = []dbtypes.TokenPriceRow{ - dbtypes.NewTokenPriceRow( + dbtypes.NewTokenPriceRow( "desmos", 100.01, 10, @@ -248,7 +244,7 @@ func (suite *DbTestSuite) TestBigDipperDb_SaveTokenPriceHistory() { 20, time.Date(2020, 10, 10, 15, 02, 00, 000, time.UTC), ), - + dbtypes.NewTokenPriceRow( "atom", 10, diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index af11890aa..8d349f146 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -2,10 +2,12 @@ package pricefeed import ( "fmt" - "github.com/forbole/bdjuno/v2/types" + "github.com/go-co-op/gocron" "github.com/rs/zerolog/log" + "github.com/forbole/bdjuno/v2/types" + "github.com/forbole/bdjuno/v2/modules/pricefeed/coingecko" "github.com/forbole/bdjuno/v2/modules/utils" ) From 72212c33c3c204167c4cfe540f08faa67f47b7fb Mon Sep 17 00:00:00 2001 From: Riccardo Montagnin Date: Thu, 10 Mar 2022 14:44:09 +0100 Subject: [PATCH 9/9] added back custom timestamp --- modules/pricefeed/handle_periodic_operations.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/pricefeed/handle_periodic_operations.go b/modules/pricefeed/handle_periodic_operations.go index d9c266392..00f4332c2 100644 --- a/modules/pricefeed/handle_periodic_operations.go +++ b/modules/pricefeed/handle_periodic_operations.go @@ -2,6 +2,7 @@ package pricefeed import ( "fmt" + "time" "github.com/go-co-op/gocron" "github.com/rs/zerolog/log" @@ -90,6 +91,15 @@ func (m *Module) updatePricesHistory() error { return err } + // Normally, the last updated value reflects the time when the price was last updated. + // If price hasn't changed, the returned timestamp will be the same as one hour ago, and it will not + // be stored in db as it will be a duplicated value. + // To fix this, we set each price timestamp to be the same as other ones. + timestamp := time.Now() + for _, price := range prices { + price.Timestamp = timestamp + } + err = m.db.SaveTokenPricesHistory(prices) if err != nil { return fmt.Errorf("error while saving token prices history: %s", err)