From 24b247d68a2202c82e12ed0c06c3bb4dfe01a315 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 8 May 2021 19:15:20 +0300 Subject: [PATCH] :recycle: Refactoring CBR.ru client for support many currencies (Closes #39) --- README.md | 5 +- investments/currency.py | 6 +- investments/data_providers/cbr.py | 76 ++++++++++++----------- investments/ibtax/ibtax.py | 15 ++--- tests/data_providers/cbr_test.py | 16 ++--- tests/ibtax/prepare_trades_report_test.py | 4 +- tests/report_parsers/ib_test.py | 9 ++- 7 files changed, 68 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 3d020af..5f38df4 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ $ pip3 install investments --upgrade --user - расчет сделок по методу ФИФО, учет даты расчетов (settle date) - конвертация по курсу ЦБ +- поддержка валют USD, RUB, EUR, AUD, GBP, CAD, CZK, DKK, HKD, HUF, YEN, KRW, NOK, PLN, SGD, ZAR, SEK, CHF, TRY - раздельный результат сделок по акциям и опционам + дивиденды - учёт начисленных процентов на остаток по счету -- учитывает комисии по сделкам -- пока **НЕ** поддерживаются сделки в валютах, отличных от USD +- учёт комисий по сделкам +- пока **НЕ** поддерживаются валюты CNH, ILS, MXN, NZD - пока **НЕ** поддерживаются сплиты - пока **НЕ** поддерживаются сделки Forex, сделка пропускается и выводится сообщение о том, что это может повлиять на итоговый отчет diff --git a/investments/currency.py b/investments/currency.py index a964c1d..65ee73e 100644 --- a/investments/currency.py +++ b/investments/currency.py @@ -45,8 +45,10 @@ def __init__(self, aliases: Tuple[str], iso_code: str, cbr_code: str): @staticmethod def parse(search: str): try: - return [currency_item for _, currency_item in Currency.__members__.items() # noqa: WPS609 - if search in currency_item.aliases][0] + return [ + currency_item for _, currency_item in Currency.__members__.items() # noqa: WPS609 + if search in currency_item.aliases + ][0] except IndexError: raise ValueError(search) diff --git a/investments/data_providers/cbr.py b/investments/data_providers/cbr.py index f36db1f..f376fda 100644 --- a/investments/data_providers/cbr.py +++ b/investments/data_providers/cbr.py @@ -1,14 +1,14 @@ """ Клиент к API ЦБ РФ с курсами валют относительно рубля. -Необходим для перевода сумм сделок в рубли по курсу ЦБ на дату поставки в соответствии с НК РФ +Необходим для перевода сумм сделок в других валютах в рубли по курсу ЦБ на дату поставки в соответствии с НК РФ """ import datetime import logging import xml.etree.ElementTree as ET # type: ignore -from typing import List, Tuple +from typing import Dict, List, Optional, Tuple import pandas # type: ignore import requests @@ -19,35 +19,57 @@ class ExchangeRatesRUB: - currency_codes = { - Currency.USD: 'R01235', - Currency.EUR: 'R01239', - } - currency: Currency - _df: pandas.DataFrame + _year_from: int + _cache_dir: Optional[str] + _frames_loaded: Dict[str, pandas.DataFrame] - def __init__(self, currency: Currency, year_from: int = 2000, cache_dir: str = None): - self.currency = currency + def __init__(self, year_from: int = 2000, cache_dir: str = None): + self._year_from = year_from + self._cache_dir = cache_dir + self._frames_loaded = {} - currency_code = self.currency_codes.get(self.currency) - if not currency_code: - raise NotImplementedError(f'only USD and EUR currencies supported [{self.currency} requested]') + def get_rate(self, currency: Currency, dt: datetime.datetime) -> Money: + if currency is Currency.RUB: + return Money(1, Currency.RUB) - cache = DataFrameCache(cache_dir, f'cbrates_{currency_code}_since{year_from}.cache', datetime.timedelta(days=1)) + if currency.name not in self._frames_loaded: + self._fetch_currency_rates(currency) + + rates = self._frames_loaded.get(currency.name) + assert rates is not None + + return rates.loc[dt].item() + + def convert_to_rub(self, source: Money, rate_date: datetime.datetime) -> Money: + assert isinstance(rate_date, datetime.datetime) + + if source.currency == Currency.RUB: + return Money(source.amount, Currency.RUB) + + rate = self.get_rate(source.currency, rate_date) + return Money(source.amount * rate.amount, rate.currency) + + def _fetch_currency_rates(self, currency: Currency): + """Загружаем курс запрошенной валюты из кеша или с cbr.ru.""" + cache_key = f'cbrates_{currency.cbr_code}_since{self._year_from}.cache' + logging.info(f'load currency rates from cbr.ru {currency} {cache_key}') + frame_key = currency.name + + cache = DataFrameCache(self._cache_dir, cache_key, datetime.timedelta(days=1)) df = cache.get() if df is not None: - logging.info('CBR cache hit') - self._df = df + logging.info('cache hit') + self._frames_loaded[frame_key] = df return end_date = (datetime.datetime.utcnow() + datetime.timedelta(days=1)).strftime('%d/%m/%Y') - r = requests.get(f'http://www.cbr.ru/scripts/XML_dynamic.asp?date_req1=01/01/{year_from}&date_req2={end_date}&VAL_NM_RQ={currency_code}') + r = requests.get(f'http://www.cbr.ru/scripts/XML_dynamic.asp?date_req1=01/01/{self._year_from}&date_req2={end_date}&VAL_NM_RQ={currency.cbr_code}') tree = ET.fromstring(r.text) rates_data: List[Tuple[datetime.date, Money]] = [] for rec in tree.findall('Record'): - assert rec.get('Id') == currency_code + assert rec.get('Id') == currency.cbr_code d = datetime.datetime.strptime(rec.attrib['Date'], '%d.%m.%Y').date() v = rec.findtext('Value') assert isinstance(v, str) @@ -60,20 +82,4 @@ def __init__(self, currency: Currency, year_from: int = 2000, cache_dir: str = N df['rate'].fillna(method='pad', inplace=True) cache.put(df) - self._df = df - - @property - def dataframe(self) -> pandas.DataFrame: - return self._df - - def get_rate(self, dt: datetime.datetime) -> Money: - return self._df.loc[dt].item() - - def convert_to_rub(self, source: Money, rate_date: datetime.datetime) -> Money: - assert isinstance(rate_date, datetime.datetime) - - if source.currency == Currency.RUB: - return Money(source.amount, Currency.RUB) - - rate = self.get_rate(rate_date) - return Money(source.amount * rate.amount, rate.currency) + self._frames_loaded[frame_key] = df diff --git a/investments/ibtax/ibtax.py b/investments/ibtax/ibtax.py index 97fa8d7..4640ba5 100644 --- a/investments/ibtax/ibtax.py +++ b/investments/ibtax/ibtax.py @@ -55,8 +55,8 @@ def prepare_trades_report(finished_trades: List[FinishedTrade], cbr_client_usd: axis=1, ) - df = df.join(cbr_client_usd.dataframe['rate'].rename('settle_rate'), how='left', on=tax_date_column) - df = df.join(cbr_client_usd.dataframe['rate'].rename('fee_rate'), how='left', on=trade_date_column) + df['settle_rate'] = df.apply(lambda x: cbr_client_usd.get_rate(x['price'].currency, x[tax_date_column]), axis=1) + df['fee_rate'] = df.apply(lambda x: cbr_client_usd.get_rate(x['fee_per_piece'].currency, x[trade_date_column]), axis=1) df['profit_rub'] = df['total_rub'] profit = df.groupby('N')['profit_rub'].sum().reset_index().set_index('N') @@ -76,9 +76,7 @@ def prepare_dividends_report(dividends: List[Dividend], cbr_client_usd: cbr.Exch df = pandas.DataFrame(df_data, columns=['N', 'ticker', 'date', 'amount', 'tax_paid']) df['tax_year'] = df[operation_date_column].map(lambda x: x.year) - - df = df.join(cbr_client_usd.dataframe, how='left', on=operation_date_column) - + df['rate'] = df.apply(lambda x: cbr_client_usd.get_rate(x['amount'].currency, x[operation_date_column]), axis=1) df['amount_rub'] = df.apply(lambda x: cbr_client_usd.convert_to_rub(x['amount'], x[operation_date_column]), axis=1) df['tax_paid_rub'] = df.apply(lambda x: cbr_client_usd.convert_to_rub(x['tax_paid'], x[operation_date_column]), axis=1) df['tax_rate'] = df.apply(lambda x: round(x['tax_paid'].amount * 100 / x['amount'].amount, 2), axis=1) @@ -93,8 +91,7 @@ def prepare_fees_report(fees: List[Fee], cbr_client_usd: cbr.ExchangeRatesRUB) - for i, x in enumerate(fees) ] df = pandas.DataFrame(df_data, columns=['N', operation_date_column, 'amount', 'description', 'tax_year']) - df = df.join(cbr_client_usd.dataframe, how='left', on=operation_date_column) - + df['rate'] = df.apply(lambda x: cbr_client_usd.get_rate(x['amount'].currency, x[operation_date_column]), axis=1) df['amount_rub'] = df.apply(lambda x: cbr_client_usd.convert_to_rub(x['amount'], x[operation_date_column]), axis=1) return df @@ -106,7 +103,7 @@ def prepare_interests_report(interests: List[Interest], cbr_client_usd: cbr.Exch for i, x in enumerate(interests) ] df = pandas.DataFrame(df_data, columns=['N', operation_date_column, 'amount', 'description', 'tax_year']) - df = df.join(cbr_client_usd.dataframe, how='left', on=operation_date_column) + df['rate'] = df.apply(lambda x: cbr_client_usd.get_rate(x['amount'].currency, x[operation_date_column]), axis=1) df['amount_rub'] = df.apply(lambda x: cbr_client_usd.convert_to_rub(x['amount'], x[operation_date_column]), axis=1) return df @@ -294,7 +291,7 @@ def main(): # fixme first_year without dividends first_year = min(trades[0].trade_date.year, dividends[0].date.year) if dividends else trades[0].trade_date.year - cbr_client_usd = cbr.ExchangeRatesRUB(currency=Currency.USD, year_from=first_year, cache_dir=args.cache_dir) + cbr_client_usd = cbr.ExchangeRatesRUB(year_from=first_year, cache_dir=args.cache_dir) dividends_report = prepare_dividends_report(dividends, cbr_client_usd, args.verbose) if dividends else None fees_report = prepare_fees_report(fees, cbr_client_usd) if fees else None diff --git a/tests/data_providers/cbr_test.py b/tests/data_providers/cbr_test.py index 63ed606..2f9ca7e 100644 --- a/tests/data_providers/cbr_test.py +++ b/tests/data_providers/cbr_test.py @@ -1,4 +1,4 @@ -from datetime import datetime, date +from datetime import datetime from decimal import Decimal import pytest # type: ignore @@ -26,19 +26,19 @@ @pytest.mark.parametrize('trade_date,currency,expect_rate', test_cases) def test_exchange_rates_rub(trade_date: datetime, currency: Currency, expect_rate: Money): try: - p = ExchangeRatesRUB(currency=currency, year_from=2015, cache_dir=None) + p = ExchangeRatesRUB(year_from=2015, cache_dir=None) except ConnectionError as ex: pytest.skip(f'connection error: {ex}') return - rate = p.get_rate(trade_date) + rate = p.get_rate(currency, trade_date) assert rate == expect_rate, f'{trade_date}: {rate} != {expect_rate}' def test_convert_to_rub(): - client_usd = ExchangeRatesRUB(Currency.USD) + client_usd = ExchangeRatesRUB() rate_date = datetime(2020, 3, 31) - expected_rate = client_usd.get_rate(rate_date) + expected_rate = client_usd.get_rate(Currency.USD, rate_date) assert expected_rate.amount == Decimal('77.7325') test_usd = Money(10.98, Currency.USD) @@ -52,9 +52,3 @@ def test_convert_to_rub(): assert res.amount == Decimal('858.3066') assert res.currency == Currency.RUB - - -def test_unknown_currency(): - with pytest.raises(NotImplementedError) as ex: - ExchangeRatesRUB(currency=20, year_from=2015, cache_dir=None) - assert 'only USD and EUR currencies supported' in str(ex.value) diff --git a/tests/ibtax/prepare_trades_report_test.py b/tests/ibtax/prepare_trades_report_test.py index 27785ee..2822777 100644 --- a/tests/ibtax/prepare_trades_report_test.py +++ b/tests/ibtax/prepare_trades_report_test.py @@ -29,7 +29,7 @@ def test_simple_trades(): FinishedTrade(N=2, ticker=ticker, trade_date=datetime.datetime(2020, 2, 10, 0, 0), settle_date=datetime.datetime(2020, 2, 12, 0, 0), quantity=-10, price=Money(81.82, Currency.USD), fee_per_piece=Money('-0.101812674', Currency.USD)), ] - cbr_client = ExchangeRatesRUB(Currency.USD) + cbr_client = ExchangeRatesRUB() res: dict = prepare_trades_report(trades, cbr_client).to_dict() @@ -85,7 +85,7 @@ def test_precision(): test_case = test_trades_precision() - res: dict = prepare_trades_report(test_case, ExchangeRatesRUB(Currency.USD)).to_dict() + res: dict = prepare_trades_report(test_case, ExchangeRatesRUB()).to_dict() assert [x.amount for x in res['total_rub'].values()] == [ Decimal('-51586.552320'), # Расход: (80.62 * 10 * 63.9091) + (0.1 * 10 * 63.0359) = 51586.55232₽ diff --git a/tests/report_parsers/ib_test.py b/tests/report_parsers/ib_test.py index ed78ce5..4a4babc 100644 --- a/tests/report_parsers/ib_test.py +++ b/tests/report_parsers/ib_test.py @@ -156,6 +156,9 @@ def test_parse_interests(): Interest,Data,RUB,2020-03-04,RUB Credit Interest for Feb-2020,3.21 Interest,Data,Total,,,3.21 Interest,Data,Total in USD,,,0.04844211 +Interest,Data,CAD,2020-03-04,CAD Credit Interest for Feb-2020,7.45 +Interest,Data,Total,,,7.45 +Interest,Data,Total in USD,,,6.69 Interest,Data,USD,2020-03-04,USD Credit Interest for Feb-2020,0.09 Interest,Data,Total,,,0.09 Interest,Data,Total Interest in USD,,,0.13844211""" @@ -165,11 +168,13 @@ def test_parse_interests(): 'Interest': p._parse_interests, }) - assert len(p.interests) == 2 + assert len(p.interests) == 3 assert p.interests[0] == Interest(date=datetime.date(2020, 3, 4), amount=Money(3.21, Currency.RUB), description='RUB Credit Interest for Feb-2020') - assert p.interests[1] == Interest(date=datetime.date(2020, 3, 4), amount=Money(0.09, Currency.USD), + assert p.interests[2] == Interest(date=datetime.date(2020, 3, 4), amount=Money(0.09, Currency.USD), description='USD Credit Interest for Feb-2020') + assert p.interests[1] == Interest(date=datetime.date(2020, 3, 4), amount=Money(7.45, Currency.CAD), + description='CAD Credit Interest for Feb-2020') def test_parse_cash():