From 1f29f1fe28d7d0cf636330f6ef5cc2ee769a6c78 Mon Sep 17 00:00:00 2001 From: Carlo Revelli Date: Tue, 19 Jul 2022 20:00:03 +0100 Subject: [PATCH 1/3] implementation v2 --- js/base/Cache.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/js/base/Cache.js b/js/base/Cache.js index 451cdf745185c..722ff9962f786 100644 --- a/js/base/Cache.js +++ b/js/base/Cache.js @@ -48,10 +48,10 @@ class ArrayCache extends BaseCache { let newUpdatesValue = undefined if (symbol === undefined) { - newUpdatesValue = this.allNewUpdates + newUpdatesValue = this.allNewUpdates this.clearAllUpdates = true } else { - newUpdatesValue = this.newUpdatesBySymbol[symbol]; + newUpdatesValue = (symbol in this.newUpdatesBySymbol) ? this.newUpdatesBySymbol[symbol].size () : 0; this.clearUpdatesBySymbol[symbol] = true } @@ -72,13 +72,13 @@ class ArrayCache extends BaseCache { this.push (item) if (this.clearUpdatesBySymbol[item.symbol]) { this.clearUpdatesBySymbol[item.symbol] = false - this.newUpdatesBySymbol[item.symbol] = 0 + this.newUpdatesBySymbol[item.symbol].clear () } if (this.clearAllUpdates) { this.clearAllUpdates = false this.allNewUpdates = 0 } - this.newUpdatesBySymbol[item.symbol] = (this.newUpdatesBySymbol[item.symbol] || 0) + 1 + this.newUpdatesBySymbol[item.symbol].add (item.id) this.allNewUpdates = (this.allNewUpdates || 0) + 1 } } From 2dd4f18d22891b892aa6934efca490544c0d5f35 Mon Sep 17 00:00:00 2001 From: Carlo Revelli Date: Tue, 19 Jul 2022 20:42:45 +0100 Subject: [PATCH 2/3] implement bugfix for cache same order twice newUpdates mode --- js/base/Cache.js | 23 ++++++++++++++++++----- js/test/base/test.Cache.js | 17 +++++++++++++++++ php/ArrayCache.php | 6 +++++- php/ArrayCacheBySymbolById.php | 11 ++++++++--- python/ccxtpro/base/cache.py | 10 ++++++++-- 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/js/base/Cache.js b/js/base/Cache.js index 722ff9962f786..d5b07d8e858d2 100644 --- a/js/base/Cache.js +++ b/js/base/Cache.js @@ -22,6 +22,11 @@ class ArrayCache extends BaseCache { constructor (maxSize = undefined) { super (maxSize); + Object.defineProperty (this, 'nestedNewUpdatesBySymbol', { + __proto__: null, // make it invisible + value: false, + writable: true, + }) Object.defineProperty (this, 'newUpdatesBySymbol', { __proto__: null, // make it invisible value: {}, @@ -51,7 +56,10 @@ class ArrayCache extends BaseCache { newUpdatesValue = this.allNewUpdates this.clearAllUpdates = true } else { - newUpdatesValue = (symbol in this.newUpdatesBySymbol) ? this.newUpdatesBySymbol[symbol].size () : 0; + newUpdatesValue = this.newUpdatesBySymbol[symbol]; + if ((newUpdatesValue !== undefined) && this.nestedNewUpdatesBySymbol) { + newUpdatesValue = newUpdatesValue.size + } this.clearUpdatesBySymbol[symbol] = true } @@ -72,13 +80,13 @@ class ArrayCache extends BaseCache { this.push (item) if (this.clearUpdatesBySymbol[item.symbol]) { this.clearUpdatesBySymbol[item.symbol] = false - this.newUpdatesBySymbol[item.symbol].clear () + this.newUpdatesBySymbol[item.symbol] = 0 } if (this.clearAllUpdates) { this.clearAllUpdates = false this.allNewUpdates = 0 } - this.newUpdatesBySymbol[item.symbol].add (item.id) + this.newUpdatesBySymbol[item.symbol] = (this.newUpdatesBySymbol[item.symbol] || 0) + 1 this.allNewUpdates = (this.allNewUpdates || 0) + 1 } } @@ -146,6 +154,7 @@ class ArrayCacheBySymbolById extends ArrayCache { constructor (maxSize = undefined) { super (maxSize) + this.nestedNewUpdatesBySymbol = true Object.defineProperty (this, 'hashmap', { __proto__: null, // make it invisible value: {}, @@ -174,15 +183,19 @@ class ArrayCacheBySymbolById extends ArrayCache { delete this.hashmap[deleteReference.symbol][deleteReference.id] } this.push (item) + if (this.newUpdatesBySymbol[item.symbol] === undefined) { + this.newUpdatesBySymbol[item.symbol] = new Set () + } if (this.clearUpdatesBySymbol[item.symbol]) { this.clearUpdatesBySymbol[item.symbol] = false - this.newUpdatesBySymbol[item.symbol] = 0 + this.newUpdatesBySymbol[item.symbol].clear () } if (this.clearAllUpdates) { this.clearAllUpdates = false this.allNewUpdates = 0 } - this.newUpdatesBySymbol[item.symbol] = (this.newUpdatesBySymbol[item.symbol] || 0) + 1 + // in case an exchange updates the same order id twice + this.newUpdatesBySymbol[item.symbol].add (item.id) this.allNewUpdates = (this.allNewUpdates || 0) + 1 } } diff --git a/js/test/base/test.Cache.js b/js/test/base/test.Cache.js index af6a495319f82..efddb99270fba 100644 --- a/js/test/base/test.Cache.js +++ b/js/test/base/test.Cache.js @@ -204,6 +204,7 @@ let limited = cache.getLimit (symbol, undefined); assert (initialLength === limited); +cache = new ArrayCacheBySymbolById (); let appendItemsLength = 3; for (let i = 0; i < appendItemsLength; i++) { cache.append ({ @@ -240,6 +241,7 @@ limited = cache.getLimit (symbol, undefined); assert (initialLength === limited); +cache = new ArrayCacheBySymbolById (); appendItemsLength = 3; for (let i = 0; i < appendItemsLength; i++) { cache.append ({ @@ -258,6 +260,21 @@ limited = cache.getLimit (symbol, outsideLimit); assert (outsideLimit === limited); + +// ---------------------------------------------------------------------------- +// test ArrayCacheBySymbolById, same order should not increase the limit + +cache = new ArrayCacheBySymbolById (); +symbol = 'BTC/USDT'; + +cache.append ({ 'symbol': symbol, 'id': 'singleId', 'i': 3 }); +cache.append ({ 'symbol': symbol, 'id': 'singleId', 'i': 3 }); + +outsideLimit = 5; +limited = cache.getLimit (symbol, outsideLimit); + +assert (1 == limited); + // ---------------------------------------------------------------------------- // test testLimitArrayCacheByTimestamp limit diff --git a/php/ArrayCache.php b/php/ArrayCache.php index 70a475b70b9c0..c6d5bc604821b 100644 --- a/php/ArrayCache.php +++ b/php/ArrayCache.php @@ -8,6 +8,7 @@ class ArrayCache extends BaseCache { public function __construct($max_size = null) { parent::__construct($max_size); + $this->nested_new_updates_by_symbol = false; $this->new_updates_by_symbol = array(); $this->clear_updates_by_symbol = array(); $this->all_new_updates = 0; @@ -22,9 +23,12 @@ public function getLimit($symbol, $limit) { $this->clear_all_updates = true; } else { $new_updates_value = $this->new_updates_by_symbol[$symbol]; + if (($new_updates_value !== null) && $this->nested_new_updates_by_symbol) { + $new_updates_value = count($new_updates_value); + } $this->clear_updates_by_symbol[$symbol] = true; } - + if ($new_updates_value === null) { return $limit; } diff --git a/php/ArrayCacheBySymbolById.php b/php/ArrayCacheBySymbolById.php index 364b1b7d91f5a..47a47e77b6e1a 100644 --- a/php/ArrayCacheBySymbolById.php +++ b/php/ArrayCacheBySymbolById.php @@ -3,6 +3,7 @@ namespace ccxtpro; use Ds\Deque; +use Ds\Set; class ArrayCacheBySymbolById extends ArrayCache { public $hashmap; @@ -10,6 +11,7 @@ class ArrayCacheBySymbolById extends ArrayCache { public function __construct($max_size = null) { parent::__construct($max_size); + $this->nested_new_updates_by_symbol = true; $this->hashmap = array(); $this->index = new Deque(); } @@ -41,15 +43,18 @@ public function append($item) { $this->deque->push(null); $this->deque[$this->deque->count() - 1] = &$item; $this->index->push($item['id']); + if (!array_key_exists($item['symbol'], $this->new_updates_by_symbol)) { + $this->new_updates_by_symbol[$item['symbol']] = new Set(); + } if ($this->clear_updates_by_symbol[$item['symbol']] ?? false) { $this->clear_updates_by_symbol[$item['symbol']] = false; - $this->new_updates_by_symbol[$item['symbol']] = 0; + $this->new_updates_by_symbol[$item['symbol']]->clear(); } if ($this->clear_all_updates) { $this->clear_all_updates = false; $this->all_new_updates = 0; } - $this->new_updates_by_symbol[$item['symbol']] = ($this->new_updates_by_symbol[$item['symbol']] ?? 0) + 1; + $this->new_updates_by_symbol[$item['symbol']]->add($item['id']); $this->all_new_updates = ($this->all_new_updates ?? 0) + 1; } -} \ No newline at end of file +} diff --git a/python/ccxtpro/base/cache.py b/python/ccxtpro/base/cache.py index 810e0d9852a8b..d1cfaf2e32bcd 100644 --- a/python/ccxtpro/base/cache.py +++ b/python/ccxtpro/base/cache.py @@ -51,6 +51,7 @@ def __getitem__(self, item): class ArrayCache(BaseCache): def __init__(self, max_size=None): super(ArrayCache, self).__init__(max_size) + self._nested_new_updates_by_symbol = False self._new_updates_by_symbol = {} self._clear_updates_by_symbol = {} self._all_new_updates = 0 @@ -63,6 +64,8 @@ def getLimit(self, symbol, limit): self._clear_all_updates = True else: new_updates_value = self._new_updates_by_symbol.get(symbol) + if new_updates_value is not None and self._nested_new_updates_by_symbol: + new_updates_value = len(new_updates_value) self._clear_updates_by_symbol[symbol] = True if new_updates_value is None: @@ -119,6 +122,7 @@ def append(self, item): class ArrayCacheBySymbolById(ArrayCache): def __init__(self, max_size=None): super(ArrayCacheBySymbolById, self).__init__(max_size) + self._nested_new_updates_by_symbol = True self.hashmap = {} self._index = collections.deque([], max_size) @@ -140,11 +144,13 @@ def append(self, item): del self.hashmap[delete_item['symbol']][delete_item['id']] self._deque.append(item) self._index.append(item['id']) + if item['symbol'] not in self._new_updates_by_symbol: + self._new_updates_by_symbol[item['symbol']] = set() if self._clear_updates_by_symbol.get(item['symbol']): self._clear_updates_by_symbol[item['symbol']] = False - self._new_updates_by_symbol[item['symbol']] = 0 + self._new_updates_by_symbol[item['symbol']].clear() if self._clear_all_updates: self._clear_all_updates = False self._all_new_updates = 0 - self._new_updates_by_symbol[item['symbol']] = self._new_updates_by_symbol.get(item['symbol'], 0) + 1 + self._new_updates_by_symbol[item['symbol']].add(item['id']) self._all_new_updates = (self._all_new_updates or 0) + 1 From 52584d05f46e049e980f57656e97d65d4a90734c Mon Sep 17 00:00:00 2001 From: Carlo Revelli Date: Tue, 19 Jul 2022 20:47:46 +0100 Subject: [PATCH 3/3] restore edit --- js/test/base/test.Cache.js | 1 - 1 file changed, 1 deletion(-) diff --git a/js/test/base/test.Cache.js b/js/test/base/test.Cache.js index efddb99270fba..39747b005d35e 100644 --- a/js/test/base/test.Cache.js +++ b/js/test/base/test.Cache.js @@ -241,7 +241,6 @@ limited = cache.getLimit (symbol, undefined); assert (initialLength === limited); -cache = new ArrayCacheBySymbolById (); appendItemsLength = 3; for (let i = 0; i < appendItemsLength; i++) { cache.append ({