Skip to content

Commit

Permalink
Merge pull request ccxt#367 from carlorevelli/carlo-fix-cache-ids
Browse files Browse the repository at this point in the history 
fix cache ids
  • Loading branch information
@frosty00
frosty00 authored Jul 19, 2022
2 parents af6c670 + 52584d0 commit da55f19
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 9 deletions.
19 changes: 16 additions & 3 deletions js/base/Cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ class ArrayCache extends BaseCache {

constructor (maxSize = undefined) {
super (maxSize);
Object.defineProperty (this, 'nestedNewUpdatesBySymbol', {
__proto__: null, // make it invisible
value: false,
writable: true,
})
Object.defineProperty (this, 'newUpdatesBySymbol', {
__proto__: null, // make it invisible
value: {},
Expand All @@ -48,10 +53,13 @@ class ArrayCache extends BaseCache {
let newUpdatesValue = undefined

if (symbol === undefined) {
newUpdatesValue = this.allNewUpdates
newUpdatesValue = this.allNewUpdates
this.clearAllUpdates = true
} else {
newUpdatesValue = this.newUpdatesBySymbol[symbol];
if ((newUpdatesValue !== undefined) && this.nestedNewUpdatesBySymbol) {
newUpdatesValue = newUpdatesValue.size
}
this.clearUpdatesBySymbol[symbol] = true
}

Expand Down Expand Up @@ -146,6 +154,7 @@ class ArrayCacheBySymbolById extends ArrayCache {

constructor (maxSize = undefined) {
super (maxSize)
this.nestedNewUpdatesBySymbol = true
Object.defineProperty (this, 'hashmap', {
__proto__: null, // make it invisible
value: {},
Expand Down Expand Up @@ -174,15 +183,19 @@ class ArrayCacheBySymbolById extends ArrayCache {
delete this.hashmap[deleteReference.symbol][deleteReference.id]
}
this.push (item)
if (this.newUpdatesBySymbol[item.symbol] === undefined) {
this.newUpdatesBySymbol[item.symbol] = new Set ()
}
if (this.clearUpdatesBySymbol[item.symbol]) {
this.clearUpdatesBySymbol[item.symbol] = false
this.newUpdatesBySymbol[item.symbol] = 0
this.newUpdatesBySymbol[item.symbol].clear ()
}
if (this.clearAllUpdates) {
this.clearAllUpdates = false
this.allNewUpdates = 0
}
this.newUpdatesBySymbol[item.symbol] = (this.newUpdatesBySymbol[item.symbol] || 0) + 1
// in case an exchange updates the same order id twice
this.newUpdatesBySymbol[item.symbol].add (item.id)
this.allNewUpdates = (this.allNewUpdates || 0) + 1
}
}
Expand Down
16 changes: 16 additions & 0 deletions js/test/base/test.Cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ let limited = cache.getLimit (symbol, undefined);

assert (initialLength === limited);

cache = new ArrayCacheBySymbolById ();
let appendItemsLength = 3;
for (let i = 0; i < appendItemsLength; i++) {
cache.append ({
Expand Down Expand Up @@ -258,6 +259,21 @@ limited = cache.getLimit (symbol, outsideLimit);

assert (outsideLimit === limited);


// ----------------------------------------------------------------------------
// test ArrayCacheBySymbolById, same order should not increase the limit

cache = new ArrayCacheBySymbolById ();
symbol = 'BTC/USDT';

cache.append ({ 'symbol': symbol, 'id': 'singleId', 'i': 3 });
cache.append ({ 'symbol': symbol, 'id': 'singleId', 'i': 3 });

outsideLimit = 5;
limited = cache.getLimit (symbol, outsideLimit);

assert (1 == limited);

// ----------------------------------------------------------------------------
// test testLimitArrayCacheByTimestamp limit

Expand Down
6 changes: 5 additions & 1 deletion php/ArrayCache.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ class ArrayCache extends BaseCache {

public function __construct($max_size = null) {
parent::__construct($max_size);
$this->nested_new_updates_by_symbol = false;
$this->new_updates_by_symbol = array();
$this->clear_updates_by_symbol = array();
$this->all_new_updates = 0;
Expand All @@ -22,9 +23,12 @@ public function getLimit($symbol, $limit) {
$this->clear_all_updates = true;
} else {
$new_updates_value = $this->new_updates_by_symbol[$symbol];
if (($new_updates_value !== null) && $this->nested_new_updates_by_symbol) {
$new_updates_value = count($new_updates_value);
}
$this->clear_updates_by_symbol[$symbol] = true;
}

if ($new_updates_value === null) {
return $limit;
}
Expand Down
11 changes: 8 additions & 3 deletions php/ArrayCacheBySymbolById.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
namespace ccxtpro;

use Ds\Deque;
use Ds\Set;

class ArrayCacheBySymbolById extends ArrayCache {
public $hashmap;
private $index;

public function __construct($max_size = null) {
parent::__construct($max_size);
$this->nested_new_updates_by_symbol = true;
$this->hashmap = array();
$this->index = new Deque();
}
Expand Down Expand Up @@ -41,15 +43,18 @@ public function append($item) {
$this->deque->push(null);
$this->deque[$this->deque->count() - 1] = &$item;
$this->index->push($item['id']);
if (!array_key_exists($item['symbol'], $this->new_updates_by_symbol)) {
$this->new_updates_by_symbol[$item['symbol']] = new Set();
}
if ($this->clear_updates_by_symbol[$item['symbol']] ?? false) {
$this->clear_updates_by_symbol[$item['symbol']] = false;
$this->new_updates_by_symbol[$item['symbol']] = 0;
$this->new_updates_by_symbol[$item['symbol']]->clear();
}
if ($this->clear_all_updates) {
$this->clear_all_updates = false;
$this->all_new_updates = 0;
}
$this->new_updates_by_symbol[$item['symbol']] = ($this->new_updates_by_symbol[$item['symbol']] ?? 0) + 1;
$this->new_updates_by_symbol[$item['symbol']]->add($item['id']);
$this->all_new_updates = ($this->all_new_updates ?? 0) + 1;
}
}
}
10 changes: 8 additions & 2 deletions python/ccxtpro/base/cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def __getitem__(self, item):
class ArrayCache(BaseCache):
def __init__(self, max_size=None):
super(ArrayCache, self).__init__(max_size)
self._nested_new_updates_by_symbol = False
self._new_updates_by_symbol = {}
self._clear_updates_by_symbol = {}
self._all_new_updates = 0
Expand All @@ -63,6 +64,8 @@ def getLimit(self, symbol, limit):
self._clear_all_updates = True
else:
new_updates_value = self._new_updates_by_symbol.get(symbol)
if new_updates_value is not None and self._nested_new_updates_by_symbol:
new_updates_value = len(new_updates_value)
self._clear_updates_by_symbol[symbol] = True

if new_updates_value is None:
Expand Down Expand Up @@ -119,6 +122,7 @@ def append(self, item):
class ArrayCacheBySymbolById(ArrayCache):
def __init__(self, max_size=None):
super(ArrayCacheBySymbolById, self).__init__(max_size)
self._nested_new_updates_by_symbol = True
self.hashmap = {}
self._index = collections.deque([], max_size)

Expand All @@ -140,11 +144,13 @@ def append(self, item):
del self.hashmap[delete_item['symbol']][delete_item['id']]
self._deque.append(item)
self._index.append(item['id'])
if item['symbol'] not in self._new_updates_by_symbol:
self._new_updates_by_symbol[item['symbol']] = set()
if self._clear_updates_by_symbol.get(item['symbol']):
self._clear_updates_by_symbol[item['symbol']] = False
self._new_updates_by_symbol[item['symbol']] = 0
self._new_updates_by_symbol[item['symbol']].clear()
if self._clear_all_updates:
self._clear_all_updates = False
self._all_new_updates = 0
self._new_updates_by_symbol[item['symbol']] = self._new_updates_by_symbol.get(item['symbol'], 0) + 1
self._new_updates_by_symbol[item['symbol']].add(item['id'])
self._all_new_updates = (self._all_new_updates or 0) + 1

0 comments on commit da55f19

Please sign in to comment.