From aaec10f7298a3fcbaaf0bf5c5c3317a710223eb9 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:03:12 -0400 Subject: [PATCH 01/41] Revert "Add Orderbook Mid Price Cache (#2289)" (backport #2333) (#2334) Co-authored-by: Adam Fraser --- .../caches/orderbook-mid-prices-cache.test.ts | 106 ----------------- .../src/caches/orderbook-mid-prices-cache.ts | 107 ------------------ indexer/packages/redis/src/caches/scripts.ts | 4 - indexer/packages/redis/src/index.ts | 1 - .../redis/src/scripts/add_market_price.lua | 17 --- .../src/scripts/get_market_median_price.lua | 23 ---- .../__tests__/lib/candles-generator.test.ts | 53 ++++----- .../ender/src/lib/candles-generator.ts | 6 +- .../tasks/cache-orderbook-mid-prices.test.ts | 98 ---------------- indexer/services/roundtable/src/config.ts | 4 - indexer/services/roundtable/src/index.ts | 9 -- .../src/tasks/cache-orderbook-mid-prices.ts | 40 ------- 12 files changed, 30 insertions(+), 438 deletions(-) delete mode 100644 indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts delete mode 100644 indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts delete mode 100644 indexer/packages/redis/src/scripts/add_market_price.lua delete mode 100644 indexer/packages/redis/src/scripts/get_market_median_price.lua delete mode 100644 indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts delete mode 100644 indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts diff --git a/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts deleted file mode 100644 index 70ed134e67..0000000000 --- a/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { deleteAllAsync } from '../../src/helpers/redis'; -import { redis as client } from '../helpers/utils'; -import { - setPrice, - getMedianPrice, - ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX, -} from '../../src/caches/orderbook-mid-prices-cache'; - -describe('orderbook-mid-prices-cache', () => { - const ticker: string = 'BTC-USD'; - - beforeEach(async () => { - await deleteAllAsync(client); - }); - - afterEach(async () => { - await deleteAllAsync(client); - }); - - describe('setPrice', () => { - it('sets a price for a ticker', async () => { - await setPrice(client, ticker, '50000'); - - await client.zrange( - `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`, - 0, - -1, - (_: any, response: string[]) => { - expect(response[0]).toBe('50000'); - }, - ); - }); - - it('sets multiple prices for a ticker', async () => { - await Promise.all([ - setPrice(client, ticker, '50000'), - setPrice(client, ticker, '51000'), - setPrice(client, ticker, '49000'), - ]); - - await client.zrange( - `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`, - 0, - -1, - (_: any, response: string[]) => { - expect(response).toEqual(['49000', '50000', '51000']); - }, - ); - }); - }); - - describe('getMedianPrice', () => { - it('returns null when no prices are set', async () => { - const result = await getMedianPrice(client, ticker); - expect(result).toBeNull(); - }); - - it('returns the median price for odd number of prices', async () => { - await Promise.all([ - setPrice(client, ticker, '50000'), - setPrice(client, ticker, '51000'), - setPrice(client, ticker, '49000'), - ]); - - const result = await getMedianPrice(client, ticker); - expect(result).toBe('50000'); - }); - - it('returns the median price for even number of prices', async () => { - await Promise.all([ - setPrice(client, ticker, '50000'), - setPrice(client, ticker, '51000'), - setPrice(client, ticker, '49000'), - setPrice(client, ticker, '52000'), - ]); - - const result = await getMedianPrice(client, ticker); - expect(result).toBe('50500'); - }); - - it('returns the correct median price after 5 seconds', async () => { - jest.useFakeTimers(); - - const nowSeconds = Math.floor(Date.now() / 1000); - jest.setSystemTime(nowSeconds * 1000); - - await Promise.all([ - setPrice(client, ticker, '50000'), - setPrice(client, ticker, '51000'), - ]); - - jest.advanceTimersByTime(6000); // Advance time by 6 seconds - await Promise.all([ - setPrice(client, ticker, '49000'), - setPrice(client, ticker, '48000'), - setPrice(client, ticker, '52000'), - setPrice(client, ticker, '53000'), - ]); - - const result = await getMedianPrice(client, ticker); - expect(result).toBe('50500'); - - jest.useRealTimers(); - }); - }); -}); diff --git a/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts deleted file mode 100644 index f2857b70e9..0000000000 --- a/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { Callback, RedisClient } from 'redis'; - -import { - addMarketPriceScript, - getMarketMedianScript, -} from './scripts'; - -// Cache of orderbook prices for each clob pair -// Each price is cached for a 5 second window and in a ZSET -export const ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX: string = 'v4/orderbook_mid_prices/'; - -/** - * Generates a cache key for a given ticker's orderbook mid price. - * @param ticker The ticker symbol - * @returns The cache key string - */ -function getOrderbookMidPriceCacheKey(ticker: string): string { - return `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`; -} - -/** - * Adds a price to the market prices cache for a given ticker. - * Uses a Lua script to add the price with a timestamp to a sorted set in Redis. - * @param client The Redis client - * @param ticker The ticker symbol - * @param price The price to be added - * @returns A promise that resolves when the operation is complete - */ -export async function setPrice( - client: RedisClient, - ticker: string, - price: string, -): Promise { - // Number of keys for the lua script. - const numKeys: number = 1; - - let evalAsync: ( - marketCacheKey: string, - ) => Promise = (marketCacheKey) => { - - return new Promise((resolve, reject) => { - const callback: Callback = ( - err: Error | null, - ) => { - if (err) { - return reject(err); - } - return resolve(); - }; - - const nowSeconds = Math.floor(Date.now() / 1000); // Current time in seconds - client.evalsha( - addMarketPriceScript.hash, - numKeys, - marketCacheKey, - price, - nowSeconds, - callback, - ); - - }); - }; - evalAsync = evalAsync.bind(client); - - return evalAsync( - getOrderbookMidPriceCacheKey(ticker), - ); -} - -/** - * Retrieves the median price for a given ticker from the cache. - * Uses a Lua script to calculate the median price from the sorted set in Redis. - * @param client The Redis client - * @param ticker The ticker symbol - * @returns A promise that resolves with the median price as a string, or null if not found - */ -export async function getMedianPrice(client: RedisClient, ticker: string): Promise { - let evalAsync: ( - marketCacheKey: string, - ) => Promise = ( - marketCacheKey, - ) => { - return new Promise((resolve, reject) => { - const callback: Callback = ( - err: Error | null, - results: string, - ) => { - if (err) { - return reject(err); - } - return resolve(results); - }; - - client.evalsha( - getMarketMedianScript.hash, - 1, - marketCacheKey, - callback, - ); - }); - }; - evalAsync = evalAsync.bind(client); - - return evalAsync( - getOrderbookMidPriceCacheKey(ticker), - ); -} diff --git a/indexer/packages/redis/src/caches/scripts.ts b/indexer/packages/redis/src/caches/scripts.ts index f4f74bffd5..3e1032c6f2 100644 --- a/indexer/packages/redis/src/caches/scripts.ts +++ b/indexer/packages/redis/src/caches/scripts.ts @@ -63,8 +63,6 @@ export const removeOrderScript: LuaScript = newLuaScript('removeOrder', '../scri export const addCanceledOrderIdScript: LuaScript = newLuaScript('addCanceledOrderId', '../scripts/add_canceled_order_id.lua'); export const addStatefulOrderUpdateScript: LuaScript = newLuaScript('addStatefulOrderUpdate', '../scripts/add_stateful_order_update.lua'); export const removeStatefulOrderUpdateScript: LuaScript = newLuaScript('removeStatefulOrderUpdate', '../scripts/remove_stateful_order_update.lua'); -export const addMarketPriceScript: LuaScript = newLuaScript('addMarketPrice', '../scripts/add_market_price.lua'); -export const getMarketMedianScript: LuaScript = newLuaScript('getMarketMedianPrice', '../scripts/get_market_median_price.lua'); export const allLuaScripts: LuaScript[] = [ deleteZeroPriceLevelScript, @@ -77,6 +75,4 @@ export const allLuaScripts: LuaScript[] = [ addCanceledOrderIdScript, addStatefulOrderUpdateScript, removeStatefulOrderUpdateScript, - addMarketPriceScript, - getMarketMedianScript, ]; diff --git a/indexer/packages/redis/src/index.ts b/indexer/packages/redis/src/index.ts index 5c7aeed103..2ce64e9b88 100644 --- a/indexer/packages/redis/src/index.ts +++ b/indexer/packages/redis/src/index.ts @@ -12,7 +12,6 @@ export * as CanceledOrdersCache from './caches/canceled-orders-cache'; export * as StatefulOrderUpdatesCache from './caches/stateful-order-updates-cache'; export * as StateFilledQuantumsCache from './caches/state-filled-quantums-cache'; export * as LeaderboardPnlProcessedCache from './caches/leaderboard-processed-cache'; -export * as OrderbookMidPricesCache from './caches/orderbook-mid-prices-cache'; export { placeOrder } from './caches/place-order'; export { removeOrder } from './caches/remove-order'; export { updateOrder } from './caches/update-order'; diff --git a/indexer/packages/redis/src/scripts/add_market_price.lua b/indexer/packages/redis/src/scripts/add_market_price.lua deleted file mode 100644 index 0e1467bb31..0000000000 --- a/indexer/packages/redis/src/scripts/add_market_price.lua +++ /dev/null @@ -1,17 +0,0 @@ --- Key for the ZSET storing price data -local priceCacheKey = KEYS[1] --- Price to be added -local price = tonumber(ARGV[1]) --- Current timestamp -local nowSeconds = tonumber(ARGV[2]) --- Time window (5 seconds) -local fiveSeconds = 5 - --- 1. Add the price to the sorted set (score is the current timestamp) -redis.call("zadd", priceCacheKey, nowSeconds, price) - --- 2. Remove any entries older than 5 seconds -local cutoffTime = nowSeconds - fiveSeconds -redis.call("zremrangebyscore", priceCacheKey, "-inf", cutoffTime) - -return true \ No newline at end of file diff --git a/indexer/packages/redis/src/scripts/get_market_median_price.lua b/indexer/packages/redis/src/scripts/get_market_median_price.lua deleted file mode 100644 index 281da9bed8..0000000000 --- a/indexer/packages/redis/src/scripts/get_market_median_price.lua +++ /dev/null @@ -1,23 +0,0 @@ --- Key for the sorted set storing price data -local priceCacheKey = KEYS[1] - --- Get all the prices from the sorted set (ascending order) -local prices = redis.call('zrange', priceCacheKey, 0, -1) - --- If no prices are found, return nil -if #prices == 0 then - return nil -end - --- Calculate the middle index -local middle = math.floor(#prices / 2) - --- Calculate median -if #prices % 2 == 0 then - -- If even, return the average of the two middle elements - local median = (tonumber(prices[middle]) + tonumber(prices[middle + 1])) / 2 - return tostring(median) -else - -- If odd, return the middle element - return prices[middle + 1] -end diff --git a/indexer/services/ender/__tests__/lib/candles-generator.test.ts b/indexer/services/ender/__tests__/lib/candles-generator.test.ts index d58ebd0d2a..d1c257b80b 100644 --- a/indexer/services/ender/__tests__/lib/candles-generator.test.ts +++ b/indexer/services/ender/__tests__/lib/candles-generator.test.ts @@ -18,6 +18,7 @@ import { testMocks, Transaction, helpers, + OrderSide, } from '@dydxprotocol-indexer/postgres'; import { CandleMessage, CandleMessage_Resolution } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -31,11 +32,9 @@ import { KafkaPublisher } from '../../src/lib/kafka-publisher'; import { ConsolidatedKafkaEvent } from '../../src/lib/types'; import { defaultTradeContent, defaultTradeKafkaEvent } from '../helpers/constants'; import { contentToSingleTradeMessage, createConsolidatedKafkaEventFromTrade } from '../helpers/kafka-publisher-helpers'; +import { updatePriceLevel } from '../helpers/redis-helpers'; import { redisClient } from '../../src/helpers/redis/redis-controller'; -import { - redis, - OrderbookMidPricesCache, -} from '@dydxprotocol-indexer/redis'; +import { redis } from '@dydxprotocol-indexer/redis'; describe('candleHelper', () => { beforeAll(async () => { @@ -114,12 +113,9 @@ describe('candleHelper', () => { defaultTradeKafkaEvent2, ]); - const ticker = 'BTC-USD'; - await Promise.all([ - OrderbookMidPricesCache.setPrice(redisClient, ticker, '100000'), - OrderbookMidPricesCache.setPrice(redisClient, ticker, '105000'), - OrderbookMidPricesCache.setPrice(redisClient, ticker, '110000'), - ]); + // Create Orderbook levels to set orderbookMidPrice open & close + await updatePriceLevel('BTC-USD', '100000', OrderSide.BUY); + await updatePriceLevel('BTC-USD', '110000', OrderSide.SELL); await runUpdateCandles(publisher); @@ -159,12 +155,8 @@ describe('candleHelper', () => { defaultTradeKafkaEvent2, ]); - const ticker = 'BTC-USD'; - await Promise.all([ - OrderbookMidPricesCache.setPrice(redisClient, ticker, '80000'), - OrderbookMidPricesCache.setPrice(redisClient, ticker, '81000'), - OrderbookMidPricesCache.setPrice(redisClient, ticker, '80500'), - ]); + await updatePriceLevel('BTC-USD', '80000', OrderSide.BUY); + await updatePriceLevel('BTC-USD', '81000', OrderSide.SELL); // Create Perpetual Position to set open position const openInterest: string = '100'; @@ -435,7 +427,9 @@ describe('candleHelper', () => { containsKafkaMessages: boolean = true, orderbookMidPrice: number, ) => { - await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', orderbookMidPrice.toFixed()); + const midPriceSpread = 10; + await updatePriceLevel('BTC-USD', String(orderbookMidPrice + midPriceSpread), OrderSide.SELL); + await updatePriceLevel('BTC-USD', String(orderbookMidPrice - midPriceSpread), OrderSide.BUY); if (initialCandle !== undefined) { await CandleTable.create(initialCandle); @@ -500,7 +494,9 @@ describe('candleHelper', () => { ); await startCandleCache(); - await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); + // Update Orderbook levels + await updatePriceLevel('BTC-USD', '10010', OrderSide.SELL); + await updatePriceLevel('BTC-USD', '10000', OrderSide.BUY); const publisher: KafkaPublisher = new KafkaPublisher(); publisher.addEvents([ @@ -598,7 +594,9 @@ describe('candleHelper', () => { ); await startCandleCache(); - await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); + // Update Orderbook levels + await updatePriceLevel('BTC-USD', '10010', OrderSide.SELL); + await updatePriceLevel('BTC-USD', '10000', OrderSide.BUY); const publisher: KafkaPublisher = new KafkaPublisher(); publisher.addEvents([]); @@ -662,19 +660,22 @@ describe('candleHelper', () => { }); it('successfully creates an orderbook price map for each market', async () => { - await Promise.all([ - OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '105000'), - OrderbookMidPricesCache.setPrice(redisClient, 'ISO-USD', '115000'), - OrderbookMidPricesCache.setPrice(redisClient, 'ETH-USD', '150000'), - ]); + await updatePriceLevel('BTC-USD', '100000', OrderSide.BUY); + await updatePriceLevel('BTC-USD', '110000', OrderSide.SELL); + + await updatePriceLevel('ISO-USD', '110000', OrderSide.BUY); + await updatePriceLevel('ISO-USD', '120000', OrderSide.SELL); + + await updatePriceLevel('ETH-USD', '100000', OrderSide.BUY); + await updatePriceLevel('ETH-USD', '200000', OrderSide.SELL); const map = await getOrderbookMidPriceMap(); expect(map).toEqual({ 'BTC-USD': '105000', 'ETH-USD': '150000', 'ISO-USD': '115000', - 'ISO2-USD': null, - 'SHIB-USD': null, + 'ISO2-USD': undefined, + 'SHIB-USD': undefined, }); }); }); diff --git a/indexer/services/ender/src/lib/candles-generator.ts b/indexer/services/ender/src/lib/candles-generator.ts index b232a66eb0..d7dd7bba34 100644 --- a/indexer/services/ender/src/lib/candles-generator.ts +++ b/indexer/services/ender/src/lib/candles-generator.ts @@ -20,7 +20,7 @@ import { TradeMessageContents, helpers, } from '@dydxprotocol-indexer/postgres'; -import { OrderbookMidPricesCache } from '@dydxprotocol-indexer/redis'; +import { OrderbookLevelsCache } from '@dydxprotocol-indexer/redis'; import { CandleMessage } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; import _ from 'lodash'; @@ -538,9 +538,9 @@ export async function getOrderbookMidPriceMap(): Promise<{ [ticker: string]: Ord const perpetualMarkets = Object.values(perpetualMarketRefresher.getPerpetualMarketsMap()); const promises = perpetualMarkets.map(async (perpetualMarket: PerpetualMarketFromDatabase) => { - const price = await OrderbookMidPricesCache.getMedianPrice( - redisClient, + const price = await OrderbookLevelsCache.getOrderBookMidPrice( perpetualMarket.ticker, + redisClient, ); return { [perpetualMarket.ticker]: price === undefined ? undefined : price }; }); diff --git a/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts b/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts deleted file mode 100644 index cd0eee3970..0000000000 --- a/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { - dbHelpers, - testConstants, - testMocks, -} from '@dydxprotocol-indexer/postgres'; -import { - OrderbookMidPricesCache, - OrderbookLevelsCache, - redis, -} from '@dydxprotocol-indexer/redis'; -import { redisClient } from '../../src/helpers/redis'; -import runTask from '../../src/tasks/cache-orderbook-mid-prices'; - -jest.mock('@dydxprotocol-indexer/base', () => ({ - ...jest.requireActual('@dydxprotocol-indexer/base'), - logger: { - info: jest.fn(), - error: jest.fn(), - }, -})); - -jest.mock('@dydxprotocol-indexer/redis', () => ({ - ...jest.requireActual('@dydxprotocol-indexer/redis'), - OrderbookLevelsCache: { - getOrderBookMidPrice: jest.fn(), - }, -})); - -describe('cache-orderbook-mid-prices', () => { - beforeAll(async () => { - await dbHelpers.migrate(); - }); - - beforeEach(async () => { - await dbHelpers.clearData(); - await redis.deleteAllAsync(redisClient); - await testMocks.seedData(); - }); - - afterAll(async () => { - await dbHelpers.teardown(); - jest.resetAllMocks(); - }); - - it('caches mid prices for all markets', async () => { - const market1 = testConstants.defaultPerpetualMarket; - const market2 = testConstants.defaultPerpetualMarket2; - - const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); - mockGetOrderBookMidPrice.mockResolvedValueOnce('100.5'); // For market1 - mockGetOrderBookMidPrice.mockResolvedValueOnce('200.75'); // For market2 - - await runTask(); - - // Check if the mock was called with the correct arguments - expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market1.ticker, redisClient); - expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market2.ticker, redisClient); - - // Check if prices were cached correctly - const price1 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market1.ticker); - const price2 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market2.ticker); - - expect(price1).toBe('100.5'); - expect(price2).toBe('200.75'); - }); - - it('handles undefined prices', async () => { - const market = testConstants.defaultPerpetualMarket; - - const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); - mockGetOrderBookMidPrice.mockResolvedValueOnce(undefined); - - await runTask(); - - const price = await OrderbookMidPricesCache.getMedianPrice(redisClient, market.ticker); - expect(price).toBeNull(); - - // Check that a log message was created - expect(jest.requireMock('@dydxprotocol-indexer/base').logger.info).toHaveBeenCalledWith({ - at: 'cache-orderbook-mid-prices#runTask', - message: `undefined price for ${market.ticker}`, - }); - }); - - it('handles errors', async () => { - // Mock OrderbookLevelsCache.getOrderBookMidPrice to throw an error - const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); - mockGetOrderBookMidPrice.mockRejectedValueOnce(new Error('Test error')); - - await runTask(); - - expect(jest.requireMock('@dydxprotocol-indexer/base').logger.error).toHaveBeenCalledWith({ - at: 'cache-orderbook-mid-prices#runTask', - message: 'Test error', - error: expect.any(Error), - }); - }); -}); diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index 9f0f00c487..a8ec2cb87b 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -60,7 +60,6 @@ export const configSchema = { LOOPS_ENABLED_UPDATE_WALLET_TOTAL_VOLUME: parseBoolean({ default: true }), LOOPS_ENABLED_UPDATE_AFFILIATE_INFO: parseBoolean({ default: true }), LOOPS_ENABLED_DELETE_OLD_FIREBASE_NOTIFICATION_TOKENS: parseBoolean({ default: true }), - LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES: parseBoolean({ default: true }), // Loop Timing LOOPS_INTERVAL_MS_MARKET_UPDATER: parseInteger({ @@ -138,9 +137,6 @@ export const configSchema = { LOOPS_INTERVAL_MS_DELETE_FIREBASE_NOTIFICATION_TOKENS_MONTHLY: parseInteger({ default: 30 * ONE_DAY_IN_MILLISECONDS, }), - LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES: parseInteger({ - default: ONE_SECOND_IN_MILLISECONDS, - }), // Start delay START_DELAY_ENABLED: parseBoolean({ default: true }), diff --git a/indexer/services/roundtable/src/index.ts b/indexer/services/roundtable/src/index.ts index bfdee334c7..f52903ac19 100644 --- a/indexer/services/roundtable/src/index.ts +++ b/indexer/services/roundtable/src/index.ts @@ -10,7 +10,6 @@ import { connect as connectToRedis, } from './helpers/redis'; import aggregateTradingRewardsTasks from './tasks/aggregate-trading-rewards'; -import cacheOrderbookMidPrices from './tasks/cache-orderbook-mid-prices'; import cancelStaleOrdersTask from './tasks/cancel-stale-orders'; import createLeaderboardTask from './tasks/create-leaderboard'; import createPnlTicksTask from './tasks/create-pnl-ticks'; @@ -273,14 +272,6 @@ async function start(): Promise { ); } - if (config.LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES) { - startLoop( - cacheOrderbookMidPrices, - 'cache_orderbook_mid_prices', - config.LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES, - ); - } - logger.info({ at: 'index', message: 'Successfully started', diff --git a/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts b/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts deleted file mode 100644 index 644f50df6f..0000000000 --- a/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - logger, -} from '@dydxprotocol-indexer/base'; -import { - PerpetualMarketFromDatabase, - PerpetualMarketTable, -} from '@dydxprotocol-indexer/postgres'; -import { - OrderbookMidPricesCache, - OrderbookLevelsCache, -} from '@dydxprotocol-indexer/redis'; - -import { redisClient } from '../helpers/redis'; - -/** - * Updates OrderbookMidPricesCache with current orderbook mid price for each market - */ -export default async function runTask(): Promise { - const markets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll({}, []); - - for (const market of markets) { - try { - const price = await OrderbookLevelsCache.getOrderBookMidPrice(market.ticker, redisClient); - if (price) { - await OrderbookMidPricesCache.setPrice(redisClient, market.ticker, price); - } else { - logger.info({ - at: 'cache-orderbook-mid-prices#runTask', - message: `undefined price for ${market.ticker}`, - }); - } - } catch (error) { - logger.error({ - at: 'cache-orderbook-mid-prices#runTask', - message: error.message, - error, - }); - } - } -} From ce8f48415259afa50e3e6ba4a4ef8679aa31510e Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:05:53 -0400 Subject: [PATCH 02/41] Add Orderbook Mid Price Cache (backport #2338) (#2340) Co-authored-by: Adam Fraser --- .../caches/orderbook-mid-prices-cache.test.ts | 136 ++++++++++++++++++ .../src/caches/orderbook-mid-prices-cache.ts | 127 ++++++++++++++++ indexer/packages/redis/src/caches/scripts.ts | 4 + indexer/packages/redis/src/index.ts | 1 + .../redis/src/scripts/add_market_price.lua | 17 +++ .../src/scripts/get_market_median_price.lua | 22 +++ .../__tests__/lib/candles-generator.test.ts | 53 ++++--- .../ender/src/lib/candles-generator.ts | 6 +- .../tasks/cache-orderbook-mid-prices.test.ts | 98 +++++++++++++ indexer/services/roundtable/src/config.ts | 4 + indexer/services/roundtable/src/index.ts | 9 ++ .../src/tasks/cache-orderbook-mid-prices.ts | 40 ++++++ 12 files changed, 487 insertions(+), 30 deletions(-) create mode 100644 indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts create mode 100644 indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts create mode 100644 indexer/packages/redis/src/scripts/add_market_price.lua create mode 100644 indexer/packages/redis/src/scripts/get_market_median_price.lua create mode 100644 indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts create mode 100644 indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts diff --git a/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts new file mode 100644 index 0000000000..5dfd662f68 --- /dev/null +++ b/indexer/packages/redis/__tests__/caches/orderbook-mid-prices-cache.test.ts @@ -0,0 +1,136 @@ +import { deleteAllAsync } from '../../src/helpers/redis'; +import { redis as client } from '../helpers/utils'; +import { + setPrice, + getMedianPrice, + ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX, +} from '../../src/caches/orderbook-mid-prices-cache'; + +describe('orderbook-mid-prices-cache', () => { + const ticker: string = 'BTC-USD'; + + beforeEach(async () => { + await deleteAllAsync(client); + }); + + describe('setPrice', () => { + it('sets a price for a ticker', async () => { + await setPrice(client, ticker, '50000'); + + await client.zrange( + `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`, + 0, + -1, + (_: any, response: string[]) => { + expect(response[0]).toBe('50000'); + }, + ); + }); + + it('sets multiple prices for a ticker', async () => { + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + setPrice(client, ticker, '49000'), + ]); + + await client.zrange( + `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`, + 0, + -1, + (_: any, response: string[]) => { + expect(response).toEqual(['49000', '50000', '51000']); + }, + ); + }); + }); + + describe('getMedianPrice', () => { + it('returns null when no prices are set', async () => { + const result = await getMedianPrice(client, ticker); + expect(result).toBeNull(); + }); + + it('returns the median price for odd number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + setPrice(client, ticker, '49000'), + ]); + + const result = await getMedianPrice(client, ticker); + expect(result).toBe('50000'); + }); + + it('returns the median price for even number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + setPrice(client, ticker, '49000'), + setPrice(client, ticker, '52000'), + ]); + + const result = await getMedianPrice(client, ticker); + expect(result).toBe('50500'); + }); + + it('returns the correct median price after 5 seconds', async () => { + jest.useFakeTimers(); + + const nowSeconds = Math.floor(Date.now() / 1000); + jest.setSystemTime(nowSeconds * 1000); + + await Promise.all([ + setPrice(client, ticker, '50000'), + setPrice(client, ticker, '51000'), + ]); + + jest.advanceTimersByTime(6000); // Advance time by 6 seconds + await Promise.all([ + setPrice(client, ticker, '49000'), + setPrice(client, ticker, '48000'), + setPrice(client, ticker, '52000'), + setPrice(client, ticker, '53000'), + ]); + + const result = await getMedianPrice(client, ticker); + expect(result).toBe('50500'); + + jest.useRealTimers(); + }); + + it('returns the correct median price for small numbers with even number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '0.00000000002345'), + setPrice(client, ticker, '0.00000000002346'), + ]); + + const midPrice1 = await getMedianPrice(client, ticker); + expect(midPrice1).toEqual('0.000000000023455'); + }); + + it('returns the correct median price for small numbers with odd number of prices', async () => { + await Promise.all([ + setPrice(client, ticker, '0.00000000001'), + setPrice(client, ticker, '0.00000000002'), + setPrice(client, ticker, '0.00000000003'), + setPrice(client, ticker, '0.00000000004'), + setPrice(client, ticker, '0.00000000005'), + ]); + + const midPrice1 = await getMedianPrice(client, ticker); + expect(midPrice1).toEqual('0.00000000003'); + + await deleteAllAsync(client); + + await Promise.all([ + setPrice(client, ticker, '0.00000847007'), + setPrice(client, ticker, '0.00000847006'), + setPrice(client, ticker, '0.00000847008'), + ]); + + const midPrice2 = await getMedianPrice(client, ticker); + expect(midPrice2).toEqual('0.00000847007'); + }); + }); +}); diff --git a/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts new file mode 100644 index 0000000000..ece95a3ca2 --- /dev/null +++ b/indexer/packages/redis/src/caches/orderbook-mid-prices-cache.ts @@ -0,0 +1,127 @@ +import Big from 'big.js'; +import { Callback, RedisClient } from 'redis'; + +import { + addMarketPriceScript, + getMarketMedianScript, +} from './scripts'; + +// Cache of orderbook prices for each clob pair +// Each price is cached for a 5 second window and in a ZSET +export const ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX: string = 'v4/orderbook_mid_prices/'; + +/** + * Generates a cache key for a given ticker's orderbook mid price. + * @param ticker The ticker symbol + * @returns The cache key string + */ +function getOrderbookMidPriceCacheKey(ticker: string): string { + return `${ORDERBOOK_MID_PRICES_CACHE_KEY_PREFIX}${ticker}`; +} + +/** + * Adds a price to the market prices cache for a given ticker. + * Uses a Lua script to add the price with a timestamp to a sorted set in Redis. + * @param client The Redis client + * @param ticker The ticker symbol + * @param price The price to be added + * @returns A promise that resolves when the operation is complete + */ +export async function setPrice( + client: RedisClient, + ticker: string, + price: string, +): Promise { + // Number of keys for the lua script. + const numKeys: number = 1; + + let evalAsync: ( + marketCacheKey: string, + ) => Promise = (marketCacheKey) => { + + return new Promise((resolve, reject) => { + const callback: Callback = ( + err: Error | null, + ) => { + if (err) { + return reject(err); + } + return resolve(); + }; + + const nowSeconds = Math.floor(Date.now() / 1000); // Current time in seconds + client.evalsha( + addMarketPriceScript.hash, + numKeys, + marketCacheKey, + price, + nowSeconds, + callback, + ); + + }); + }; + evalAsync = evalAsync.bind(client); + + return evalAsync( + getOrderbookMidPriceCacheKey(ticker), + ); +} + +/** + * Retrieves the median price for a given ticker from the cache. + * Uses a Lua script to fetch either the middle element (for odd number of prices) + * or the two middle elements (for even number of prices) from a sorted set in Redis. + * If two middle elements are returned, their average is calculated in JavaScript. + * @param client The Redis client + * @param ticker The ticker symbol + * @returns A promise that resolves with the median price as a string, or null if not found + */ +export async function getMedianPrice(client: RedisClient, ticker: string): Promise { + let evalAsync: ( + marketCacheKey: string, + ) => Promise = ( + marketCacheKey, + ) => { + return new Promise((resolve, reject) => { + const callback: Callback = ( + err: Error | null, + results: string[], + ) => { + if (err) { + return reject(err); + } + return resolve(results); + }; + + client.evalsha( + getMarketMedianScript.hash, + 1, + marketCacheKey, + callback, + ); + }); + }; + evalAsync = evalAsync.bind(client); + + const prices = await evalAsync( + getOrderbookMidPriceCacheKey(ticker), + ); + + if (!prices || prices.length === 0) { + return null; + } + + if (prices.length === 1) { + return Big(prices[0]).toFixed(); + } + + if (prices.length === 2) { + const [price1, price2] = prices.map((price) => { + return Big(price); + }); + return price1.plus(price2).div(2).toFixed(); + } + + return null; +} diff --git a/indexer/packages/redis/src/caches/scripts.ts b/indexer/packages/redis/src/caches/scripts.ts index 3e1032c6f2..f4f74bffd5 100644 --- a/indexer/packages/redis/src/caches/scripts.ts +++ b/indexer/packages/redis/src/caches/scripts.ts @@ -63,6 +63,8 @@ export const removeOrderScript: LuaScript = newLuaScript('removeOrder', '../scri export const addCanceledOrderIdScript: LuaScript = newLuaScript('addCanceledOrderId', '../scripts/add_canceled_order_id.lua'); export const addStatefulOrderUpdateScript: LuaScript = newLuaScript('addStatefulOrderUpdate', '../scripts/add_stateful_order_update.lua'); export const removeStatefulOrderUpdateScript: LuaScript = newLuaScript('removeStatefulOrderUpdate', '../scripts/remove_stateful_order_update.lua'); +export const addMarketPriceScript: LuaScript = newLuaScript('addMarketPrice', '../scripts/add_market_price.lua'); +export const getMarketMedianScript: LuaScript = newLuaScript('getMarketMedianPrice', '../scripts/get_market_median_price.lua'); export const allLuaScripts: LuaScript[] = [ deleteZeroPriceLevelScript, @@ -75,4 +77,6 @@ export const allLuaScripts: LuaScript[] = [ addCanceledOrderIdScript, addStatefulOrderUpdateScript, removeStatefulOrderUpdateScript, + addMarketPriceScript, + getMarketMedianScript, ]; diff --git a/indexer/packages/redis/src/index.ts b/indexer/packages/redis/src/index.ts index 2ce64e9b88..5c7aeed103 100644 --- a/indexer/packages/redis/src/index.ts +++ b/indexer/packages/redis/src/index.ts @@ -12,6 +12,7 @@ export * as CanceledOrdersCache from './caches/canceled-orders-cache'; export * as StatefulOrderUpdatesCache from './caches/stateful-order-updates-cache'; export * as StateFilledQuantumsCache from './caches/state-filled-quantums-cache'; export * as LeaderboardPnlProcessedCache from './caches/leaderboard-processed-cache'; +export * as OrderbookMidPricesCache from './caches/orderbook-mid-prices-cache'; export { placeOrder } from './caches/place-order'; export { removeOrder } from './caches/remove-order'; export { updateOrder } from './caches/update-order'; diff --git a/indexer/packages/redis/src/scripts/add_market_price.lua b/indexer/packages/redis/src/scripts/add_market_price.lua new file mode 100644 index 0000000000..0e1467bb31 --- /dev/null +++ b/indexer/packages/redis/src/scripts/add_market_price.lua @@ -0,0 +1,17 @@ +-- Key for the ZSET storing price data +local priceCacheKey = KEYS[1] +-- Price to be added +local price = tonumber(ARGV[1]) +-- Current timestamp +local nowSeconds = tonumber(ARGV[2]) +-- Time window (5 seconds) +local fiveSeconds = 5 + +-- 1. Add the price to the sorted set (score is the current timestamp) +redis.call("zadd", priceCacheKey, nowSeconds, price) + +-- 2. Remove any entries older than 5 seconds +local cutoffTime = nowSeconds - fiveSeconds +redis.call("zremrangebyscore", priceCacheKey, "-inf", cutoffTime) + +return true \ No newline at end of file diff --git a/indexer/packages/redis/src/scripts/get_market_median_price.lua b/indexer/packages/redis/src/scripts/get_market_median_price.lua new file mode 100644 index 0000000000..a318296f20 --- /dev/null +++ b/indexer/packages/redis/src/scripts/get_market_median_price.lua @@ -0,0 +1,22 @@ +-- Key for the sorted set storing price data +local priceCacheKey = KEYS[1] + +-- Get all the prices from the sorted set (ascending order) +local prices = redis.call('zrange', priceCacheKey, 0, -1) + +-- If no prices are found, return nil +if #prices == 0 then + return nil +end + +-- Calculate the middle index +local middle = math.floor(#prices / 2) + +-- Calculate median +if #prices % 2 == 0 then + -- If even, return both prices, division will be handled in Javascript + return {prices[middle], prices[middle + 1]} +else + -- If odd, return the middle element + return {prices[middle + 1]} +end diff --git a/indexer/services/ender/__tests__/lib/candles-generator.test.ts b/indexer/services/ender/__tests__/lib/candles-generator.test.ts index d1c257b80b..d58ebd0d2a 100644 --- a/indexer/services/ender/__tests__/lib/candles-generator.test.ts +++ b/indexer/services/ender/__tests__/lib/candles-generator.test.ts @@ -18,7 +18,6 @@ import { testMocks, Transaction, helpers, - OrderSide, } from '@dydxprotocol-indexer/postgres'; import { CandleMessage, CandleMessage_Resolution } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -32,9 +31,11 @@ import { KafkaPublisher } from '../../src/lib/kafka-publisher'; import { ConsolidatedKafkaEvent } from '../../src/lib/types'; import { defaultTradeContent, defaultTradeKafkaEvent } from '../helpers/constants'; import { contentToSingleTradeMessage, createConsolidatedKafkaEventFromTrade } from '../helpers/kafka-publisher-helpers'; -import { updatePriceLevel } from '../helpers/redis-helpers'; import { redisClient } from '../../src/helpers/redis/redis-controller'; -import { redis } from '@dydxprotocol-indexer/redis'; +import { + redis, + OrderbookMidPricesCache, +} from '@dydxprotocol-indexer/redis'; describe('candleHelper', () => { beforeAll(async () => { @@ -113,9 +114,12 @@ describe('candleHelper', () => { defaultTradeKafkaEvent2, ]); - // Create Orderbook levels to set orderbookMidPrice open & close - await updatePriceLevel('BTC-USD', '100000', OrderSide.BUY); - await updatePriceLevel('BTC-USD', '110000', OrderSide.SELL); + const ticker = 'BTC-USD'; + await Promise.all([ + OrderbookMidPricesCache.setPrice(redisClient, ticker, '100000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '105000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '110000'), + ]); await runUpdateCandles(publisher); @@ -155,8 +159,12 @@ describe('candleHelper', () => { defaultTradeKafkaEvent2, ]); - await updatePriceLevel('BTC-USD', '80000', OrderSide.BUY); - await updatePriceLevel('BTC-USD', '81000', OrderSide.SELL); + const ticker = 'BTC-USD'; + await Promise.all([ + OrderbookMidPricesCache.setPrice(redisClient, ticker, '80000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '81000'), + OrderbookMidPricesCache.setPrice(redisClient, ticker, '80500'), + ]); // Create Perpetual Position to set open position const openInterest: string = '100'; @@ -427,9 +435,7 @@ describe('candleHelper', () => { containsKafkaMessages: boolean = true, orderbookMidPrice: number, ) => { - const midPriceSpread = 10; - await updatePriceLevel('BTC-USD', String(orderbookMidPrice + midPriceSpread), OrderSide.SELL); - await updatePriceLevel('BTC-USD', String(orderbookMidPrice - midPriceSpread), OrderSide.BUY); + await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', orderbookMidPrice.toFixed()); if (initialCandle !== undefined) { await CandleTable.create(initialCandle); @@ -494,9 +500,7 @@ describe('candleHelper', () => { ); await startCandleCache(); - // Update Orderbook levels - await updatePriceLevel('BTC-USD', '10010', OrderSide.SELL); - await updatePriceLevel('BTC-USD', '10000', OrderSide.BUY); + await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); const publisher: KafkaPublisher = new KafkaPublisher(); publisher.addEvents([ @@ -594,9 +598,7 @@ describe('candleHelper', () => { ); await startCandleCache(); - // Update Orderbook levels - await updatePriceLevel('BTC-USD', '10010', OrderSide.SELL); - await updatePriceLevel('BTC-USD', '10000', OrderSide.BUY); + await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); const publisher: KafkaPublisher = new KafkaPublisher(); publisher.addEvents([]); @@ -660,22 +662,19 @@ describe('candleHelper', () => { }); it('successfully creates an orderbook price map for each market', async () => { - await updatePriceLevel('BTC-USD', '100000', OrderSide.BUY); - await updatePriceLevel('BTC-USD', '110000', OrderSide.SELL); - - await updatePriceLevel('ISO-USD', '110000', OrderSide.BUY); - await updatePriceLevel('ISO-USD', '120000', OrderSide.SELL); - - await updatePriceLevel('ETH-USD', '100000', OrderSide.BUY); - await updatePriceLevel('ETH-USD', '200000', OrderSide.SELL); + await Promise.all([ + OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '105000'), + OrderbookMidPricesCache.setPrice(redisClient, 'ISO-USD', '115000'), + OrderbookMidPricesCache.setPrice(redisClient, 'ETH-USD', '150000'), + ]); const map = await getOrderbookMidPriceMap(); expect(map).toEqual({ 'BTC-USD': '105000', 'ETH-USD': '150000', 'ISO-USD': '115000', - 'ISO2-USD': undefined, - 'SHIB-USD': undefined, + 'ISO2-USD': null, + 'SHIB-USD': null, }); }); }); diff --git a/indexer/services/ender/src/lib/candles-generator.ts b/indexer/services/ender/src/lib/candles-generator.ts index d7dd7bba34..b232a66eb0 100644 --- a/indexer/services/ender/src/lib/candles-generator.ts +++ b/indexer/services/ender/src/lib/candles-generator.ts @@ -20,7 +20,7 @@ import { TradeMessageContents, helpers, } from '@dydxprotocol-indexer/postgres'; -import { OrderbookLevelsCache } from '@dydxprotocol-indexer/redis'; +import { OrderbookMidPricesCache } from '@dydxprotocol-indexer/redis'; import { CandleMessage } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; import _ from 'lodash'; @@ -538,9 +538,9 @@ export async function getOrderbookMidPriceMap(): Promise<{ [ticker: string]: Ord const perpetualMarkets = Object.values(perpetualMarketRefresher.getPerpetualMarketsMap()); const promises = perpetualMarkets.map(async (perpetualMarket: PerpetualMarketFromDatabase) => { - const price = await OrderbookLevelsCache.getOrderBookMidPrice( - perpetualMarket.ticker, + const price = await OrderbookMidPricesCache.getMedianPrice( redisClient, + perpetualMarket.ticker, ); return { [perpetualMarket.ticker]: price === undefined ? undefined : price }; }); diff --git a/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts b/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts new file mode 100644 index 0000000000..cd0eee3970 --- /dev/null +++ b/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts @@ -0,0 +1,98 @@ +import { + dbHelpers, + testConstants, + testMocks, +} from '@dydxprotocol-indexer/postgres'; +import { + OrderbookMidPricesCache, + OrderbookLevelsCache, + redis, +} from '@dydxprotocol-indexer/redis'; +import { redisClient } from '../../src/helpers/redis'; +import runTask from '../../src/tasks/cache-orderbook-mid-prices'; + +jest.mock('@dydxprotocol-indexer/base', () => ({ + ...jest.requireActual('@dydxprotocol-indexer/base'), + logger: { + info: jest.fn(), + error: jest.fn(), + }, +})); + +jest.mock('@dydxprotocol-indexer/redis', () => ({ + ...jest.requireActual('@dydxprotocol-indexer/redis'), + OrderbookLevelsCache: { + getOrderBookMidPrice: jest.fn(), + }, +})); + +describe('cache-orderbook-mid-prices', () => { + beforeAll(async () => { + await dbHelpers.migrate(); + }); + + beforeEach(async () => { + await dbHelpers.clearData(); + await redis.deleteAllAsync(redisClient); + await testMocks.seedData(); + }); + + afterAll(async () => { + await dbHelpers.teardown(); + jest.resetAllMocks(); + }); + + it('caches mid prices for all markets', async () => { + const market1 = testConstants.defaultPerpetualMarket; + const market2 = testConstants.defaultPerpetualMarket2; + + const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); + mockGetOrderBookMidPrice.mockResolvedValueOnce('100.5'); // For market1 + mockGetOrderBookMidPrice.mockResolvedValueOnce('200.75'); // For market2 + + await runTask(); + + // Check if the mock was called with the correct arguments + expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market1.ticker, redisClient); + expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market2.ticker, redisClient); + + // Check if prices were cached correctly + const price1 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market1.ticker); + const price2 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market2.ticker); + + expect(price1).toBe('100.5'); + expect(price2).toBe('200.75'); + }); + + it('handles undefined prices', async () => { + const market = testConstants.defaultPerpetualMarket; + + const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); + mockGetOrderBookMidPrice.mockResolvedValueOnce(undefined); + + await runTask(); + + const price = await OrderbookMidPricesCache.getMedianPrice(redisClient, market.ticker); + expect(price).toBeNull(); + + // Check that a log message was created + expect(jest.requireMock('@dydxprotocol-indexer/base').logger.info).toHaveBeenCalledWith({ + at: 'cache-orderbook-mid-prices#runTask', + message: `undefined price for ${market.ticker}`, + }); + }); + + it('handles errors', async () => { + // Mock OrderbookLevelsCache.getOrderBookMidPrice to throw an error + const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); + mockGetOrderBookMidPrice.mockRejectedValueOnce(new Error('Test error')); + + await runTask(); + + expect(jest.requireMock('@dydxprotocol-indexer/base').logger.error).toHaveBeenCalledWith({ + at: 'cache-orderbook-mid-prices#runTask', + message: 'Test error', + error: expect.any(Error), + }); + }); +}); diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index a8ec2cb87b..9f0f00c487 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -60,6 +60,7 @@ export const configSchema = { LOOPS_ENABLED_UPDATE_WALLET_TOTAL_VOLUME: parseBoolean({ default: true }), LOOPS_ENABLED_UPDATE_AFFILIATE_INFO: parseBoolean({ default: true }), LOOPS_ENABLED_DELETE_OLD_FIREBASE_NOTIFICATION_TOKENS: parseBoolean({ default: true }), + LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES: parseBoolean({ default: true }), // Loop Timing LOOPS_INTERVAL_MS_MARKET_UPDATER: parseInteger({ @@ -137,6 +138,9 @@ export const configSchema = { LOOPS_INTERVAL_MS_DELETE_FIREBASE_NOTIFICATION_TOKENS_MONTHLY: parseInteger({ default: 30 * ONE_DAY_IN_MILLISECONDS, }), + LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES: parseInteger({ + default: ONE_SECOND_IN_MILLISECONDS, + }), // Start delay START_DELAY_ENABLED: parseBoolean({ default: true }), diff --git a/indexer/services/roundtable/src/index.ts b/indexer/services/roundtable/src/index.ts index f52903ac19..bfdee334c7 100644 --- a/indexer/services/roundtable/src/index.ts +++ b/indexer/services/roundtable/src/index.ts @@ -10,6 +10,7 @@ import { connect as connectToRedis, } from './helpers/redis'; import aggregateTradingRewardsTasks from './tasks/aggregate-trading-rewards'; +import cacheOrderbookMidPrices from './tasks/cache-orderbook-mid-prices'; import cancelStaleOrdersTask from './tasks/cancel-stale-orders'; import createLeaderboardTask from './tasks/create-leaderboard'; import createPnlTicksTask from './tasks/create-pnl-ticks'; @@ -272,6 +273,14 @@ async function start(): Promise { ); } + if (config.LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES) { + startLoop( + cacheOrderbookMidPrices, + 'cache_orderbook_mid_prices', + config.LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES, + ); + } + logger.info({ at: 'index', message: 'Successfully started', diff --git a/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts b/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts new file mode 100644 index 0000000000..644f50df6f --- /dev/null +++ b/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts @@ -0,0 +1,40 @@ +import { + logger, +} from '@dydxprotocol-indexer/base'; +import { + PerpetualMarketFromDatabase, + PerpetualMarketTable, +} from '@dydxprotocol-indexer/postgres'; +import { + OrderbookMidPricesCache, + OrderbookLevelsCache, +} from '@dydxprotocol-indexer/redis'; + +import { redisClient } from '../helpers/redis'; + +/** + * Updates OrderbookMidPricesCache with current orderbook mid price for each market + */ +export default async function runTask(): Promise { + const markets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll({}, []); + + for (const market of markets) { + try { + const price = await OrderbookLevelsCache.getOrderBookMidPrice(market.ticker, redisClient); + if (price) { + await OrderbookMidPricesCache.setPrice(redisClient, market.ticker, price); + } else { + logger.info({ + at: 'cache-orderbook-mid-prices#runTask', + message: `undefined price for ${market.ticker}`, + }); + } + } catch (error) { + logger.error({ + at: 'cache-orderbook-mid-prices#runTask', + message: error.message, + error, + }); + } + } +} From 8a0a7484bb6613a8b7a4b0337fa1aae9db74812a Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:38:32 -0400 Subject: [PATCH 03/41] [OTE-784] Limit addresses for compliance check to dydx wallets with deposit (backport #2330) (#2353) Co-authored-by: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> --- .../src/clients/elliptic-provider.ts | 4 +- indexer/packages/compliance/src/index.ts | 1 + .../stores/compliance-data-table.test.ts | 25 ++++++++ .../postgres/src/stores/compliance-table.ts | 11 ++++ .../postgres/src/types/query-types.ts | 2 + .../api/v4/compliance-controller.test.ts | 34 ++++++++++- .../api/v4/compliance-controller.ts | 17 +++++- .../tasks/update-compliance-data.test.ts | 60 +++++++++++++++++++ .../src/tasks/update-compliance-data.ts | 28 ++++++++- 9 files changed, 178 insertions(+), 4 deletions(-) diff --git a/indexer/packages/compliance/src/clients/elliptic-provider.ts b/indexer/packages/compliance/src/clients/elliptic-provider.ts index 74df5c538d..9004470047 100644 --- a/indexer/packages/compliance/src/clients/elliptic-provider.ts +++ b/indexer/packages/compliance/src/clients/elliptic-provider.ts @@ -26,6 +26,8 @@ export const API_PATH: string = '/v2/wallet/synchronous'; export const API_URI: string = `https://aml-api.elliptic.co${API_PATH}`; export const RISK_SCORE_KEY: string = 'risk_score'; export const NO_RULES_TRIGGERED_RISK_SCORE: number = -1; +// We use different negative values of risk score to represent different elliptic response states +export const NOT_IN_BLOCKCHAIN_RISK_SCORE: number = -2; export class EllipticProviderClient extends ComplianceClient { private apiKey: string; @@ -98,7 +100,7 @@ export class EllipticProviderClient extends ComplianceClient { `${config.SERVICE_NAME}.get_elliptic_risk_score.status_code`, { status: '404' }, ); - return NO_RULES_TRIGGERED_RISK_SCORE; + return NOT_IN_BLOCKCHAIN_RISK_SCORE; } if (error?.response?.status === 429) { diff --git a/indexer/packages/compliance/src/index.ts b/indexer/packages/compliance/src/index.ts index b3a587079a..0fc7422a05 100644 --- a/indexer/packages/compliance/src/index.ts +++ b/indexer/packages/compliance/src/index.ts @@ -5,3 +5,4 @@ export * from './geoblocking/util'; export * from './types'; export * from './config'; export * from './constants'; +export * from './clients/elliptic-provider'; diff --git a/indexer/packages/postgres/__tests__/stores/compliance-data-table.test.ts b/indexer/packages/postgres/__tests__/stores/compliance-data-table.test.ts index 2802726c83..f620ca50a1 100644 --- a/indexer/packages/postgres/__tests__/stores/compliance-data-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/compliance-data-table.test.ts @@ -1,5 +1,6 @@ import { ComplianceDataFromDatabase, ComplianceProvider } from '../../src/types'; import * as ComplianceDataTable from '../../src/stores/compliance-table'; +import * as WalletTable from '../../src/stores/wallet-table'; import { clearData, migrate, @@ -9,6 +10,7 @@ import { blockedComplianceData, blockedAddress, nonBlockedComplianceData, + defaultWallet, } from '../helpers/constants'; import { DateTime } from 'luxon'; @@ -139,6 +141,29 @@ describe('Compliance data store', () => { expect(complianceData).toEqual(blockedComplianceData); }); + it('Successfully filters by onlyDydxAddressWithDeposit', async () => { + // Create two compliance entries, one with a corresponding wallet entry and another without + await Promise.all([ + WalletTable.create(defaultWallet), + ComplianceDataTable.create(nonBlockedComplianceData), + ComplianceDataTable.create({ + ...nonBlockedComplianceData, + address: 'not_dydx_address', + }), + ]); + + const complianceData: ComplianceDataFromDatabase[] = await ComplianceDataTable.findAll( + { + addressInWalletsTable: true, + }, + [], + { readReplica: true }, + ); + + expect(complianceData.length).toEqual(1); + expect(complianceData[0]).toEqual(nonBlockedComplianceData); + }); + it('Unable finds compliance data', async () => { const complianceData: ComplianceDataFromDatabase | undefined = await ComplianceDataTable.findByAddressAndProvider( diff --git a/indexer/packages/postgres/src/stores/compliance-table.ts b/indexer/packages/postgres/src/stores/compliance-table.ts index 162edff16d..d3d53c9bb3 100644 --- a/indexer/packages/postgres/src/stores/compliance-table.ts +++ b/indexer/packages/postgres/src/stores/compliance-table.ts @@ -13,6 +13,7 @@ import { } from '../helpers/stores-helpers'; import Transaction from '../helpers/transaction'; import ComplianceDataModel from '../models/compliance-data-model'; +import WalletModel from '../models/wallet-model'; import { ComplianceDataFromDatabase, ComplianceDataQueryConfig, @@ -34,6 +35,7 @@ export async function findAll( provider, blocked, limit, + addressInWalletsTable, }: ComplianceDataQueryConfig, requiredFields: QueryableField[], options: Options = DEFAULT_POSTGRES_OPTIONS, @@ -45,6 +47,7 @@ export async function findAll( provider, blocked, limit, + addressInWalletsTable, } as QueryConfig, requiredFields, ); @@ -70,6 +73,14 @@ export async function findAll( baseQuery = baseQuery.where(ComplianceDataColumns.blocked, blocked); } + if (addressInWalletsTable === true) { + baseQuery = baseQuery.innerJoin( + WalletModel.tableName, + `${ComplianceDataModel.tableName}.${ComplianceDataColumns.address}`, + '=', + `${WalletModel.tableName}.${WalletModel.idColumn}`); + } + if (options.orderBy !== undefined) { for (const [column, order] of options.orderBy) { baseQuery = baseQuery.orderBy( diff --git a/indexer/packages/postgres/src/types/query-types.ts b/indexer/packages/postgres/src/types/query-types.ts index c5c18cab71..3b5cd025e2 100644 --- a/indexer/packages/postgres/src/types/query-types.ts +++ b/indexer/packages/postgres/src/types/query-types.ts @@ -92,6 +92,7 @@ export enum QueryableField { REFEREE_ADDRESS = 'refereeAddress', KEY = 'key', TOKEN = 'token', + ADDRESS_IN_WALLETS_TABLE = 'addressInWalletsTable', } export interface QueryConfig { @@ -291,6 +292,7 @@ export interface ComplianceDataQueryConfig extends QueryConfig { [QueryableField.UPDATED_BEFORE_OR_AT]?: string, [QueryableField.PROVIDER]?: string, [QueryableField.BLOCKED]?: boolean, + [QueryableField.ADDRESS_IN_WALLETS_TABLE]?: boolean, } export interface ComplianceStatusQueryConfig extends QueryConfig { diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/compliance-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/compliance-controller.test.ts index 34e12bddaa..571abd389f 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/compliance-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/compliance-controller.test.ts @@ -9,7 +9,11 @@ import { } from '@dydxprotocol-indexer/postgres'; import { stats } from '@dydxprotocol-indexer/base'; import { complianceProvider } from '../../../../src/helpers/compliance/compliance-clients'; -import { ComplianceClientResponse, INDEXER_COMPLIANCE_BLOCKED_PAYLOAD } from '@dydxprotocol-indexer/compliance'; +import { + ComplianceClientResponse, + INDEXER_COMPLIANCE_BLOCKED_PAYLOAD, + NOT_IN_BLOCKCHAIN_RISK_SCORE, +} from '@dydxprotocol-indexer/compliance'; import { ratelimitRedis } from '../../../../src/caches/rate-limiters'; import { redis } from '@dydxprotocol-indexer/redis'; import { DateTime } from 'luxon'; @@ -257,5 +261,33 @@ describe('compliance-controller#V4', () => { { provider: complianceProvider.provider }, ); }); + + it('GET /screen for invalid address does not upsert compliance data', async () => { + const invalidAddress: string = 'invalidAddress'; + const notInBlockchainRiskScore: string = NOT_IN_BLOCKCHAIN_RISK_SCORE.toString(); + + jest.spyOn(complianceProvider.client, 'getComplianceResponse').mockImplementation( + (address: string): Promise => { + return Promise.resolve({ + address, + blocked, + riskScore: notInBlockchainRiskScore, + }); + }, + ); + + const response: any = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/screen?address=${invalidAddress}`, + }); + + expect(response.body).toEqual({ + restricted: false, + reason: undefined, + }); + + const data = await ComplianceTable.findAll({}, [], {}); + expect(data).toHaveLength(0); + }); }); }); diff --git a/indexer/services/comlink/src/controllers/api/v4/compliance-controller.ts b/indexer/services/comlink/src/controllers/api/v4/compliance-controller.ts index 09b5989c67..f8f5bd23e1 100644 --- a/indexer/services/comlink/src/controllers/api/v4/compliance-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/compliance-controller.ts @@ -1,5 +1,9 @@ import { logger, stats, TooManyRequestsError } from '@dydxprotocol-indexer/base'; -import { ComplianceClientResponse, INDEXER_COMPLIANCE_BLOCKED_PAYLOAD } from '@dydxprotocol-indexer/compliance'; +import { + ComplianceClientResponse, + INDEXER_COMPLIANCE_BLOCKED_PAYLOAD, + NOT_IN_BLOCKCHAIN_RISK_SCORE, +} from '@dydxprotocol-indexer/compliance'; import { ComplianceDataCreateObject, ComplianceDataFromDatabase, ComplianceTable } from '@dydxprotocol-indexer/postgres'; import express from 'express'; import { checkSchema, matchedData } from 'express-validator'; @@ -85,6 +89,17 @@ export class ComplianceControllerHelper extends Controller { ComplianceClientResponse = await complianceProvider.client.getComplianceResponse( address, ); + // Don't upsert invalid addresses (address causing ellitic error) to compliance table. + // When the elliptic request fails with 404, getComplianceResponse returns + // riskScore=NOT_IN_BLOCKCHAIN_RISK_SCORE + if (response.riskScore === undefined || + Number(response.riskScore) === NOT_IN_BLOCKCHAIN_RISK_SCORE) { + return { + restricted: false, + reason: undefined, + }; + } + complianceData = await ComplianceTable.upsert({ ..._.omitBy(response, _.isUndefined) as ComplianceDataCreateObject, provider: complianceProvider.provider, diff --git a/indexer/services/roundtable/__tests__/tasks/update-compliance-data.test.ts b/indexer/services/roundtable/__tests__/tasks/update-compliance-data.test.ts index 9b5c2f4a62..e0493b2ec7 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-compliance-data.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-compliance-data.test.ts @@ -678,6 +678,66 @@ describe('update-compliance-data', () => { config.MAX_COMPLIANCE_DATA_QUERY_PER_LOOP = defaultMaxQueries; }); + + it('Only updates old addresses that are in wallets table', async () => { + const rogueWallet: string = 'address_not_in_wallets'; + // Seed database with old compliance data, and set up subaccounts to not be active + // Create a compliance dataentry that is not in the wallets table + await Promise.all([ + setupComplianceData(config.MAX_COMPLIANCE_DATA_AGE_SECONDS * 2), + setupInitialSubaccounts(config.MAX_ACTIVE_COMPLIANCE_DATA_AGE_SECONDS * 2), + ]); + await ComplianceTable.create({ + ...testConstants.nonBlockedComplianceData, + address: rogueWallet, + }); + + const riskScore: string = '75.00'; + setupMockProvider( + mockProvider, + { [testConstants.defaultAddress]: { blocked: true, riskScore } }, + ); + + await updateComplianceDataTask(mockProvider); + + const updatedCompliancnceData: ComplianceDataFromDatabase[] = await ComplianceTable.findAll({ + address: [testConstants.defaultAddress], + }, [], {}); + const unchangedComplianceData: ComplianceDataFromDatabase[] = await ComplianceTable.findAll({ + address: [rogueWallet], + }, [], {}); + + expectUpdatedCompliance( + updatedCompliancnceData[0], + { + address: testConstants.defaultAddress, + blocked: true, + riskScore, + }, + mockProvider.provider, + ); + expectUpdatedCompliance( + unchangedComplianceData[0], + { + address: rogueWallet, + blocked: testConstants.nonBlockedComplianceData.blocked, + riskScore: testConstants.nonBlockedComplianceData.riskScore, + }, + mockProvider.provider, + ); + expectGaugeStats({ + activeAddresses: 0, + newAddresses: 0, + oldAddresses: 1, + addressesScreened: 1, + upserted: 1, + statusUpserted: 1, + activeAddressesWithStaleCompliance: 0, + inactiveAddressesWithStaleCompliance: 1, + }, + mockProvider.provider, + ); + }); }); async function setupComplianceData( diff --git a/indexer/services/roundtable/src/tasks/update-compliance-data.ts b/indexer/services/roundtable/src/tasks/update-compliance-data.ts index ba6be2e32b..04db96294c 100644 --- a/indexer/services/roundtable/src/tasks/update-compliance-data.ts +++ b/indexer/services/roundtable/src/tasks/update-compliance-data.ts @@ -1,7 +1,7 @@ import { STATS_NO_SAMPLING, delay, logger, stats, } from '@dydxprotocol-indexer/base'; -import { ComplianceClientResponse } from '@dydxprotocol-indexer/compliance'; +import { ComplianceClientResponse, NOT_IN_BLOCKCHAIN_RISK_SCORE } from '@dydxprotocol-indexer/compliance'; import { ComplianceDataColumns, ComplianceDataCreateObject, @@ -151,6 +151,7 @@ export default async function runTask( blocked: false, provider: complianceProvider.provider, updatedBeforeOrAt: ageThreshold, + addressInWalletsTable: true, }, [], { readReplica: true }, @@ -318,10 +319,19 @@ async function getComplianceData( return result.value; }, )); + const addressNotFoundResponses: + PromiseFulfilledResult[] = successResponses.filter( + (result: PromiseSettledResult): + result is PromiseFulfilledResult => { + // riskScore = NOT_IN_BLOCKCHAIN_RISK_SCORE denotes elliptic 404 responses + return result.status === 'fulfilled' && result.value.riskScore === NOT_IN_BLOCKCHAIN_RISK_SCORE.toString(); + }, + ); if (failedResponses.length > 0) { const addressesWithoutResponses: string[] = _.without( addresses, + // complianceResponses includes 404 responses ..._.map(complianceResponses, 'address'), ); stats.increment( @@ -337,6 +347,22 @@ async function getComplianceData( errors: failedResponses, }); } + + if (addressNotFoundResponses.length > 0) { + const notFoundAddresses = addressNotFoundResponses.map((result) => result.value.address); + + stats.increment( + `${config.SERVICE_NAME}.${taskName}.get_compliance_data_404`, + 1, + undefined, + { provider: complianceProvider.provider }, + ); + logger.error({ + at: 'updated-compliance-data#getComplianceData', + message: 'Failed to retrieve compliance data for the addresses due to elliptic 404', + addresses: notFoundAddresses, + }); + } stats.timing( `${config.SERVICE_NAME}.${taskName}.get_batch_compliance_data`, Date.now() - startBatch, From 0f214f1c7329d6af1ac0e41f7a85daeb9aef1eaa Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:12:12 -0400 Subject: [PATCH 04/41] enable username generation roundtable (backport #2361) (#2362) Co-authored-by: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> --- indexer/services/roundtable/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index 9f0f00c487..42b66a4882 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -51,7 +51,7 @@ export const configSchema = { LOOPS_ENABLED_AGGREGATE_TRADING_REWARDS_DAILY: parseBoolean({ default: true }), LOOPS_ENABLED_AGGREGATE_TRADING_REWARDS_WEEKLY: parseBoolean({ default: true }), LOOPS_ENABLED_AGGREGATE_TRADING_REWARDS_MONTHLY: parseBoolean({ default: true }), - LOOPS_ENABLED_SUBACCOUNT_USERNAME_GENERATOR: parseBoolean({ default: false }), + LOOPS_ENABLED_SUBACCOUNT_USERNAME_GENERATOR: parseBoolean({ default: true }), LOOPS_ENABLED_LEADERBOARD_PNL_ALL_TIME: parseBoolean({ default: false }), LOOPS_ENABLED_LEADERBOARD_PNL_DAILY: parseBoolean({ default: false }), LOOPS_ENABLED_LEADERBOARD_PNL_WEEKLY: parseBoolean({ default: false }), From a47c3277fb6e3dd4c78910a792940490b9c3bcf2 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:25:29 -0400 Subject: [PATCH 05/41] Add current equity as a pnl tick. (backport #2335) (#2365) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../api/v4/vault-controller.test.ts | 118 ++++--- .../controllers/api/v4/vault-controller.ts | 313 +++++++++++------- 2 files changed, 271 insertions(+), 160 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index c0a9fe6c6e..ff03a401f6 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -18,17 +18,22 @@ import request from 'supertest'; import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers'; import config from '../../../../src/config'; import { DateTime } from 'luxon'; +import Big from 'big.js'; describe('vault-controller#V4', () => { const experimentVaultsPrevVal: string = config.EXPERIMENT_VAULTS; const experimentVaultMarketsPrevVal: string = config.EXPERIMENT_VAULT_MARKETS; + const latestBlockHeight: string = '25'; const currentBlockHeight: string = '7'; const twoHourBlockHeight: string = '5'; const twoDayBlockHeight: string = '3'; const currentTime: DateTime = DateTime.utc().startOf('day').minus({ hour: 5 }); + const latestTime: DateTime = currentTime.plus({ second: 5 }); const twoHoursAgo: DateTime = currentTime.minus({ hour: 2 }); const twoDaysAgo: DateTime = currentTime.minus({ day: 2 }); const initialFundingIndex: string = '10000'; + const vault1Equity: number = 159500; + const vault2Equity: number = 10000; beforeAll(async () => { await dbHelpers.migrate(); @@ -61,8 +66,33 @@ describe('vault-controller#V4', () => { time: currentTime.toISO(), blockHeight: currentBlockHeight, }), + BlockTable.create({ + ...testConstants.defaultBlock, + time: latestTime.toISO(), + blockHeight: latestBlockHeight, + }), ]); await SubaccountTable.create(testConstants.vaultSubaccount); + await Promise.all([ + PerpetualPositionTable.create( + testConstants.defaultPerpetualPosition, + ), + AssetPositionTable.upsert(testConstants.defaultAssetPosition), + AssetPositionTable.upsert({ + ...testConstants.defaultAssetPosition, + subaccountId: testConstants.vaultSubaccountId, + }), + FundingIndexUpdatesTable.create({ + ...testConstants.defaultFundingIndexUpdate, + fundingIndex: initialFundingIndex, + effectiveAtHeight: testConstants.createdHeight, + }), + FundingIndexUpdatesTable.create({ + ...testConstants.defaultFundingIndexUpdate, + eventId: testConstants.defaultTendermintEventId2, + effectiveAtHeight: twoDayBlockHeight, + }), + ]); }); afterEach(async () => { @@ -93,17 +123,25 @@ describe('vault-controller#V4', () => { expectedTicksIndex: number[], ) => { const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); + const finalTick: PnlTicksFromDatabase = { + ...createdPnlTicks[expectedTicksIndex[expectedTicksIndex.length - 1]], + equity: Big(vault1Equity).toFixed(), + blockHeight: latestBlockHeight, + blockTime: latestTime.toISO(), + createdAt: latestTime.toISO(), + }; const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: `/v4/vault/v1/megavault/historicalPnl${queryParam}`, }); + expect(response.body.megavaultPnl).toHaveLength(expectedTicksIndex.length + 1); expect(response.body.megavaultPnl).toEqual( expect.arrayContaining( expectedTicksIndex.map((index: number) => { return expect.objectContaining(createdPnlTicks[index]); - }), + }).concat([finalTick]), ), ); }); @@ -127,7 +165,6 @@ describe('vault-controller#V4', () => { ].join(','); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); - const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: `/v4/vault/v1/megavault/historicalPnl${queryParam}`, @@ -138,7 +175,15 @@ describe('vault-controller#V4', () => { totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) * 2).toString(), netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) * 2).toString(), }; + const finalTick: PnlTicksFromDatabase = { + ...expectedPnlTickBase, + equity: Big(vault1Equity).add(vault2Equity).toFixed(), + blockHeight: latestBlockHeight, + blockTime: latestTime.toISO(), + createdAt: latestTime.toISO(), + }; + expect(response.body.megavaultPnl).toHaveLength(expectedTicksIndex.length + 1); expect(response.body.megavaultPnl).toEqual( expect.arrayContaining( expectedTicksIndex.map((index: number) => { @@ -148,7 +193,7 @@ describe('vault-controller#V4', () => { blockHeight: createdPnlTicks[index].blockHeight, blockTime: createdPnlTicks[index].blockTime, }); - }), + }).concat([expect.objectContaining(finalTick)]), ), ); }); @@ -175,6 +220,13 @@ describe('vault-controller#V4', () => { expectedTicksIndex: number[], ) => { const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); + const finalTick: PnlTicksFromDatabase = { + ...createdPnlTicks[expectedTicksIndex[expectedTicksIndex.length - 1]], + equity: Big(vault1Equity).toFixed(), + blockHeight: latestBlockHeight, + blockTime: latestTime.toISO(), + createdAt: latestTime.toISO(), + }; const response: request.Response = await sendRequest({ type: RequestMethod.GET, @@ -182,13 +234,13 @@ describe('vault-controller#V4', () => { }); expect(response.body.vaultsPnl).toHaveLength(1); - + expect(response.body.vaultsPnl[0].historicalPnl).toHaveLength(expectedTicksIndex.length + 1); expect(response.body.vaultsPnl[0]).toEqual({ ticker: testConstants.defaultPerpetualMarket.ticker, historicalPnl: expect.arrayContaining( expectedTicksIndex.map((index: number) => { return expect.objectContaining(createdPnlTicks[index]); - }), + }).concat(finalTick), ), }); }); @@ -213,6 +265,20 @@ describe('vault-controller#V4', () => { ].join(','); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); + const finalTick1: PnlTicksFromDatabase = { + ...createdPnlTicks[expectedTicksIndex1[expectedTicksIndex1.length - 1]], + equity: Big(vault1Equity).toFixed(), + blockHeight: latestBlockHeight, + blockTime: latestTime.toISO(), + createdAt: latestTime.toISO(), + }; + const finalTick2: PnlTicksFromDatabase = { + ...createdPnlTicks[expectedTicksIndex2[expectedTicksIndex2.length - 1]], + equity: Big(vault2Equity).toFixed(), + blockHeight: latestBlockHeight, + blockTime: latestTime.toISO(), + createdAt: latestTime.toISO(), + }; const response: request.Response = await sendRequest({ type: RequestMethod.GET, @@ -223,14 +289,14 @@ describe('vault-controller#V4', () => { ticker: testConstants.defaultPerpetualMarket.ticker, historicalPnl: expectedTicksIndex1.map((index: number) => { return createdPnlTicks[index]; - }), + }).concat(finalTick1), }; const expectedVaultPnl2: VaultHistoricalPnl = { ticker: testConstants.defaultPerpetualMarket2.ticker, historicalPnl: expectedTicksIndex2.map((index: number) => { return createdPnlTicks[index]; - }), + }).concat(finalTick2), }; expect(response.body.vaultsPnl).toEqual( @@ -260,23 +326,6 @@ describe('vault-controller#V4', () => { }); it('Get /megavault/positions with 1 vault subaccount', async () => { - await Promise.all([ - PerpetualPositionTable.create( - testConstants.defaultPerpetualPosition, - ), - AssetPositionTable.upsert(testConstants.defaultAssetPosition), - FundingIndexUpdatesTable.create({ - ...testConstants.defaultFundingIndexUpdate, - fundingIndex: initialFundingIndex, - effectiveAtHeight: testConstants.createdHeight, - }), - FundingIndexUpdatesTable.create({ - ...testConstants.defaultFundingIndexUpdate, - eventId: testConstants.defaultTendermintEventId2, - effectiveAtHeight: twoDayBlockHeight, - }), - ]); - const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/megavault/positions', @@ -334,27 +383,6 @@ describe('vault-controller#V4', () => { testConstants.defaultPerpetualMarket2.clobPairId, ].join(','); - await Promise.all([ - PerpetualPositionTable.create( - testConstants.defaultPerpetualPosition, - ), - AssetPositionTable.upsert(testConstants.defaultAssetPosition), - AssetPositionTable.upsert({ - ...testConstants.defaultAssetPosition, - subaccountId: testConstants.vaultSubaccountId, - }), - FundingIndexUpdatesTable.create({ - ...testConstants.defaultFundingIndexUpdate, - fundingIndex: initialFundingIndex, - effectiveAtHeight: testConstants.createdHeight, - }), - FundingIndexUpdatesTable.create({ - ...testConstants.defaultFundingIndexUpdate, - eventId: testConstants.defaultTendermintEventId2, - effectiveAtHeight: twoDayBlockHeight, - }), - ]); - const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/megavault/positions', diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 0403242ca4..830980955d 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -23,6 +23,7 @@ import { FundingIndexUpdatesTable, PnlTickInterval, } from '@dydxprotocol-indexer/postgres'; +import Big from 'big.js'; import express from 'express'; import { checkSchema, matchedData } from 'express-validator'; import _ from 'lodash'; @@ -68,13 +69,38 @@ class VaultController extends Controller { async getMegavaultHistoricalPnl( @Query() resolution?: PnlTickInterval, ): Promise { - const vaultPnlTicks: PnlTicksFromDatabase[] = await getVaultSubaccountPnlTicks(resolution); + const vaultSubaccounts: VaultMapping = getVaultSubaccountsFromConfig(); + const [ + vaultPnlTicks, + vaultPositions, + latestBlock, + ] : [ + PnlTicksFromDatabase[], + Map, + BlockFromDatabase, + ] = await Promise.all([ + getVaultSubaccountPnlTicks(resolution), + getVaultPositions(vaultSubaccounts), + BlockTable.getLatest(), + ]); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight const aggregatedPnlTicks: Map = aggregatePnlTicks(vaultPnlTicks); + const currentEquity: string = Array.from(vaultPositions.values()) + .map((position: VaultPosition): string => { + return position.equity; + }).reduce((acc: string, curr: string): string => { + return (Big(acc).add(Big(curr))).toFixed(); + }, '0'); + const pnlTicksWithCurrentTick: PnlTicksFromDatabase[] = getPnlTicksWithCurrentTick( + currentEquity, + Array.from(aggregatedPnlTicks.values()), + latestBlock, + ); + return { - megavaultPnl: Array.from(aggregatedPnlTicks.values()).map( + megavaultPnl: pnlTicksWithCurrentTick.map( (pnlTick: PnlTicksFromDatabase) => { return pnlTicksToResponseObject(pnlTick); }), @@ -86,7 +112,19 @@ class VaultController extends Controller { @Query() resolution?: PnlTickInterval, ): Promise { const vaultSubaccounts: VaultMapping = getVaultSubaccountsFromConfig(); - const vaultPnlTicks: PnlTicksFromDatabase[] = await getVaultSubaccountPnlTicks(resolution); + const [ + vaultPnlTicks, + vaultPositions, + latestBlock, + ] : [ + PnlTicksFromDatabase[], + Map, + BlockFromDatabase, + ] = await Promise.all([ + getVaultSubaccountPnlTicks(resolution), + getVaultPositions(vaultSubaccounts), + BlockTable.getLatest(), + ]); const groupedVaultPnlTicks: VaultHistoricalPnl[] = _(vaultPnlTicks) .groupBy('subaccountId') @@ -102,9 +140,17 @@ class VaultController extends Controller { 'a perpetual market.'); } + const vaultPosition: VaultPosition | undefined = vaultPositions.get(subaccountId); + const currentEquity: string = vaultPosition === undefined ? '0' : vaultPosition.equity; + const pnlTicksWithCurrentTick: PnlTicksFromDatabase[] = getPnlTicksWithCurrentTick( + currentEquity, + pnlTicks, + latestBlock, + ); + return { ticker: market.ticker, - historicalPnl: pnlTicks, + historicalPnl: pnlTicksWithCurrentTick, }; }) .values() @@ -118,120 +164,11 @@ class VaultController extends Controller { @Get('/megavault/positions') async getMegavaultPositions(): Promise { const vaultSubaccounts: VaultMapping = getVaultSubaccountsFromConfig(); - const vaultSubaccountIds: string[] = _.keys(vaultSubaccounts); - if (vaultSubaccountIds.length === 0) { - return { - positions: [], - }; - } - - const [ - subaccounts, - assets, - openPerpetualPositions, - assetPositions, - markets, - latestBlock, - ]: [ - SubaccountFromDatabase[], - AssetFromDatabase[], - PerpetualPositionFromDatabase[], - AssetPositionFromDatabase[], - MarketFromDatabase[], - BlockFromDatabase | undefined, - ] = await Promise.all([ - SubaccountTable.findAll( - { - id: vaultSubaccountIds, - }, - [], - ), - AssetTable.findAll( - {}, - [], - ), - PerpetualPositionTable.findAll( - { - subaccountId: vaultSubaccountIds, - status: [PerpetualPositionStatus.OPEN], - }, - [], - ), - AssetPositionTable.findAll( - { - subaccountId: vaultSubaccountIds, - assetId: [USDC_ASSET_ID], - }, - [], - ), - MarketTable.findAll( - {}, - [], - ), - BlockTable.getLatest(), - ]); - - const latestFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable - .findFundingIndexMap( - latestBlock.blockHeight, - ); - const assetPositionsBySubaccount: - { [subaccountId: string]: AssetPositionFromDatabase[] } = _.groupBy( - assetPositions, - 'subaccountId', - ); - const openPerpetualPositionsBySubaccount: - { [subaccountId: string]: PerpetualPositionFromDatabase[] } = _.groupBy( - openPerpetualPositions, - 'subaccountId', - ); - const assetIdToAsset: AssetById = _.keyBy( - assets, - AssetColumns.id, - ); - - const vaultPositions: VaultPosition[] = await Promise.all( - subaccounts.map(async (subaccount: SubaccountFromDatabase) => { - const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); - if (perpetualMarket === undefined) { - throw new Error( - `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + - 'perpetual market.'); - } - const lastUpdatedFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable - .findFundingIndexMap( - subaccount.updatedAtHeight, - ); - - const subaccountResponse: SubaccountResponseObject = getSubaccountResponse( - subaccount, - openPerpetualPositionsBySubaccount[subaccount.id] || [], - assetPositionsBySubaccount[subaccount.id] || [], - assets, - markets, - perpetualMarketRefresher.getPerpetualMarketsMap(), - latestBlock.blockHeight, - latestFundingIndexMap, - lastUpdatedFundingIndexMap, - ); - - return { - ticker: perpetualMarket.ticker, - assetPosition: subaccountResponse.assetPositions[ - assetIdToAsset[USDC_ASSET_ID].symbol - ], - perpetualPosition: subaccountResponse.openPerpetualPositions[ - perpetualMarket.ticker - ] || undefined, - equity: subaccountResponse.equity, - }; - }), - ); + const vaultPositions: Map = await getVaultPositions(vaultSubaccounts); return { - positions: _.sortBy(vaultPositions, 'ticker'), + positions: _.sortBy(Array.from(vaultPositions.values()), 'ticker'), }; } } @@ -371,6 +308,152 @@ async function getVaultSubaccountPnlTicks( return pnlTicks; } +async function getVaultPositions( + vaultSubaccounts: VaultMapping, +): Promise> { + const vaultSubaccountIds: string[] = _.keys(vaultSubaccounts); + if (vaultSubaccountIds.length === 0) { + return new Map(); + } + + const [ + subaccounts, + assets, + openPerpetualPositions, + assetPositions, + markets, + latestBlock, + ]: [ + SubaccountFromDatabase[], + AssetFromDatabase[], + PerpetualPositionFromDatabase[], + AssetPositionFromDatabase[], + MarketFromDatabase[], + BlockFromDatabase | undefined, + ] = await Promise.all([ + SubaccountTable.findAll( + { + id: vaultSubaccountIds, + }, + [], + ), + AssetTable.findAll( + {}, + [], + ), + PerpetualPositionTable.findAll( + { + subaccountId: vaultSubaccountIds, + status: [PerpetualPositionStatus.OPEN], + }, + [], + ), + AssetPositionTable.findAll( + { + subaccountId: vaultSubaccountIds, + assetId: [USDC_ASSET_ID], + }, + [], + ), + MarketTable.findAll( + {}, + [], + ), + BlockTable.getLatest(), + ]); + + const latestFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable + .findFundingIndexMap( + latestBlock.blockHeight, + ); + const assetPositionsBySubaccount: + { [subaccountId: string]: AssetPositionFromDatabase[] } = _.groupBy( + assetPositions, + 'subaccountId', + ); + const openPerpetualPositionsBySubaccount: + { [subaccountId: string]: PerpetualPositionFromDatabase[] } = _.groupBy( + openPerpetualPositions, + 'subaccountId', + ); + const assetIdToAsset: AssetById = _.keyBy( + assets, + AssetColumns.id, + ); + + const vaultPositionsAndSubaccountId: { + position: VaultPosition, + subaccountId: string, + }[] = await Promise.all( + subaccounts.map(async (subaccount: SubaccountFromDatabase) => { + const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher + .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); + if (perpetualMarket === undefined) { + throw new Error( + `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + + 'perpetual market.'); + } + const lastUpdatedFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable + .findFundingIndexMap( + subaccount.updatedAtHeight, + ); + + const subaccountResponse: SubaccountResponseObject = getSubaccountResponse( + subaccount, + openPerpetualPositionsBySubaccount[subaccount.id] || [], + assetPositionsBySubaccount[subaccount.id] || [], + assets, + markets, + perpetualMarketRefresher.getPerpetualMarketsMap(), + latestBlock.blockHeight, + latestFundingIndexMap, + lastUpdatedFundingIndexMap, + ); + + return { + position: { + ticker: perpetualMarket.ticker, + assetPosition: subaccountResponse.assetPositions[ + assetIdToAsset[USDC_ASSET_ID].symbol + ], + perpetualPosition: subaccountResponse.openPerpetualPositions[ + perpetualMarket.ticker + ] || undefined, + equity: subaccountResponse.equity, + }, + subaccountId: subaccount.id, + }; + }), + ); + + return new Map(vaultPositionsAndSubaccountId.map( + (obj: { position: VaultPosition, subaccountId: string }) : [string, VaultPosition] => { + return [ + obj.subaccountId, + obj.position, + ]; + }, + )); +} + +function getPnlTicksWithCurrentTick( + equity: string, + pnlTicks: PnlTicksFromDatabase[], + latestBlock: BlockFromDatabase, +): PnlTicksFromDatabase[] { + if (pnlTicks.length === 0) { + return []; + } + const currentTick: PnlTicksFromDatabase = { + ...pnlTicks[pnlTicks.length - 1], + equity, + blockHeight: latestBlock.blockHeight, + blockTime: latestBlock.time, + createdAt: latestBlock.time, + }; + return pnlTicks.concat([currentTick]); +} + // TODO(TRA-570): Placeholder for getting vault subaccount ids until vault table is added. function getVaultSubaccountsFromConfig(): VaultMapping { if (config.EXPERIMENT_VAULTS === '' && config.EXPERIMENT_VAULT_MARKETS === '') { From 9875783bc049ac884b71e8e53a991fd9853732b9 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:57:42 -0400 Subject: [PATCH 06/41] Use vault table rather than placeholder config flags to fetch vaults. (backport #2364) (#2367) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../api/v4/vault-controller.test.ts | 94 ++++++++++--------- indexer/services/comlink/src/config.ts | 5 - .../controllers/api/v4/vault-controller.ts | 41 ++++---- 3 files changed, 73 insertions(+), 67 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index ff03a401f6..488435f8e7 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -12,17 +12,15 @@ import { AssetPositionTable, FundingIndexUpdatesTable, PnlTicksFromDatabase, + VaultTable, } from '@dydxprotocol-indexer/postgres'; import { RequestMethod, VaultHistoricalPnl } from '../../../../src/types'; import request from 'supertest'; import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers'; -import config from '../../../../src/config'; import { DateTime } from 'luxon'; import Big from 'big.js'; describe('vault-controller#V4', () => { - const experimentVaultsPrevVal: string = config.EXPERIMENT_VAULTS; - const experimentVaultMarketsPrevVal: string = config.EXPERIMENT_VAULT_MARKETS; const latestBlockHeight: string = '25'; const currentBlockHeight: string = '7'; const twoHourBlockHeight: string = '5'; @@ -45,8 +43,6 @@ describe('vault-controller#V4', () => { describe('GET /v1', () => { beforeEach(async () => { - config.EXPERIMENT_VAULTS = testConstants.defaultPnlTick.subaccountId; - config.EXPERIMENT_VAULT_MARKETS = testConstants.defaultPerpetualMarket.clobPairId; await testMocks.seedData(); await perpetualMarketRefresher.updatePerpetualMarkets(); await liquidityTierRefresher.updateLiquidityTiers(); @@ -96,15 +92,10 @@ describe('vault-controller#V4', () => { }); afterEach(async () => { - config.EXPERIMENT_VAULTS = experimentVaultsPrevVal; - config.EXPERIMENT_VAULT_MARKETS = experimentVaultMarketsPrevVal; await dbHelpers.clearData(); }); it('Get /megavault/historicalPnl with no vault subaccounts', async () => { - config.EXPERIMENT_VAULTS = ''; - config.EXPERIMENT_VAULT_MARKETS = ''; - const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/megavault/historicalPnl', @@ -122,6 +113,11 @@ describe('vault-controller#V4', () => { queryParam: string, expectedTicksIndex: number[], ) => { + await VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.defaultSubaccount.address, + clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + }); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); const finalTick: PnlTicksFromDatabase = { ...createdPnlTicks[expectedTicksIndex[expectedTicksIndex.length - 1]], @@ -155,14 +151,18 @@ describe('vault-controller#V4', () => { queryParam: string, expectedTicksIndex: number[], ) => { - config.EXPERIMENT_VAULTS = [ - testConstants.defaultPnlTick.subaccountId, - testConstants.vaultSubaccountId, - ].join(','); - config.EXPERIMENT_VAULT_MARKETS = [ - testConstants.defaultPerpetualMarket.clobPairId, - testConstants.defaultPerpetualMarket2.clobPairId, - ].join(','); + await Promise.all([ + VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.defaultAddress, + clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + }), + VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.vaultAddress, + clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, + }), + ]); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); const response: request.Response = await sendRequest({ @@ -199,9 +199,6 @@ describe('vault-controller#V4', () => { }); it('Get /vaults/historicalPnl with no vault subaccounts', async () => { - config.EXPERIMENT_VAULTS = ''; - config.EXPERIMENT_VAULT_MARKETS = ''; - const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/vaults/historicalPnl', @@ -219,6 +216,11 @@ describe('vault-controller#V4', () => { queryParam: string, expectedTicksIndex: number[], ) => { + await VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.defaultAddress, + clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + }); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); const finalTick: PnlTicksFromDatabase = { ...createdPnlTicks[expectedTicksIndex[expectedTicksIndex.length - 1]], @@ -255,15 +257,18 @@ describe('vault-controller#V4', () => { expectedTicksIndex1: number[], expectedTicksIndex2: number[], ) => { - config.EXPERIMENT_VAULTS = [ - testConstants.defaultPnlTick.subaccountId, - testConstants.vaultSubaccountId, - ].join(','); - config.EXPERIMENT_VAULT_MARKETS = [ - testConstants.defaultPerpetualMarket.clobPairId, - testConstants.defaultPerpetualMarket2.clobPairId, - ].join(','); - + await Promise.all([ + VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.defaultAddress, + clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + }), + VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.vaultAddress, + clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, + }), + ]); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); const finalTick1: PnlTicksFromDatabase = { ...createdPnlTicks[expectedTicksIndex1[expectedTicksIndex1.length - 1]], @@ -312,9 +317,6 @@ describe('vault-controller#V4', () => { }); it('Get /megavault/positions with no vault subaccount', async () => { - config.EXPERIMENT_VAULTS = ''; - config.EXPERIMENT_VAULT_MARKETS = ''; - const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/megavault/positions', @@ -326,6 +328,11 @@ describe('vault-controller#V4', () => { }); it('Get /megavault/positions with 1 vault subaccount', async () => { + await VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.defaultAddress, + clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + }); const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/megavault/positions', @@ -374,15 +381,18 @@ describe('vault-controller#V4', () => { }); it('Get /megavault/positions with 2 vault subaccount, 1 with no perpetual', async () => { - config.EXPERIMENT_VAULTS = [ - testConstants.defaultPnlTick.subaccountId, - testConstants.vaultSubaccountId, - ].join(','); - config.EXPERIMENT_VAULT_MARKETS = [ - testConstants.defaultPerpetualMarket.clobPairId, - testConstants.defaultPerpetualMarket2.clobPairId, - ].join(','); - + await Promise.all([ + VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.defaultAddress, + clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + }), + VaultTable.create({ + ...testConstants.defaultVault, + address: testConstants.vaultAddress, + clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, + }), + ]); const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: '/v4/vault/v1/megavault/positions', diff --git a/indexer/services/comlink/src/config.ts b/indexer/services/comlink/src/config.ts index 9591c64eb0..43d2c81222 100644 --- a/indexer/services/comlink/src/config.ts +++ b/indexer/services/comlink/src/config.ts @@ -56,11 +56,6 @@ export const configSchema = { // Expose setting compliance status, only set to true in dev/staging. EXPOSE_SET_COMPLIANCE_ENDPOINT: parseBoolean({ default: false }), - // TODO(TRA-570): Placeholder data for vaults and matching set of markets for each vault until - // vaults table is added. - EXPERIMENT_VAULTS: parseString({ default: '' }), - EXPERIMENT_VAULT_MARKETS: parseString({ default: '' }), - // Affiliates config VOLUME_ELIGIBILITY_THRESHOLD: parseInteger({ default: 10_000 }), diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 830980955d..e377f40c9b 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -22,6 +22,8 @@ import { BlockFromDatabase, FundingIndexUpdatesTable, PnlTickInterval, + VaultTable, + VaultFromDatabase, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; import express from 'express'; @@ -57,8 +59,6 @@ import { const router: express.Router = express.Router(); const controllerName: string = 'vault-controller'; -// TODO(TRA-570): Placeholder interface for mapping of vault subaccounts to tickers until vaults -// table is added. interface VaultMapping { [subaccountId: string]: string, } @@ -69,7 +69,7 @@ class VaultController extends Controller { async getMegavaultHistoricalPnl( @Query() resolution?: PnlTickInterval, ): Promise { - const vaultSubaccounts: VaultMapping = getVaultSubaccountsFromConfig(); + const vaultSubaccounts: VaultMapping = await getVaultMapping(); const [ vaultPnlTicks, vaultPositions, @@ -79,7 +79,7 @@ class VaultController extends Controller { Map, BlockFromDatabase, ] = await Promise.all([ - getVaultSubaccountPnlTicks(resolution), + getVaultSubaccountPnlTicks(vaultSubaccounts, resolution), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), ]); @@ -111,7 +111,7 @@ class VaultController extends Controller { async getVaultsHistoricalPnl( @Query() resolution?: PnlTickInterval, ): Promise { - const vaultSubaccounts: VaultMapping = getVaultSubaccountsFromConfig(); + const vaultSubaccounts: VaultMapping = await getVaultMapping(); const [ vaultPnlTicks, vaultPositions, @@ -121,7 +121,7 @@ class VaultController extends Controller { Map, BlockFromDatabase, ] = await Promise.all([ - getVaultSubaccountPnlTicks(resolution), + getVaultSubaccountPnlTicks(vaultSubaccounts, resolution), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), ]); @@ -163,7 +163,7 @@ class VaultController extends Controller { @Get('/megavault/positions') async getMegavaultPositions(): Promise { - const vaultSubaccounts: VaultMapping = getVaultSubaccountsFromConfig(); + const vaultSubaccounts: VaultMapping = await getVaultMapping(); const vaultPositions: Map = await getVaultPositions(vaultSubaccounts); @@ -286,9 +286,10 @@ router.get( }); async function getVaultSubaccountPnlTicks( + vaultSubaccounts: VaultMapping, resolution?: PnlTickInterval, ): Promise { - const vaultSubaccountIds: string[] = _.keys(getVaultSubaccountsFromConfig()); + const vaultSubaccountIds: string[] = _.keys(vaultSubaccounts); if (vaultSubaccountIds.length === 0) { return []; } @@ -454,19 +455,19 @@ function getPnlTicksWithCurrentTick( return pnlTicks.concat([currentTick]); } -// TODO(TRA-570): Placeholder for getting vault subaccount ids until vault table is added. -function getVaultSubaccountsFromConfig(): VaultMapping { - if (config.EXPERIMENT_VAULTS === '' && config.EXPERIMENT_VAULT_MARKETS === '') { - return {}; - } - const vaultSubaccountIds: string[] = config.EXPERIMENT_VAULTS.split(','); - const vaultClobPairIds: string[] = config.EXPERIMENT_VAULT_MARKETS.split(','); - if (vaultSubaccountIds.length !== vaultClobPairIds.length) { - throw new Error('Expected number of vaults to match number of markets'); - } +async function getVaultMapping(): Promise { + const vaults: VaultFromDatabase[] = await VaultTable.findAll( + {}, + [], + {}, + ); return _.zipObject( - vaultSubaccountIds, - vaultClobPairIds, + vaults.map((vault: VaultFromDatabase): string => { + return SubaccountTable.uuid(vault.address, 0); + }), + vaults.map((vault: VaultFromDatabase): string => { + return vault.clobPairId; + }), ); } From d0347164128507d9964a3c67e5631698275a15c1 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:00:06 -0400 Subject: [PATCH 07/41] Add sql script latency metrics (backport #2356) (#2371) Co-authored-by: dydxwill <119354122+dydxwill@users.noreply.github.com> --- .../__tests__/handlers/funding-handler.test.ts | 8 ++++++++ .../services/ender/src/handlers/asset-handler.ts | 8 ++++++++ .../ender/src/handlers/funding-handler.ts | 7 +++++++ .../ender/src/handlers/liquidity-tier-handler.ts | 8 ++++++++ .../src/handlers/markets/market-create-handler.ts | 11 +++++++++-- .../src/handlers/markets/market-modify-handler.ts | 12 ++++++++++-- .../markets/market-price-update-handler.ts | 10 +++++++++- .../handlers/order-fills/deleveraging-handler.ts | 9 ++++++++- .../handlers/order-fills/liquidation-handler.ts | 8 ++++++++ .../src/handlers/order-fills/order-handler.ts | 9 +++++++++ .../src/handlers/perpetual-market-handler.ts | 8 ++++++++ .../conditional-order-placement-handler.ts | 8 ++++++++ .../conditional-order-triggered-handler.ts | 9 ++++++++- .../stateful-order-placement-handler.ts | 11 ++++++++++- .../stateful-order-removal-handler.ts | 10 +++++++++- .../src/handlers/subaccount-update-handler.ts | 8 ++++++++ .../ender/src/handlers/trading-rewards-handler.ts | 8 ++++++++ .../ender/src/handlers/transfer-handler.ts | 8 ++++++++ .../src/handlers/update-clob-pair-handler.ts | 8 ++++++++ .../src/handlers/update-perpetual-handler.ts | 8 ++++++++ .../dydx_block_processor_ordered_handlers.sql | 15 +++++++++++++++ .../dydx_block_processor_unordered_handlers.sql | 15 +++++++++++++++ 22 files changed, 197 insertions(+), 9 deletions(-) diff --git a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts index 669810deb9..afe0df78fa 100644 --- a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts @@ -233,6 +233,14 @@ describe('fundingHandler', () => { })); expect(stats.gauge).toHaveBeenCalledWith('ender.funding_index_update_event', 0.1, { ticker: 'BTC-USD' }); expect(stats.gauge).toHaveBeenCalledWith('ender.funding_index_update', 0.1, { ticker: 'BTC-USD' }); + expect(stats.timing).toHaveBeenCalledWith( + 'ender.handle_funding_event.sql_latency', + expect.any(Number), + { + className: 'FundingHandler', + eventType: 'FundingEvent', + }, + ); }); it('successfully processes and clears cache for a new funding rate', async () => { diff --git a/indexer/services/ender/src/handlers/asset-handler.ts b/indexer/services/ender/src/handlers/asset-handler.ts index b505f6af5c..34cd516050 100644 --- a/indexer/services/ender/src/handlers/asset-handler.ts +++ b/indexer/services/ender/src/handlers/asset-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { AssetFromDatabase, AssetModel, @@ -6,6 +7,7 @@ import { import { AssetCreateEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../config'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -18,6 +20,12 @@ export class AssetCreationHandler extends Handler { // eslint-disable-next-line @typescript-eslint/require-await public async internalHandle(resultRow: pg.QueryResultRow): Promise { + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_asset_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); const asset: AssetFromDatabase = AssetModel.fromJson( resultRow.asset) as AssetFromDatabase; assetRefresher.addAsset(asset); diff --git a/indexer/services/ender/src/handlers/funding-handler.ts b/indexer/services/ender/src/handlers/funding-handler.ts index b05af85806..5485dfe84c 100644 --- a/indexer/services/ender/src/handlers/funding-handler.ts +++ b/indexer/services/ender/src/handlers/funding-handler.ts @@ -141,6 +141,13 @@ export class FundingHandler extends Handler { }); stats.increment(`${config.SERVICE_NAME}.handle_funding_event.failure`, 1); } + + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_funding_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); } await Promise.all(promises); diff --git a/indexer/services/ender/src/handlers/liquidity-tier-handler.ts b/indexer/services/ender/src/handlers/liquidity-tier-handler.ts index 92e1f7a901..b36df1812a 100644 --- a/indexer/services/ender/src/handlers/liquidity-tier-handler.ts +++ b/indexer/services/ender/src/handlers/liquidity-tier-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { LiquidityTiersFromDatabase, LiquidityTiersModel, @@ -9,6 +10,7 @@ import { LiquidityTierUpsertEventV1, LiquidityTierUpsertEventV2 } from '@dydxpro import _ from 'lodash'; import * as pg from 'pg'; +import config from '../config'; import { generatePerpetualMarketMessage } from '../helpers/kafka-helper'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -21,6 +23,12 @@ export class LiquidityTierHandlerBase extends Handler { // eslint-disable-next-line @typescript-eslint/require-await public async internalHandle(resultRow: pg.QueryResultRow): Promise { + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_liquidity_tier_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); const liquidityTier: LiquidityTiersFromDatabase = LiquidityTiersModel.fromJson( resultRow.liquidity_tier, ) as LiquidityTiersFromDatabase; diff --git a/indexer/services/ender/src/handlers/markets/market-create-handler.ts b/indexer/services/ender/src/handlers/markets/market-create-handler.ts index 6f539e3665..aa8bc571b3 100644 --- a/indexer/services/ender/src/handlers/markets/market-create-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-create-handler.ts @@ -1,7 +1,8 @@ -import { logger } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../../config'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { Handler } from '../handler'; @@ -14,12 +15,18 @@ export class MarketCreateHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(_: pg.QueryResultRow): Promise { + public async internalHandle(resultRow: pg.QueryResultRow): Promise { logger.info({ at: 'MarketCreateHandler#handle', message: 'Received MarketEvent with MarketCreate.', event: this.event, }); + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_market_create_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); return []; } diff --git a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts index eeee9188e3..88f416ef17 100644 --- a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts @@ -1,7 +1,8 @@ -import { logger } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../../config'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { Handler } from '../handler'; @@ -14,13 +15,20 @@ export class MarketModifyHandler extends Handler { } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(_: pg.QueryResultRow): Promise { + public async internalHandle(resultRow: pg.QueryResultRow): Promise { logger.info({ at: 'MarketModifyHandler#handle', message: 'Received MarketEvent with MarketModify.', event: this.event, }); + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_market_modify_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); + return []; } } diff --git a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts index 56c9978fd9..d11dd89ade 100644 --- a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts @@ -1,4 +1,4 @@ -import { logger } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { MarketFromDatabase, OraclePriceFromDatabase, @@ -8,6 +8,7 @@ import { import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../../config'; import { generateOraclePriceContents } from '../../helpers/kafka-helper'; import { ConsolidatedKafkaEvent, @@ -35,6 +36,13 @@ export class MarketPriceUpdateHandler extends Handler { const oraclePrice: OraclePriceFromDatabase = OraclePriceModel.fromJson( resultRow.oracle_price) as OraclePriceFromDatabase; + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_market_price_update_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); + return [ this.generateKafkaEvent( oraclePrice, market.pair, diff --git a/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts b/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts index ea8687f5d0..ce352487db 100644 --- a/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/deleveraging-handler.ts @@ -1,4 +1,4 @@ -import { logger } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { FillFromDatabase, FillModel, @@ -15,6 +15,7 @@ import { import { DeleveragingEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../../config'; import { SUBACCOUNT_ORDER_FILL_EVENT_TYPE } from '../../constants'; import { annotateWithPnl, convertPerpetualPosition } from '../../helpers/kafka-helper'; import { ConsolidatedKafkaEvent } from '../../lib/types'; @@ -95,6 +96,12 @@ export class DeleveragingHandler extends AbstractOrderFillHandler { + public async internalHandle(resultRow: pg.QueryResultRow): Promise { let order: IndexerOrder; // TODO(IND-334): Remove after deprecating StatefulOrderPlacementEvent if (this.event.orderPlace !== undefined) { @@ -40,6 +43,12 @@ export class StatefulOrderPlacementHandler } else { order = this.event.longTermOrderPlacement!.order!; } + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_stateful_order_placement_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); return this.createKafkaEvents(order); } diff --git a/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts b/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts index 02715d1021..e0667e5f6e 100644 --- a/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts +++ b/indexer/services/ender/src/handlers/stateful-order/stateful-order-removal-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { OrderTable, } from '@dydxprotocol-indexer/postgres'; @@ -10,6 +11,7 @@ import { } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../../config'; import { ConsolidatedKafkaEvent } from '../../lib/types'; import { AbstractStatefulOrderHandler } from '../abstract-stateful-order-handler'; @@ -24,8 +26,14 @@ export class StatefulOrderRemovalHandler extends } // eslint-disable-next-line @typescript-eslint/require-await - public async internalHandle(_: pg.QueryResultRow): Promise { + public async internalHandle(resultRow: pg.QueryResultRow): Promise { const orderIdProto: IndexerOrderId = this.event.orderRemoval!.removedOrderId!; + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_stateful_order_removal_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); return this.createKafkaEvents(orderIdProto); } diff --git a/indexer/services/ender/src/handlers/subaccount-update-handler.ts b/indexer/services/ender/src/handlers/subaccount-update-handler.ts index 157cbcdfed..af3f60b9eb 100644 --- a/indexer/services/ender/src/handlers/subaccount-update-handler.ts +++ b/indexer/services/ender/src/handlers/subaccount-update-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { AssetPositionFromDatabase, AssetPositionModel, @@ -16,6 +17,7 @@ import { import _ from 'lodash'; import * as pg from 'pg'; +import config from '../config'; import { SUBACCOUNT_ORDER_FILL_EVENT_TYPE } from '../constants'; import { addPositionsToContents, annotateWithPnl } from '../helpers/kafka-helper'; import { SubaccountUpdate } from '../lib/translated-types'; @@ -61,6 +63,12 @@ export class SubaccountUpdateHandler extends Handler { marketIdToMarket[marketId], ); } + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_subaccount_update_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); return [ this.generateConsolidatedKafkaEvent( diff --git a/indexer/services/ender/src/handlers/trading-rewards-handler.ts b/indexer/services/ender/src/handlers/trading-rewards-handler.ts index 40cb63b474..85951a349e 100644 --- a/indexer/services/ender/src/handlers/trading-rewards-handler.ts +++ b/indexer/services/ender/src/handlers/trading-rewards-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { SubaccountMessageContents, TradingRewardFromDatabase, @@ -8,6 +9,7 @@ import { TradingRewardsEventV1 } from '@dydxprotocol-indexer/v4-protos'; import _ from 'lodash'; import * as pg from 'pg'; +import config from '../config'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -21,6 +23,12 @@ export class TradingRewardsHandler extends Handler { // eslint-disable-next-line @typescript-eslint/require-await public async internalHandle(resultRow: pg.QueryResultRow): Promise { + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_trading_rewards_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); const tradingRewards: TradingRewardFromDatabase[] = _.map( resultRow.trading_rewards, (tradingReward: object) => { diff --git a/indexer/services/ender/src/handlers/transfer-handler.ts b/indexer/services/ender/src/handlers/transfer-handler.ts index ff95eff157..ae15612534 100644 --- a/indexer/services/ender/src/handlers/transfer-handler.ts +++ b/indexer/services/ender/src/handlers/transfer-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { AssetFromDatabase, AssetModel, @@ -8,6 +9,7 @@ import { import { TransferEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../config'; import { generateTransferContents } from '../helpers/kafka-helper'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -22,6 +24,12 @@ export class TransferHandler extends Handler { // eslint-disable-next-line @typescript-eslint/require-await public async internalHandle(resultRow: pg.QueryResultRow): Promise { + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_transfer_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); const asset: AssetFromDatabase = AssetModel.fromJson( resultRow.asset) as AssetFromDatabase; const transfer: TransferFromDatabase = TransferModel.fromJson( diff --git a/indexer/services/ender/src/handlers/update-clob-pair-handler.ts b/indexer/services/ender/src/handlers/update-clob-pair-handler.ts index 0840835227..9875e2d2f2 100644 --- a/indexer/services/ender/src/handlers/update-clob-pair-handler.ts +++ b/indexer/services/ender/src/handlers/update-clob-pair-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { PerpetualMarketFromDatabase, PerpetualMarketModel, @@ -6,6 +7,7 @@ import { import { UpdateClobPairEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../config'; import { generatePerpetualMarketMessage } from '../helpers/kafka-helper'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -19,6 +21,12 @@ export class UpdateClobPairHandler extends Handler { // eslint-disable-next-line @typescript-eslint/require-await public async internalHandle(resultRow: pg.QueryResultRow): Promise { + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_clob_pair_update_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); const perpetualMarket: PerpetualMarketFromDatabase = PerpetualMarketModel.fromJson( resultRow.perpetual_market) as PerpetualMarketFromDatabase; diff --git a/indexer/services/ender/src/handlers/update-perpetual-handler.ts b/indexer/services/ender/src/handlers/update-perpetual-handler.ts index c3a9175b21..d17d48e130 100644 --- a/indexer/services/ender/src/handlers/update-perpetual-handler.ts +++ b/indexer/services/ender/src/handlers/update-perpetual-handler.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { PerpetualMarketFromDatabase, perpetualMarketRefresher, @@ -6,6 +7,7 @@ import { import { UpdatePerpetualEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; +import config from '../config'; import { generatePerpetualMarketMessage } from '../helpers/kafka-helper'; import { ConsolidatedKafkaEvent } from '../lib/types'; import { Handler } from './handler'; @@ -23,6 +25,12 @@ export class UpdatePerpetualHandler extends Handler { resultRow.perpetual_market) as PerpetualMarketFromDatabase; await perpetualMarketRefresher.upsertPerpetualMarket(perpetualMarket); + // Handle latency from resultRow + stats.timing( + `${config.SERVICE_NAME}.handle_update_perpetual_event.sql_latency`, + Number(resultRow.latency), + this.generateTimingStatsOptions(), + ); return [ this.generateConsolidatedMarketKafkaEvent( diff --git a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql index 12c818ecb6..1a6235d850 100644 --- a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql +++ b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql @@ -23,11 +23,16 @@ DECLARE event_index int; transaction_index int; event_data jsonb; + -- Latency tracking variables + event_start_time timestamp; + event_end_time timestamp; + event_latency interval; BEGIN rval = array_fill(NULL::jsonb, ARRAY[coalesce(jsonb_array_length(block->'events'), 0)]::integer[]); /** Note that arrays are 1-indexed in PostgreSQL and empty arrays return NULL for array_length. */ FOR i in 1..coalesce(array_length(rval, 1), 0) LOOP + event_start_time := clock_timestamp(); event_ = jsonb_array_element(block->'events', i-1); transaction_index = dydx_tendermint_event_to_transaction_index(event_); event_index = (event_->'eventIndex')::int; @@ -67,6 +72,16 @@ BEGIN ELSE NULL; END CASE; + + event_end_time := clock_timestamp(); + event_latency := event_end_time - event_start_time; + + -- Add the event latency in ms to the rval output for this event + rval[i] := jsonb_set( + rval[i], + '{latency}', + to_jsonb(EXTRACT(EPOCH FROM event_latency) * 1000) + ); END LOOP; RETURN rval; diff --git a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql index d6d1d4ee68..9c985c79ca 100644 --- a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql +++ b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql @@ -25,11 +25,16 @@ DECLARE event_index int; transaction_index int; event_data jsonb; + -- Latency tracking variables + event_start_time timestamp; + event_end_time timestamp; + event_latency interval; BEGIN rval = array_fill(NULL::jsonb, ARRAY[coalesce(jsonb_array_length(block->'events'), 0)]::integer[]); /** Note that arrays are 1-indexed in PostgreSQL and empty arrays return NULL for array_length. */ FOR i in 1..coalesce(array_length(rval, 1), 0) LOOP + event_start_time := clock_timestamp(); event_ = jsonb_array_element(block->'events', i-1); transaction_index = dydx_tendermint_event_to_transaction_index(event_); event_index = (event_->'eventIndex')::int; @@ -65,6 +70,16 @@ BEGIN ELSE NULL; END CASE; + + event_end_time := clock_timestamp(); + event_latency := event_end_time - event_start_time; + + -- Add the event latency in ms to the rval output for this event + rval[i] := jsonb_set( + rval[i], + '{latency}', + to_jsonb(EXTRACT(EPOCH FROM event_latency) * 1000) + ); END LOOP; RETURN rval; From 8b933c6de0668f0449adcf8bb0eed05d043fb85f Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:02:10 -0400 Subject: [PATCH 08/41] Add oracle prices index on ("marketId", "effectiveAtHeight") (backport #2368) (#2374) Co-authored-by: dydxwill <119354122+dydxwill@users.noreply.github.com> --- ...rices_market_id_effective_at_height_index.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240926133526_create_oracle_prices_market_id_effective_at_height_index.ts diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240926133526_create_oracle_prices_market_id_effective_at_height_index.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240926133526_create_oracle_prices_market_id_effective_at_height_index.ts new file mode 100644 index 0000000000..8f80339ebf --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240926133526_create_oracle_prices_market_id_effective_at_height_index.ts @@ -0,0 +1,17 @@ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS "oracle_prices_marketid_effectiveatheight_index" ON "oracle_prices" ("marketId", "effectiveAtHeight"); + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(` + DROP INDEX CONCURRENTLY IF EXISTS "oracle_prices_marketid_effectiveatheight_index"; + `); +} + +export const config = { + transaction: false, +}; From 0502f73833dab40b85c97dd23e62df858cd56125 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:39:19 -0400 Subject: [PATCH 09/41] Include getting main subaccount equity / pnl for megavault PnL query. (backport #2376) (#2378) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- indexer/packages/postgres/src/constants.ts | 5 ++ .../api/v4/vault-controller.test.ts | 66 ++++++++++++++++--- .../controllers/api/v4/vault-controller.ts | 27 ++++++-- 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/indexer/packages/postgres/src/constants.ts b/indexer/packages/postgres/src/constants.ts index ac9a38e6cb..aa3dbc9bda 100644 --- a/indexer/packages/postgres/src/constants.ts +++ b/indexer/packages/postgres/src/constants.ts @@ -129,3 +129,8 @@ export const DEFAULT_POSTGRES_OPTIONS : Options = config.USE_READ_REPLICA export const MAX_PARENT_SUBACCOUNTS: number = 128; export const CHILD_SUBACCOUNT_MULTIPLIER: number = 1000; + +// From https://github.com/dydxprotocol/v4-chain/blob/protocol/v7.0.0-dev0/protocol/app/module_accounts_test.go#L41 +export const MEGAVAULT_MODULE_ADDRESS: string = 'dydx18tkxrnrkqc2t0lr3zxr5g6a4hdvqksylxqje4r'; +// Generated from the module address + subaccount number 0. +export const MEGAVAULT_SUBACCOUNT_ID: string = 'c7169f81-0c80-54c5-a41f-9cbb6a538fdf'; diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 488435f8e7..e53a9b5b76 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -13,6 +13,8 @@ import { FundingIndexUpdatesTable, PnlTicksFromDatabase, VaultTable, + MEGAVAULT_MODULE_ADDRESS, + MEGAVAULT_SUBACCOUNT_ID, } from '@dydxprotocol-indexer/postgres'; import { RequestMethod, VaultHistoricalPnl } from '../../../../src/types'; import request from 'supertest'; @@ -32,6 +34,7 @@ describe('vault-controller#V4', () => { const initialFundingIndex: string = '10000'; const vault1Equity: number = 159500; const vault2Equity: number = 10000; + const mainVaultEquity: number = 10000; beforeAll(async () => { await dbHelpers.migrate(); @@ -69,6 +72,12 @@ describe('vault-controller#V4', () => { }), ]); await SubaccountTable.create(testConstants.vaultSubaccount); + await SubaccountTable.create({ + address: MEGAVAULT_MODULE_ADDRESS, + subaccountNumber: 0, + updatedAt: latestTime.toISO(), + updatedAtHeight: latestBlockHeight, + }); await Promise.all([ PerpetualPositionTable.create( testConstants.defaultPerpetualPosition, @@ -146,7 +155,7 @@ describe('vault-controller#V4', () => { ['no resolution', '', [1, 2]], ['daily resolution', '?resolution=day', [1, 2]], ['hourly resolution', '?resolution=hour', [1, 2, 3]], - ])('Get /megavault/historicalPnl with 2 vault subaccounts (%s)', async ( + ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s)', async ( _name: string, queryParam: string, expectedTicksIndex: number[], @@ -162,22 +171,28 @@ describe('vault-controller#V4', () => { address: testConstants.vaultAddress, clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, }), + AssetPositionTable.upsert({ + ...testConstants.defaultAssetPosition, + subaccountId: MEGAVAULT_SUBACCOUNT_ID, + }), ]); - const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); + const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks( + true, // createMainSubaccounPnlTicks + ); const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: `/v4/vault/v1/megavault/historicalPnl${queryParam}`, }); const expectedPnlTickBase: any = { - equity: (parseFloat(testConstants.defaultPnlTick.equity) * 2).toString(), - totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) * 2).toString(), - netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) * 2).toString(), + equity: (parseFloat(testConstants.defaultPnlTick.equity) * 3).toString(), + totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) * 3).toString(), + netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) * 3).toString(), }; const finalTick: PnlTicksFromDatabase = { ...expectedPnlTickBase, - equity: Big(vault1Equity).add(vault2Equity).toFixed(), + equity: Big(vault1Equity).add(vault2Equity).add(mainVaultEquity).toFixed(), blockHeight: latestBlockHeight, blockTime: latestTime.toISO(), createdAt: latestTime.toISO(), @@ -449,8 +464,10 @@ describe('vault-controller#V4', () => { }); }); - async function createPnlTicks(): Promise { - return Promise.all([ + async function createPnlTicks( + createMainSubaccountPnlTicks: boolean = false, + ): Promise { + const createdTicks: PnlTicksFromDatabase[] = await Promise.all([ PnlTicksTable.create(testConstants.defaultPnlTick), PnlTicksTable.create({ ...testConstants.defaultPnlTick, @@ -496,5 +513,38 @@ describe('vault-controller#V4', () => { blockHeight: currentBlockHeight, }), ]); + + if (createMainSubaccountPnlTicks) { + const mainSubaccountTicks: PnlTicksFromDatabase[] = await Promise.all([ + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + subaccountId: MEGAVAULT_SUBACCOUNT_ID, + }), + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + subaccountId: MEGAVAULT_SUBACCOUNT_ID, + blockTime: twoDaysAgo.toISO(), + createdAt: twoDaysAgo.toISO(), + blockHeight: twoDayBlockHeight, + }), + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + subaccountId: MEGAVAULT_SUBACCOUNT_ID, + blockTime: twoHoursAgo.toISO(), + createdAt: twoHoursAgo.toISO(), + blockHeight: twoHourBlockHeight, + }), + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + subaccountId: MEGAVAULT_SUBACCOUNT_ID, + blockTime: currentTime.toISO(), + createdAt: currentTime.toISO(), + blockHeight: currentBlockHeight, + }), + ]); + createdTicks.push(...mainSubaccountTicks); + } + + return createdTicks; } }); diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index e377f40c9b..6b967f3872 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -24,6 +24,7 @@ import { PnlTickInterval, VaultTable, VaultFromDatabase, + MEGAVAULT_SUBACCOUNT_ID, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; import express from 'express'; @@ -70,18 +71,24 @@ class VaultController extends Controller { @Query() resolution?: PnlTickInterval, ): Promise { const vaultSubaccounts: VaultMapping = await getVaultMapping(); + const vaultSubaccountIdsWithMainSubaccount: string[] = _ + .keys(vaultSubaccounts) + .concat([MEGAVAULT_SUBACCOUNT_ID]); const [ vaultPnlTicks, vaultPositions, latestBlock, + mainSubaccountEquity, ] : [ PnlTicksFromDatabase[], Map, BlockFromDatabase, + string, ] = await Promise.all([ - getVaultSubaccountPnlTicks(vaultSubaccounts, resolution), + getVaultSubaccountPnlTicks(vaultSubaccountIdsWithMainSubaccount, resolution), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), + getMainSubaccountEquity(), ]); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight @@ -92,7 +99,7 @@ class VaultController extends Controller { return position.equity; }).reduce((acc: string, curr: string): string => { return (Big(acc).add(Big(curr))).toFixed(); - }, '0'); + }, mainSubaccountEquity); const pnlTicksWithCurrentTick: PnlTicksFromDatabase[] = getPnlTicksWithCurrentTick( currentEquity, Array.from(aggregatedPnlTicks.values()), @@ -100,7 +107,7 @@ class VaultController extends Controller { ); return { - megavaultPnl: pnlTicksWithCurrentTick.map( + megavaultPnl: _.sortBy(pnlTicksWithCurrentTick, 'blockTime').map( (pnlTick: PnlTicksFromDatabase) => { return pnlTicksToResponseObject(pnlTick); }), @@ -121,7 +128,7 @@ class VaultController extends Controller { Map, BlockFromDatabase, ] = await Promise.all([ - getVaultSubaccountPnlTicks(vaultSubaccounts, resolution), + getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), resolution), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), ]); @@ -286,10 +293,9 @@ router.get( }); async function getVaultSubaccountPnlTicks( - vaultSubaccounts: VaultMapping, + vaultSubaccountIds: string[], resolution?: PnlTickInterval, ): Promise { - const vaultSubaccountIds: string[] = _.keys(vaultSubaccounts); if (vaultSubaccountIds.length === 0) { return []; } @@ -437,6 +443,15 @@ async function getVaultPositions( )); } +async function getMainSubaccountEquity(): Promise { + // Main vault subaccount should only ever hold a USDC and never any perpetuals. + const usdcBalance: {[subaccountId: string]: Big} = await AssetPositionTable + .findUsdcPositionForSubaccounts( + [MEGAVAULT_SUBACCOUNT_ID], + ); + return usdcBalance[MEGAVAULT_SUBACCOUNT_ID]?.toFixed() || '0'; +} + function getPnlTicksWithCurrentTick( equity: string, pnlTicks: PnlTicksFromDatabase[], From 1389214a64d5ebc0207f02ba93604e38fccbcbd9 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:25:05 -0400 Subject: [PATCH 10/41] Add function to fetch availability zone id (#2326) (backport #2390) (#2410) Co-authored-by: roy-dydx <133032749+roy-dydx@users.noreply.github.com> --- indexer/packages/base/package.json | 1 + indexer/packages/base/src/az-id.ts | 51 ++ indexer/packages/base/src/config.ts | 1 + indexer/packages/base/src/index.ts | 1 + indexer/pnpm-lock.yaml | 903 +++++++++++++++++++++++++++- 5 files changed, 955 insertions(+), 2 deletions(-) create mode 100644 indexer/packages/base/src/az-id.ts diff --git a/indexer/packages/base/package.json b/indexer/packages/base/package.json index b4c9e344eb..25219008df 100644 --- a/indexer/packages/base/package.json +++ b/indexer/packages/base/package.json @@ -32,6 +32,7 @@ }, "homepage": "https://github.com/dydxprotocol/indexer#readme", "dependencies": { + "@aws-sdk/client-ec2": "^3.354.0", "axios": "^1.2.1", "big.js": "^6.2.1", "bignumber.js": "^9.0.2", diff --git a/indexer/packages/base/src/az-id.ts b/indexer/packages/base/src/az-id.ts new file mode 100644 index 0000000000..d790c5f8cc --- /dev/null +++ b/indexer/packages/base/src/az-id.ts @@ -0,0 +1,51 @@ +import { DescribeAvailabilityZonesCommand, EC2Client } from '@aws-sdk/client-ec2'; + +import { axiosRequest } from './axios'; +import config from './config'; +import logger from './logger'; + +export async function getAvailabilityZoneId(): Promise { + if (config.ECS_CONTAINER_METADATA_URI_V4 !== '' && config.AWS_REGION !== '') { + const taskUrl = `${config.ECS_CONTAINER_METADATA_URI_V4}/task`; + try { + const response = await axiosRequest({ + method: 'GET', + url: taskUrl, + }) as { AvailabilityZone: string }; + const client = new EC2Client({ region: config.AWS_REGION }); + const command = new DescribeAvailabilityZonesCommand({ + ZoneNames: [response.AvailabilityZone], + }); + try { + const ec2Response = await client.send(command); + const zoneId = ec2Response.AvailabilityZones![0].ZoneId!; + logger.info({ + at: 'az-id#getAvailabilityZoneId', + message: `Got availability zone id ${zoneId}.`, + }); + return ec2Response.AvailabilityZones![0].ZoneId!; + } catch (error) { + logger.error({ + at: 'az-id#getAvailabilityZoneId', + message: 'Failed to fetch availabilty zone id from EC2. ', + error, + }); + return ''; + } + } catch (error) { + logger.error({ + at: 'az-id#getAvailabilityZoneId', + message: 'Failed to retrieve availability zone from metadata endpoint. No availabilty zone id found.', + error, + taskUrl, + }); + return ''; + } + } else { + logger.error({ + at: 'az-id#getAvailabilityZoneId', + message: 'No metadata URI or region. No availabilty zone id found.', + }); + return ''; + } +} diff --git a/indexer/packages/base/src/config.ts b/indexer/packages/base/src/config.ts index 1d88cc3284..4cee7802e7 100644 --- a/indexer/packages/base/src/config.ts +++ b/indexer/packages/base/src/config.ts @@ -39,6 +39,7 @@ export const baseConfigSchema = { STATSD_PORT: parseInteger({ default: 8125 }), LOG_LEVEL: parseString({ default: 'debug' }), ECS_CONTAINER_METADATA_URI_V4: parseString({ default: '' }), + AWS_REGION: parseString({ default: '' }), }; export default parseSchema(baseConfigSchema); diff --git a/indexer/packages/base/src/index.ts b/indexer/packages/base/src/index.ts index 5ec5dae958..77bbdf34e8 100644 --- a/indexer/packages/base/src/index.ts +++ b/indexer/packages/base/src/index.ts @@ -14,6 +14,7 @@ export * from './bugsnag'; export * from './stats-util'; export * from './date-helpers'; export * from './instance-id'; +export * from './az-id'; // Do this outside logger.ts to avoid a dependency cycle with logger transports that may trigger // additional logging. diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 1ba4eae427..3f7362293d 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -13,6 +13,7 @@ importers: packages/base: specifiers: + '@aws-sdk/client-ec2': ^3.354.0 '@bugsnag/core': ^7.18.0 '@bugsnag/js': ^7.18.0 '@bugsnag/node': ^7.18.0 @@ -36,6 +37,7 @@ importers: winston: ^3.8.1 winston-transport: ^4.5.0 dependencies: + '@aws-sdk/client-ec2': 3.658.1 '@bugsnag/core': 7.18.0 '@bugsnag/js': 7.18.0 '@bugsnag/node': 7.18.0 @@ -888,6 +890,18 @@ packages: tslib: 1.14.1 dev: false + /@aws-crypto/sha256-browser/5.2.0: + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-locate-window': 3.310.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.7.0 + dev: false + /@aws-crypto/sha256-js/3.0.0: resolution: {integrity: sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==} dependencies: @@ -896,12 +910,27 @@ packages: tslib: 1.14.1 dev: false + /@aws-crypto/sha256-js/5.2.0: + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.654.0 + tslib: 2.7.0 + dev: false + /@aws-crypto/supports-web-crypto/3.0.0: resolution: {integrity: sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==} dependencies: tslib: 1.14.1 dev: false + /@aws-crypto/supports-web-crypto/5.2.0: + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + dependencies: + tslib: 2.7.0 + dev: false + /@aws-crypto/util/3.0.0: resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==} dependencies: @@ -910,6 +939,14 @@ packages: tslib: 1.14.1 dev: false + /@aws-crypto/util/5.2.0: + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.7.0 + dev: false + /@aws-sdk/abort-controller/3.347.0: resolution: {integrity: sha512-P/2qE6ntYEmYG4Ez535nJWZbXqgbkJx8CMz7ChEuEg3Gp3dvVYEKg+iEUEvlqQ2U5dWP5J3ehw5po9t86IsVPQ==} engines: {node: '>=14.0.0'} @@ -918,6 +955,58 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/client-ec2/3.658.1: + resolution: {integrity: sha512-J/TdGg7Z8pwIL826QKwaX/EgND5Tst5N5hKcjwnj0jGfsJOkRTMdZTwOgvShYWgs6BplFFZqkl3t2dKsNfsVcg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.658.1_@aws-sdk+client-sts@3.658.1 + '@aws-sdk/client-sts': 3.658.1 + '@aws-sdk/core': 3.658.1 + '@aws-sdk/credential-provider-node': 3.658.1_48432e9e1d4872afb099a0b2260c0550 + '@aws-sdk/middleware-host-header': 3.654.0 + '@aws-sdk/middleware-logger': 3.654.0 + '@aws-sdk/middleware-recursion-detection': 3.654.0 + '@aws-sdk/middleware-sdk-ec2': 3.658.1 + '@aws-sdk/middleware-user-agent': 3.654.0 + '@aws-sdk/region-config-resolver': 3.654.0 + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-endpoints': 3.654.0 + '@aws-sdk/util-user-agent-browser': 3.654.0 + '@aws-sdk/util-user-agent-node': 3.654.0 + '@smithy/config-resolver': 3.0.8 + '@smithy/core': 2.4.6 + '@smithy/fetch-http-handler': 3.2.8 + '@smithy/hash-node': 3.0.6 + '@smithy/invalid-dependency': 3.0.6 + '@smithy/middleware-content-length': 3.0.8 + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/middleware-retry': 3.0.21 + '@smithy/middleware-serde': 3.0.6 + '@smithy/middleware-stack': 3.0.6 + '@smithy/node-config-provider': 3.1.7 + '@smithy/node-http-handler': 3.2.3 + '@smithy/protocol-http': 4.1.3 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/url-parser': 3.0.6 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.21 + '@smithy/util-defaults-mode-node': 3.0.21 + '@smithy/util-endpoints': 2.1.2 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-retry': 3.0.6 + '@smithy/util-utf8': 3.0.0 + '@smithy/util-waiter': 3.1.5 + tslib: 2.7.0 + uuid: 9.0.1 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/client-ecr/3.354.0: resolution: {integrity: sha512-gy6cNm2y4TatqCoGkUspAgPfEGT2fsMIZGrfyPcx7cLsOiq+L5Wbs5AWFw6jywaRC88c6raUrpFLkPeVZjDZiQ==} engines: {node: '>=14.0.0'} @@ -1138,6 +1227,56 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso-oidc/3.658.1_@aws-sdk+client-sts@3.658.1: + resolution: {integrity: sha512-RGcZAI3qEA05JszPKwa0cAyp8rnS1nUvs0Sqw4hqLNQ1kD7b7V6CPjRXe7EFQqCOMvM4kGqx0+cEEVTOmBsFLw==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sts': ^3.658.1 + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.658.1 + '@aws-sdk/core': 3.658.1 + '@aws-sdk/credential-provider-node': 3.658.1_48432e9e1d4872afb099a0b2260c0550 + '@aws-sdk/middleware-host-header': 3.654.0 + '@aws-sdk/middleware-logger': 3.654.0 + '@aws-sdk/middleware-recursion-detection': 3.654.0 + '@aws-sdk/middleware-user-agent': 3.654.0 + '@aws-sdk/region-config-resolver': 3.654.0 + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-endpoints': 3.654.0 + '@aws-sdk/util-user-agent-browser': 3.654.0 + '@aws-sdk/util-user-agent-node': 3.654.0 + '@smithy/config-resolver': 3.0.8 + '@smithy/core': 2.4.6 + '@smithy/fetch-http-handler': 3.2.8 + '@smithy/hash-node': 3.0.6 + '@smithy/invalid-dependency': 3.0.6 + '@smithy/middleware-content-length': 3.0.8 + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/middleware-retry': 3.0.21 + '@smithy/middleware-serde': 3.0.6 + '@smithy/middleware-stack': 3.0.6 + '@smithy/node-config-provider': 3.1.7 + '@smithy/node-http-handler': 3.2.3 + '@smithy/protocol-http': 4.1.3 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/url-parser': 3.0.6 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.21 + '@smithy/util-defaults-mode-node': 3.0.21 + '@smithy/util-endpoints': 2.1.2 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-retry': 3.0.6 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/client-sso/3.353.0: resolution: {integrity: sha512-/dP5jLvZYskk6eVxI/5uaC1AVEbE7B2yuQ+9O3Z9plPIlZXyZxzXHf06s4gwsS4hAc7TDs3DaB+AnfMVLOPHbQ==} engines: {node: '>=14.0.0'} @@ -1220,6 +1359,52 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso/3.658.1: + resolution: {integrity: sha512-lOuaBtqPTYGn6xpXlQF4LsNDsQ8Ij2kOdnk+i69Kp6yS76TYvtUuukyLL5kx8zE1c8WbYtxj9y8VNw9/6uKl7Q==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.658.1 + '@aws-sdk/middleware-host-header': 3.654.0 + '@aws-sdk/middleware-logger': 3.654.0 + '@aws-sdk/middleware-recursion-detection': 3.654.0 + '@aws-sdk/middleware-user-agent': 3.654.0 + '@aws-sdk/region-config-resolver': 3.654.0 + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-endpoints': 3.654.0 + '@aws-sdk/util-user-agent-browser': 3.654.0 + '@aws-sdk/util-user-agent-node': 3.654.0 + '@smithy/config-resolver': 3.0.8 + '@smithy/core': 2.4.6 + '@smithy/fetch-http-handler': 3.2.8 + '@smithy/hash-node': 3.0.6 + '@smithy/invalid-dependency': 3.0.6 + '@smithy/middleware-content-length': 3.0.8 + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/middleware-retry': 3.0.21 + '@smithy/middleware-serde': 3.0.6 + '@smithy/middleware-stack': 3.0.6 + '@smithy/node-config-provider': 3.1.7 + '@smithy/node-http-handler': 3.2.3 + '@smithy/protocol-http': 4.1.3 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/url-parser': 3.0.6 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.21 + '@smithy/util-defaults-mode-node': 3.0.21 + '@smithy/util-endpoints': 2.1.2 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-retry': 3.0.6 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/client-sts/3.353.0: resolution: {integrity: sha512-jOnh242TtxG6st60AxLSav0MTgYlJn4c8ZDxk4Wk4+n5bypnXRrqgVXob99lyVnCRfP3OsDl1eilcVp94EXzVw==} engines: {node: '>=14.0.0'} @@ -1310,6 +1495,54 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sts/3.658.1: + resolution: {integrity: sha512-yw9hc5blTnbT1V6mR7Cx9HGc9KQpcLQ1QXj8rntiJi6tIYu3aFNVEyy81JHL7NsuBSeQulJTvHO3y6r3O0sfRg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.658.1_@aws-sdk+client-sts@3.658.1 + '@aws-sdk/core': 3.658.1 + '@aws-sdk/credential-provider-node': 3.658.1_48432e9e1d4872afb099a0b2260c0550 + '@aws-sdk/middleware-host-header': 3.654.0 + '@aws-sdk/middleware-logger': 3.654.0 + '@aws-sdk/middleware-recursion-detection': 3.654.0 + '@aws-sdk/middleware-user-agent': 3.654.0 + '@aws-sdk/region-config-resolver': 3.654.0 + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-endpoints': 3.654.0 + '@aws-sdk/util-user-agent-browser': 3.654.0 + '@aws-sdk/util-user-agent-node': 3.654.0 + '@smithy/config-resolver': 3.0.8 + '@smithy/core': 2.4.6 + '@smithy/fetch-http-handler': 3.2.8 + '@smithy/hash-node': 3.0.6 + '@smithy/invalid-dependency': 3.0.6 + '@smithy/middleware-content-length': 3.0.8 + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/middleware-retry': 3.0.21 + '@smithy/middleware-serde': 3.0.6 + '@smithy/middleware-stack': 3.0.6 + '@smithy/node-config-provider': 3.1.7 + '@smithy/node-http-handler': 3.2.3 + '@smithy/protocol-http': 4.1.3 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/url-parser': 3.0.6 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.21 + '@smithy/util-defaults-mode-node': 3.0.21 + '@smithy/util-endpoints': 2.1.2 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-retry': 3.0.6 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/config-resolver/3.353.0: resolution: {integrity: sha512-rJJ1ebb8E4vfdGWym6jql1vodV+NUEATI1QqlwxQ0AZ8MGPIsT3uR52VyX7gp+yIrLZBJZdGYVNwrWSJgZ3B3w==} engines: {node: '>=14.0.0'} @@ -1330,6 +1563,22 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/core/3.658.1: + resolution: {integrity: sha512-vJVMoMcSKXK2gBRSu9Ywwv6wQ7tXH8VL1fqB1uVxgCqBZ3IHfqNn4zvpMPWrwgO2/3wv7XFyikGQ5ypPTCw4jA==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/core': 2.4.6 + '@smithy/node-config-provider': 3.1.7 + '@smithy/property-provider': 3.1.6 + '@smithy/protocol-http': 4.1.3 + '@smithy/signature-v4': 4.1.4 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/util-middleware': 3.0.6 + fast-xml-parser: 4.4.1 + tslib: 2.7.0 + dev: false + /@aws-sdk/credential-provider-env/3.353.0: resolution: {integrity: sha512-Y4VsNS8O1FAD5J7S5itOhnOghQ5LIXlZ44t35nF8cbcF+JPvY3ToKzYpjYN1jM7DXKqU4shtqgYpzSqxlvEgKQ==} engines: {node: '>=14.0.0'} @@ -1339,6 +1588,31 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/credential-provider-env/3.654.0: + resolution: {integrity: sha512-kogsx3Ql81JouHS7DkheCDU9MYAvK0AokxjcshDveGmf7BbgbWCA8Fnb9wjQyNDaOXNvkZu8Z8rgkX91z324/w==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@aws-sdk/credential-provider-http/3.658.1: + resolution: {integrity: sha512-4ubkJjEVCZflxkZnV1JDQv8P2pburxk1LrEp55telfJRzXrnowzBKwuV2ED0QMNC448g2B3VCaffS+Ct7c4IWQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/fetch-http-handler': 3.2.8 + '@smithy/node-http-handler': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/protocol-http': 4.1.3 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/util-stream': 3.1.8 + tslib: 2.7.0 + dev: false + /@aws-sdk/credential-provider-imds/3.353.0: resolution: {integrity: sha512-n70yvXBN7E6NX7vA/wLTqyVayu/QKYsPvVn8Y+0A/j5oXXlVY+hQvjjEaNo0Zq1U8Z0L/kj3mutDpe57nTLKSg==} engines: {node: '>=14.0.0'} @@ -1395,6 +1669,29 @@ packages: - aws-crt dev: false + /@aws-sdk/credential-provider-ini/3.658.1_48432e9e1d4872afb099a0b2260c0550: + resolution: {integrity: sha512-2uwOamQg5ppwfegwen1ddPu5HM3/IBSnaGlaKLFhltkdtZ0jiqTZWUtX2V+4Q+buLnT0hQvLS/frQ+7QUam+0Q==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sts': ^3.658.1 + dependencies: + '@aws-sdk/client-sts': 3.658.1 + '@aws-sdk/credential-provider-env': 3.654.0 + '@aws-sdk/credential-provider-http': 3.658.1 + '@aws-sdk/credential-provider-process': 3.654.0 + '@aws-sdk/credential-provider-sso': 3.658.1_@aws-sdk+client-sso-oidc@3.658.1 + '@aws-sdk/credential-provider-web-identity': 3.654.0_@aws-sdk+client-sts@3.658.1 + '@aws-sdk/types': 3.654.0 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + dev: false + /@aws-sdk/credential-provider-node/3.353.0: resolution: {integrity: sha512-OIyZ7OG1OQJ1aQGAu78hggSkK4jiWO1/Sm6wj5wvwylbST8NnR+dHjikZGFB3hoYt1uEe2O2LeGW67bI54VIEQ==} engines: {node: '>=14.0.0'} @@ -1431,6 +1728,28 @@ packages: - aws-crt dev: false + /@aws-sdk/credential-provider-node/3.658.1_48432e9e1d4872afb099a0b2260c0550: + resolution: {integrity: sha512-XwxW6N+uPXPYAuyq+GfOEdfL/MZGAlCSfB5gEWtLBFmFbikhmEuqfWtI6CD60OwudCUOh6argd21BsJf8o1SJA==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/credential-provider-env': 3.654.0 + '@aws-sdk/credential-provider-http': 3.658.1 + '@aws-sdk/credential-provider-ini': 3.658.1_48432e9e1d4872afb099a0b2260c0550 + '@aws-sdk/credential-provider-process': 3.654.0 + '@aws-sdk/credential-provider-sso': 3.658.1_@aws-sdk+client-sso-oidc@3.658.1 + '@aws-sdk/credential-provider-web-identity': 3.654.0_@aws-sdk+client-sts@3.658.1 + '@aws-sdk/types': 3.654.0 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - '@aws-sdk/client-sts' + - aws-crt + dev: false + /@aws-sdk/credential-provider-process/3.353.0: resolution: {integrity: sha512-IBkuxj3pCdmnTzIcRXhq+5sp1hsWACQLi9fHLK+mDEgaiaO+u2r3Th5tV3rJUfNhZY4qa62QNGsHwsVstVxGvw==} engines: {node: '>=14.0.0'} @@ -1451,6 +1770,17 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/credential-provider-process/3.654.0: + resolution: {integrity: sha512-PmQoo8sZ9Q2Ow8OMzK++Z9lI7MsRUG7sNq3E72DVA215dhtTICTDQwGlXH2AAmIp7n+G9LLRds+4wo2ehG4mkg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/credential-provider-sso/3.353.0: resolution: {integrity: sha512-S16tpQ7Zra2O3PNCV4a89wn8wVEgv8oRwjF7p87AM902fXEuag4VHIhaI/TgANQT737JDA/ZCFL2XSilCbHxYQ==} engines: {node: '>=14.0.0'} @@ -1479,6 +1809,22 @@ packages: - aws-crt dev: false + /@aws-sdk/credential-provider-sso/3.658.1_@aws-sdk+client-sso-oidc@3.658.1: + resolution: {integrity: sha512-YOagVEsZEk9DmgJEBg+4MBXrPcw/tYas0VQ5OVBqC5XHNbi2OBGJqgmjVPesuu393E7W0VQxtJFDS00O1ewQgA==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/client-sso': 3.658.1 + '@aws-sdk/token-providers': 3.654.0_@aws-sdk+client-sso-oidc@3.658.1 + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + dev: false + /@aws-sdk/credential-provider-web-identity/3.353.0: resolution: {integrity: sha512-l3TdZB6tEDhLIl0oLIIy1njlxogpyIXSMW9fpuHBt7LDUwfBdCwVPE6+JpGXra6tJAfRQSv5l0lYx5osSLq98g==} engines: {node: '>=14.0.0'} @@ -1497,6 +1843,19 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/credential-provider-web-identity/3.654.0_@aws-sdk+client-sts@3.658.1: + resolution: {integrity: sha512-6a2g9gMtZToqSu+CusjNK5zvbLJahQ9di7buO3iXgbizXpLXU1rnawCpWxwslMpT5fLgMSKDnKDrr6wdEk7jSw==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sts': ^3.654.0 + dependencies: + '@aws-sdk/client-sts': 3.658.1 + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/eventstream-codec/3.347.0: resolution: {integrity: sha512-61q+SyspjsaQ4sdgjizMyRgVph2CiW4aAtfpoH69EJFJfTxTR/OqnZ9Jx/3YiYi0ksrvDenJddYodfWWJqD8/w==} dependencies: @@ -1604,6 +1963,16 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/middleware-host-header/3.654.0: + resolution: {integrity: sha512-rxGgVHWKp8U2ubMv+t+vlIk7QYUaRCHaVpmUlJv0Wv6Q0KeO9a42T9FxHphjOTlCGQOLcjCreL9CF8Qhtb4mdQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/protocol-http': 4.1.3 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/middleware-logger/3.347.0: resolution: {integrity: sha512-NYC+Id5UCkVn+3P1t/YtmHt75uED06vwaKyxDy0UmB2K66PZLVtwWbLpVWrhbroaw1bvUHYcRyQ9NIfnVcXQjA==} engines: {node: '>=14.0.0'} @@ -1612,6 +1981,15 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/middleware-logger/3.654.0: + resolution: {integrity: sha512-OQYb+nWlmASyXfRb989pwkJ9EVUMP1CrKn2eyTk3usl20JZmKo2Vjis6I0tLUkMSxMhnBJJlQKyWkRpD/u1FVg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/middleware-recursion-detection/3.347.0: resolution: {integrity: sha512-qfnSvkFKCAMjMHR31NdsT0gv5Sq/ZHTUD4yQsSLpbVQ6iYAS834lrzXt41iyEHt57Y514uG7F/Xfvude3u4icQ==} engines: {node: '>=14.0.0'} @@ -1621,6 +1999,16 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/middleware-recursion-detection/3.654.0: + resolution: {integrity: sha512-gKSomgltKVmsT8sC6W7CrADZ4GHwX9epk3GcH6QhebVO3LA9LRbkL3TwOPUXakxxOLLUTYdOZLIOtFf7iH00lg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/protocol-http': 4.1.3 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/middleware-retry/3.353.0: resolution: {integrity: sha512-v81NEzDGGvnpvFUy388razpicn7STwBA5gItlr3Ukz8ZWWudfQarTBr0nfVyODXb+76du2LwzEQOd6YtfoOZ+w==} engines: {node: '>=14.0.0'} @@ -1647,6 +2035,20 @@ packages: uuid: 8.3.2 dev: false + /@aws-sdk/middleware-sdk-ec2/3.658.1: + resolution: {integrity: sha512-CnkMajiLD8c+PyiqMjdRt3n87oZnd8jw+8mbtB0jX7Q9ED2z+oeG+RTZMXp2QEiZ0Q+7RyKjXf/PLRhARppFog==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-format-url': 3.654.0 + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/protocol-http': 4.1.3 + '@smithy/signature-v4': 4.1.4 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/middleware-sdk-sts/3.353.0: resolution: {integrity: sha512-GDpjznRBjvCvBfyLEhWb/FSmsnFR+nhBQC0N7d8pqWRqI084sy2ZRyQ6hNDWnImi6AvOabTBSfDm6cB5RexDow==} engines: {node: '>=14.0.0'} @@ -1714,6 +2116,17 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/middleware-user-agent/3.654.0: + resolution: {integrity: sha512-liCcqPAyRsr53cy2tYu4qeH4MMN0eh9g6k56XzI5xd4SghXH5YWh4qOYAlQ8T66ZV4nPMtD8GLtLXGzsH8moFg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@aws-sdk/util-endpoints': 3.654.0 + '@smithy/protocol-http': 4.1.3 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/node-config-provider/3.353.0: resolution: {integrity: sha512-4j0dFHAIa0NwQOPZ/PgkyfCWRaaLhilGbL/cOHkndtUdV54WtG+9+21pKNtakfxncF0irtZvVOv/CW/5x909ZQ==} engines: {node: '>=14.0.0'} @@ -1778,6 +2191,18 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/region-config-resolver/3.654.0: + resolution: {integrity: sha512-ydGOrXJxj3x0sJhsXyTmvJVLAE0xxuTWFJihTl67RtaO7VRNtd82I3P3bwoMMaDn5WpmV5mPo8fEUDRlBm3fPg==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/node-config-provider': 3.1.7 + '@smithy/types': 3.4.2 + '@smithy/util-config-provider': 3.0.0 + '@smithy/util-middleware': 3.0.6 + tslib: 2.7.0 + dev: false + /@aws-sdk/service-error-classification/3.347.0: resolution: {integrity: sha512-xZ3MqSY81Oy2gh5g0fCtooAbahqh9VhsF8vcKjVX8+XPbGC8y+kej82+MsMg4gYL8gRFB9u4hgYbNgIS6JTAvg==} engines: {node: '>=14.0.0'} @@ -1862,6 +2287,20 @@ packages: - aws-crt dev: false + /@aws-sdk/token-providers/3.654.0_@aws-sdk+client-sso-oidc@3.658.1: + resolution: {integrity: sha512-D8GeJYmvbfWkQDtTB4owmIobSMexZel0fOoetwvgCQ/7L8VPph3Q2bn1TRRIXvH7wdt6DcDxA3tKMHPBkT3GlA==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@aws-sdk/client-sso-oidc': ^3.654.0 + dependencies: + '@aws-sdk/client-sso-oidc': 3.658.1_@aws-sdk+client-sts@3.658.1 + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/types/3.347.0: resolution: {integrity: sha512-GkCMy79mdjU9OTIe5KT58fI/6uqdf8UmMdWqVHmFJ+UpEzOci7L/uw4sOXWo7xpPzLs6cJ7s5ouGZW4GRPmHFA==} engines: {node: '>=14.0.0'} @@ -1869,6 +2308,14 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/types/3.654.0: + resolution: {integrity: sha512-VWvbED3SV+10QJIcmU/PKjsKilsTV16d1I7/on4bvD/jo1qGeMXqLDBSen3ks/tuvXZF/mFc7ZW/W2DiLVtO7A==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/url-parser/3.347.0: resolution: {integrity: sha512-lhrnVjxdV7hl+yCnJfDZOaVLSqKjxN20MIOiijRiqaWGLGEAiSqBreMhL89X1WKCifxAs4zZf9YB9SbdziRpAA==} dependencies: @@ -1955,6 +2402,26 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/util-endpoints/3.654.0: + resolution: {integrity: sha512-i902fcBknHs0Irgdpi62+QMvzxE+bczvILXigYrlHL4+PiEnlMVpni5L5W1qCkNZXf8AaMrSBuR1NZAGp6UOUw==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/types': 3.4.2 + '@smithy/util-endpoints': 2.1.2 + tslib: 2.7.0 + dev: false + + /@aws-sdk/util-format-url/3.654.0: + resolution: {integrity: sha512-2yAlJ/l1uTJhS52iu4+/EvdIyQhDBL+nATY8rEjFI0H+BHGVrJIH2CL4DByhvi2yvYwsqQX0HYah6pF/yoXukA==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/querystring-builder': 3.0.6 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/util-hex-encoding/3.310.0: resolution: {integrity: sha512-sVN7mcCCDSJ67pI1ZMtk84SKGqyix6/0A1Ab163YKn+lFBQRMKexleZzpYzNGxYzmQS6VanP/cfU7NiLQOaSfA==} engines: {node: '>=14.0.0'} @@ -1966,7 +2433,7 @@ packages: resolution: {integrity: sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==} engines: {node: '>=14.0.0'} dependencies: - tslib: 2.5.0 + tslib: 2.7.0 dev: false /@aws-sdk/util-middleware/3.347.0: @@ -1999,6 +2466,15 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/util-user-agent-browser/3.654.0: + resolution: {integrity: sha512-ykYAJqvnxLt7wfrqya28wuH3/7NdrwzfiFd7NqEVQf7dXVxL5RPEpD7DxjcyQo3DsHvvdUvGZVaQhozycn1pzA==} + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/types': 3.4.2 + bowser: 2.11.0 + tslib: 2.7.0 + dev: false + /@aws-sdk/util-user-agent-node/3.353.0: resolution: {integrity: sha512-wAviGE0NFqGnaBi6JdjCjp/3DA4AprXQayg9fGphRmP6ncOHNHGonPj/60l+Itu+m78V2CbIS76jqCdUtyAZEQ==} engines: {node: '>=14.0.0'} @@ -2027,6 +2503,21 @@ packages: tslib: 2.5.0 dev: false + /@aws-sdk/util-user-agent-node/3.654.0: + resolution: {integrity: sha512-a0ojjdBN6pqv6gB4H/QPPSfhs7mFtlVwnmKCM/QrTaFzN0U810PJ1BST3lBx5sa23I5jWHGaoFY+5q65C3clLQ==} + engines: {node: '>=16.0.0'} + peerDependencies: + aws-crt: '>=1.0.0' + peerDependenciesMeta: + aws-crt: + optional: true + dependencies: + '@aws-sdk/types': 3.654.0 + '@smithy/node-config-provider': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@aws-sdk/util-utf8-browser/3.259.0: resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==} dependencies: @@ -5900,6 +6391,175 @@ packages: dependencies: '@sinonjs/commons': 1.8.3 + /@smithy/abort-controller/3.1.4: + resolution: {integrity: sha512-VupaALAQlXViW3/enTf/f5l5JZYSAxoJL7f0nanhNNKnww6DGCg1oYIuNP78KDugnkwthBO6iEcym16HhWV8RQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/config-resolver/3.0.8: + resolution: {integrity: sha512-Tv1obAC18XOd2OnDAjSWmmthzx6Pdeh63FbLin8MlPiuJ2ATpKkq0NcNOJFr0dO+JmZXnwu8FQxKJ3TKJ3Hulw==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/node-config-provider': 3.1.7 + '@smithy/types': 3.4.2 + '@smithy/util-config-provider': 3.0.0 + '@smithy/util-middleware': 3.0.6 + tslib: 2.7.0 + dev: false + + /@smithy/core/2.4.6: + resolution: {integrity: sha512-6lQQp99hnyuNNIzeTYSzCUXJHwvvFLY7hfdFGSJM95tjRDJGfzWYFRBXPaM9766LiiTsQ561KErtbufzUFSYUg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/middleware-retry': 3.0.21 + '@smithy/middleware-serde': 3.0.6 + '@smithy/protocol-http': 4.1.3 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/credential-provider-imds/3.2.3: + resolution: {integrity: sha512-VoxMzSzdvkkjMJNE38yQgx4CfnmT+Z+5EUXkg4x7yag93eQkVQgZvN3XBSHC/ylfBbLbAtdu7flTCChX9I+mVg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/node-config-provider': 3.1.7 + '@smithy/property-provider': 3.1.6 + '@smithy/types': 3.4.2 + '@smithy/url-parser': 3.0.6 + tslib: 2.7.0 + dev: false + + /@smithy/fetch-http-handler/3.2.8: + resolution: {integrity: sha512-Lqe0B8F5RM7zkw//6avq1SJ8AfaRd3ubFUS1eVp5WszV7p6Ne5hQ4dSuMHDpNRPhgTvj4va9Kd/pcVigHEHRow==} + dependencies: + '@smithy/protocol-http': 4.1.3 + '@smithy/querystring-builder': 3.0.6 + '@smithy/types': 3.4.2 + '@smithy/util-base64': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/hash-node/3.0.6: + resolution: {integrity: sha512-c/FHEdKK/7DU2z6ZE91L36ahyXWayR3B+FzELjnYq7wH5YqIseM24V+pWCS9kFn1Ln8OFGTf+pyYPiHZuX0s/Q==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + '@smithy/util-buffer-from': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/invalid-dependency/3.0.6: + resolution: {integrity: sha512-czM7Ioq3s8pIXht7oD+vmgy4Wfb4XavU/k/irO8NdXFFOx7YAlsCCcKOh/lJD1mJSYQqiR7NmpZ9JviryD/7AQ==} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/is-array-buffer/2.2.0: + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/is-array-buffer/3.0.0: + resolution: {integrity: sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==} + engines: {node: '>=16.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/middleware-content-length/3.0.8: + resolution: {integrity: sha512-VuyszlSO49WKh3H9/kIO2kf07VUwGV80QRiaDxUfP8P8UKlokz381ETJvwLhwuypBYhLymCYyNhB3fLAGBX2og==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/protocol-http': 4.1.3 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/middleware-endpoint/3.1.3: + resolution: {integrity: sha512-KeM/OrK8MVFUsoJsmCN0MZMVPjKKLudn13xpgwIMpGTYpA8QZB2Xq5tJ+RE6iu3A6NhOI4VajDTwBsm8pwwrhg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/middleware-serde': 3.0.6 + '@smithy/node-config-provider': 3.1.7 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + '@smithy/url-parser': 3.0.6 + '@smithy/util-middleware': 3.0.6 + tslib: 2.7.0 + dev: false + + /@smithy/middleware-retry/3.0.21: + resolution: {integrity: sha512-/h0fElV95LekVVEJuSw+aI11S1Y3zIUwBc6h9ZbUv43Gl2weXsbQwjLoet6j/Qtb0phfrSxS6pNg6FqgJOWZkA==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/node-config-provider': 3.1.7 + '@smithy/protocol-http': 4.1.3 + '@smithy/service-error-classification': 3.0.6 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-retry': 3.0.6 + tslib: 2.7.0 + uuid: 9.0.1 + dev: false + + /@smithy/middleware-serde/3.0.6: + resolution: {integrity: sha512-KKTUSl1MzOM0MAjGbudeaVNtIDo+PpekTBkCNwvfZlKndodrnvRo+00USatiyLOc0ujjO9UydMRu3O9dYML7ag==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/middleware-stack/3.0.6: + resolution: {integrity: sha512-2c0eSYhTQ8xQqHMcRxLMpadFbTXg6Zla5l0mwNftFCZMQmuhI7EbAJMx6R5eqfuV3YbJ3QGyS3d5uSmrHV8Khg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/node-config-provider/3.1.7: + resolution: {integrity: sha512-g3mfnC3Oo8pOI0dYuPXLtdW1WGVb3bR2tkV21GNkm0ZvQjLTtamXAwCWt/FCb0HGvKt3gHHmF1XerG0ICfalOg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/node-http-handler/3.2.3: + resolution: {integrity: sha512-/gcm5DJ3k1b1zEInzBGAZC8ntJ+jwrz1NcSIu+9dSXd1FfG0G6QgkDI40tt8/WYUbHtLyo8fEqtm2v29koWo/w==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/abort-controller': 3.1.4 + '@smithy/protocol-http': 4.1.3 + '@smithy/querystring-builder': 3.0.6 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/property-provider/3.1.6: + resolution: {integrity: sha512-NK3y/T7Q/Bw+Z8vsVs9MYIQ5v7gOX7clyrXcwhhIBQhbPgRl6JDrZbusO9qWDhcEus75Tg+VCxtIRfo3H76fpw==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@smithy/protocol-http/1.0.1: resolution: {integrity: sha512-9OrEn0WfOVtBNYJUjUAn9AOiJ4lzERCJJ/JeZs8E6yajTGxBaFRxUnNBHiNqoDJVg076hY36UmEnPx7xXrvUSg==} engines: {node: '>=14.0.0'} @@ -5908,6 +6568,72 @@ packages: tslib: 2.5.0 dev: false + /@smithy/protocol-http/4.1.3: + resolution: {integrity: sha512-GcbMmOYpH9iRqtC05RbRnc/0FssxSTHlmaNhYBTgSgNCYpdR3Kt88u5GAZTBmouzv+Zlj/VRv92J9ruuDeJuEw==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/querystring-builder/3.0.6: + resolution: {integrity: sha512-sQe08RunoObe+Usujn9+R2zrLuQERi3CWvRO3BvnoWSYUaIrLKuAIeY7cMeDax6xGyfIP3x/yFWbEKSXvOnvVg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + '@smithy/util-uri-escape': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/querystring-parser/3.0.6: + resolution: {integrity: sha512-UJKw4LlEkytzz2Wq+uIdHf6qOtFfee/o7ruH0jF5I6UAuU+19r9QV7nU3P/uI0l6+oElRHmG/5cBBcGJrD7Ozg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/service-error-classification/3.0.6: + resolution: {integrity: sha512-53SpchU3+DUZrN7J6sBx9tBiCVGzsib2e4sc512Q7K9fpC5zkJKs6Z9s+qbMxSYrkEkle6hnMtrts7XNkMJJMg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + dev: false + + /@smithy/shared-ini-file-loader/3.1.7: + resolution: {integrity: sha512-IA4K2qTJYXkF5OfVN4vsY1hfnUZjaslEE8Fsr/gGFza4TAC2A9NfnZuSY2srQIbt9bwtjHiAayrRVgKse4Q7fA==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/signature-v4/4.1.4: + resolution: {integrity: sha512-72MiK7xYukNsnLJI9NqvUHqTu0ziEsfMsYNlWpiJfuGQnCTFKpckThlEatirvcA/LmT1h7rRO+pJD06PYsPu9Q==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/is-array-buffer': 3.0.0 + '@smithy/protocol-http': 4.1.3 + '@smithy/types': 3.4.2 + '@smithy/util-hex-encoding': 3.0.0 + '@smithy/util-middleware': 3.0.6 + '@smithy/util-uri-escape': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/smithy-client/3.3.5: + resolution: {integrity: sha512-7IZi8J3Dr9n3tX+lcpmJ/5tCYIqoXdblFBaPuv0SEKZFRpCxE+TqIWL6I3t7jLlk9TWu3JSvEZAhtjB9yvB+zA==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/middleware-endpoint': 3.1.3 + '@smithy/middleware-stack': 3.0.6 + '@smithy/protocol-http': 4.1.3 + '@smithy/types': 3.4.2 + '@smithy/util-stream': 3.1.8 + tslib: 2.7.0 + dev: false + /@smithy/types/1.0.0: resolution: {integrity: sha512-kc1m5wPBHQCTixwuaOh9vnak/iJm21DrSf9UK6yDE5S3mQQ4u11pqAUiKWnlrZnYkeLfAI9UEHj9OaMT1v5Umg==} engines: {node: '>=14.0.0'} @@ -5915,6 +6641,169 @@ packages: tslib: 2.5.0 dev: false + /@smithy/types/3.4.2: + resolution: {integrity: sha512-tHiFcfcVedVBHpmHUEUHOCCih8iZbIAYn9NvPsNzaPm/237I3imdDdZoOC8c87H5HBAVEa06tTgb+OcSWV9g5w==} + engines: {node: '>=16.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/url-parser/3.0.6: + resolution: {integrity: sha512-47Op/NU8Opt49KyGpHtVdnmmJMsp2hEwBdyjuFB9M2V5QVOwA7pBhhxKN5z6ztKGrMw76gd8MlbPuzzvaAncuQ==} + dependencies: + '@smithy/querystring-parser': 3.0.6 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/util-base64/3.0.0: + resolution: {integrity: sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/util-buffer-from': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-body-length-browser/3.0.0: + resolution: {integrity: sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/util-body-length-node/3.0.0: + resolution: {integrity: sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==} + engines: {node: '>=16.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/util-buffer-from/2.2.0: + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-buffer-from/3.0.0: + resolution: {integrity: sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/is-array-buffer': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-config-provider/3.0.0: + resolution: {integrity: sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==} + engines: {node: '>=16.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/util-defaults-mode-browser/3.0.21: + resolution: {integrity: sha512-M/FhTBk4c/SsB91dD/M4gMGfJO7z/qJaM9+XQQIqBOf4qzZYMExnP7R4VdGwxxH8IKMGW+8F0I4rNtVRrcfPoA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@smithy/property-provider': 3.1.6 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + bowser: 2.11.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-defaults-mode-node/3.0.21: + resolution: {integrity: sha512-NiLinPvF86U3S2Pdx/ycqd4bnY5dmFSPNL5KYRwbNjqQFS09M5Wzqk8BNk61/47xCYz1X/6KeiSk9qgYPTtuDw==} + engines: {node: '>= 10.0.0'} + dependencies: + '@smithy/config-resolver': 3.0.8 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/node-config-provider': 3.1.7 + '@smithy/property-provider': 3.1.6 + '@smithy/smithy-client': 3.3.5 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/util-endpoints/2.1.2: + resolution: {integrity: sha512-FEISzffb4H8DLzGq1g4MuDpcv6CIG15fXoQzDH9SjpRJv6h7J++1STFWWinilG0tQh9H1v2UKWG19Jjr2B16zQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/node-config-provider': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/util-hex-encoding/3.0.0: + resolution: {integrity: sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==} + engines: {node: '>=16.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/util-middleware/3.0.6: + resolution: {integrity: sha512-BxbX4aBhI1O9p87/xM+zWy0GzT3CEVcXFPBRDoHAM+pV0eSW156pR+PSYEz0DQHDMYDsYAflC2bQNz2uaDBUZQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/util-retry/3.0.6: + resolution: {integrity: sha512-BRZiuF7IwDntAbevqMco67an0Sr9oLQJqqRCsSPZZHYRnehS0LHDAkJk/pSmI7Z8c/1Vet294H7fY2fWUgB+Rg==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/service-error-classification': 3.0.6 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + + /@smithy/util-stream/3.1.8: + resolution: {integrity: sha512-hoKOqSmb8FD3WLObuB5hwbM7bNIWgcnvkThokTvVq7J5PKjlLUK5qQQcB9zWLHIoSaIlf3VIv2OxZY2wtQjcRQ==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/fetch-http-handler': 3.2.8 + '@smithy/node-http-handler': 3.2.3 + '@smithy/types': 3.4.2 + '@smithy/util-base64': 3.0.0 + '@smithy/util-buffer-from': 3.0.0 + '@smithy/util-hex-encoding': 3.0.0 + '@smithy/util-utf8': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-uri-escape/3.0.0: + resolution: {integrity: sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==} + engines: {node: '>=16.0.0'} + dependencies: + tslib: 2.7.0 + dev: false + + /@smithy/util-utf8/2.3.0: + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-utf8/3.0.0: + resolution: {integrity: sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/util-buffer-from': 3.0.0 + tslib: 2.7.0 + dev: false + + /@smithy/util-waiter/3.1.5: + resolution: {integrity: sha512-jYOSvM3H6sZe3CHjzD2VQNCjWBJs+4DbtwBMvUp9y5EnnwNa7NQxTeYeQw0CKCAdGGZ3QvVkyJmvbvs5M/B10A==} + engines: {node: '>=16.0.0'} + dependencies: + '@smithy/abort-controller': 3.1.4 + '@smithy/types': 3.4.2 + tslib: 2.7.0 + dev: false + /@tootallnate/once/2.0.0: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} @@ -9024,6 +9913,13 @@ packages: strnum: 1.0.5 dev: false + /fast-xml-parser/4.4.1: + resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==} + hasBin: true + dependencies: + strnum: 1.0.5 + dev: false + /fast-xml-parser/4.5.0: resolution: {integrity: sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==} hasBin: true @@ -14246,6 +15142,10 @@ packages: /tslib/2.5.0: resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==} + /tslib/2.7.0: + resolution: {integrity: sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==} + dev: false + /tsoa/5.1.1: resolution: {integrity: sha512-U6+5CyD3+u9Dtza0fBnv4+lgmbZEskYljzRpKf3edGCAGtMKD2rfjtDw9jUdTfWb1FEDvsnR3pRvsSGBXaOdsA==} engines: {node: '>=12.0.0', yarn: '>=1.9.4'} @@ -14523,7 +15423,6 @@ packages: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true dev: false - optional: true /v8-compile-cache-lib/3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} From 6dc0501ce335c8df5fb3965305a920d6842b40ea Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:44:47 -0400 Subject: [PATCH 11/41] Create consumer using rack id (#2352) (backport #2393) (#2411) Co-authored-by: roy-dydx <133032749+roy-dydx@users.noreply.github.com> --- .../packages/kafka/__tests__/consumer.test.ts | 4 +- indexer/packages/kafka/src/consumer.ts | 75 +++++++++++-------- .../src/helpers/kafka/kafka-controller.ts | 6 +- indexer/services/scripts/src/print-block.ts | 6 +- .../src/helpers/kafka/kafka-controller.ts | 5 +- .../src/helpers/kafka/kafka-controller.ts | 6 +- 6 files changed, 56 insertions(+), 46 deletions(-) diff --git a/indexer/packages/kafka/__tests__/consumer.test.ts b/indexer/packages/kafka/__tests__/consumer.test.ts index de801b2dfe..e05d67d5e3 100644 --- a/indexer/packages/kafka/__tests__/consumer.test.ts +++ b/indexer/packages/kafka/__tests__/consumer.test.ts @@ -10,10 +10,10 @@ import { TO_ENDER_TOPIC } from '../src'; describe.skip('consumer', () => { beforeAll(async () => { await Promise.all([ - consumer.connect(), + consumer!.connect(), producer.connect(), ]); - await consumer.subscribe({ topic: TO_ENDER_TOPIC }); + await consumer!.subscribe({ topic: TO_ENDER_TOPIC }); await startConsumer(); }); diff --git a/indexer/packages/kafka/src/consumer.ts b/indexer/packages/kafka/src/consumer.ts index 93c41d12c5..82c26cf1e0 100644 --- a/indexer/packages/kafka/src/consumer.ts +++ b/indexer/packages/kafka/src/consumer.ts @@ -1,4 +1,5 @@ import { + getAvailabilityZoneId, logger, } from '@dydxprotocol-indexer/base'; import { @@ -13,15 +14,10 @@ const groupIdPrefix: string = config.SERVICE_NAME; const groupIdSuffix: string = config.KAFKA_ENABLE_UNIQUE_CONSUMER_GROUP_IDS ? `_${uuidv4()}` : ''; const groupId: string = `${groupIdPrefix}${groupIdSuffix}`; -export const consumer: Consumer = kafka.consumer({ - groupId, - sessionTimeout: config.KAFKA_SESSION_TIMEOUT_MS, - rebalanceTimeout: config.KAFKA_REBALANCE_TIMEOUT_MS, - heartbeatInterval: config.KAFKA_HEARTBEAT_INTERVAL_MS, - maxWaitTimeInMs: config.KAFKA_WAIT_MAX_TIME_MS, - readUncommitted: false, - maxBytes: 4194304, // 4MB -}); +// As a hack, we made this mutable since CommonJS doesn't support top level await. +// Top level await would needed to fetch the az id (used as rack id). +// eslint-disable-next-line import/no-mutable-exports +export let consumer: Consumer | undefined; // List of functions to run per message consumed. let onMessageFunction: (topic: string, message: KafkaMessage) => Promise; @@ -51,38 +47,51 @@ export function updateOnBatchFunction( // Whether the consumer is stopped. let stopped: boolean = false; -consumer.on('consumer.disconnect', async () => { +export async function stopConsumer(): Promise { logger.info({ - at: 'consumers#disconnect', - message: 'Kafka consumer disconnected', + at: 'kafka-consumer#stop', + message: 'Stopping kafka consumer', groupId, }); - if (!stopped) { - await consumer.connect(); - logger.info({ - at: 'kafka-consumer#disconnect', - message: 'Kafka consumer reconnected', - groupId, - }); - } else { + stopped = true; + await consumer!.disconnect(); +} + +export async function initConsumer(): Promise { + consumer = kafka.consumer({ + groupId, + sessionTimeout: config.KAFKA_SESSION_TIMEOUT_MS, + rebalanceTimeout: config.KAFKA_REBALANCE_TIMEOUT_MS, + heartbeatInterval: config.KAFKA_HEARTBEAT_INTERVAL_MS, + maxWaitTimeInMs: config.KAFKA_WAIT_MAX_TIME_MS, + readUncommitted: false, + maxBytes: 4194304, // 4MB + rackId: await getAvailabilityZoneId(), + }); + + consumer!.on('consumer.disconnect', async () => { logger.info({ - at: 'kafka-consumer#disconnect', - message: 'Not reconnecting since task is shutting down', + at: 'consumers#disconnect', + message: 'Kafka consumer disconnected', groupId, }); - } -}); -export async function stopConsumer(): Promise { - logger.info({ - at: 'kafka-consumer#stop', - message: 'Stopping kafka consumer', - groupId, + if (!stopped) { + await consumer!.connect(); + logger.info({ + at: 'kafka-consumer#disconnect', + message: 'Kafka consumer reconnected', + groupId, + }); + } else { + logger.info({ + at: 'kafka-consumer#disconnect', + message: 'Not reconnecting since task is shutting down', + groupId, + }); + } }); - - stopped = true; - await consumer.disconnect(); } export async function startConsumer(batchProcessing: boolean = false): Promise { @@ -104,7 +113,7 @@ export async function startConsumer(batchProcessing: boolean = false): Promise { await Promise.all([ - consumer.connect(), + initConsumer(), producer.connect(), ]); - await consumer.subscribe({ + await consumer!.subscribe({ topic: TO_ENDER_TOPIC, // https://kafka.js.org/docs/consuming#a-name-from-beginning-a-frombeginning // Need to set fromBeginning to true, so when ender restarts, it will consume all messages diff --git a/indexer/services/scripts/src/print-block.ts b/indexer/services/scripts/src/print-block.ts index 05855229ca..34d32a7728 100644 --- a/indexer/services/scripts/src/print-block.ts +++ b/indexer/services/scripts/src/print-block.ts @@ -42,7 +42,7 @@ export function seek(offset: bigint): void { offset: offset.toString(), }); - consumer.seek({ + consumer!.seek({ topic: TO_ENDER_TOPIC, partition: 0, offset: offset.toString(), @@ -57,11 +57,11 @@ export function seek(offset: bigint): void { export async function connect(height: number): Promise { await Promise.all([ - consumer.connect(), + consumer!.connect(), producer.connect(), ]); - await consumer.subscribe({ + await consumer!.subscribe({ topic: TO_ENDER_TOPIC, fromBeginning: true, }); diff --git a/indexer/services/socks/src/helpers/kafka/kafka-controller.ts b/indexer/services/socks/src/helpers/kafka/kafka-controller.ts index 03409f2849..247819f8bf 100644 --- a/indexer/services/socks/src/helpers/kafka/kafka-controller.ts +++ b/indexer/services/socks/src/helpers/kafka/kafka-controller.ts @@ -2,18 +2,19 @@ import { logger } from '@dydxprotocol-indexer/base'; import { WebsocketTopics, consumer, + initConsumer, stopConsumer, } from '@dydxprotocol-indexer/kafka'; export async function connect(): Promise { - await consumer.connect(); + await initConsumer(); logger.info({ at: 'kafka-controller#connect', message: 'Connected to Kafka', }); - await consumer.subscribe({ topics: Object.values(WebsocketTopics) }); + await consumer!.subscribe({ topics: Object.values(WebsocketTopics) }); } export async function disconnect(): Promise { diff --git a/indexer/services/vulcan/src/helpers/kafka/kafka-controller.ts b/indexer/services/vulcan/src/helpers/kafka/kafka-controller.ts index ae2038f0f3..528eb4b851 100644 --- a/indexer/services/vulcan/src/helpers/kafka/kafka-controller.ts +++ b/indexer/services/vulcan/src/helpers/kafka/kafka-controller.ts @@ -1,6 +1,6 @@ import { logger } from '@dydxprotocol-indexer/base'; import { - consumer, producer, KafkaTopics, updateOnMessageFunction, updateOnBatchFunction, + consumer, initConsumer, producer, KafkaTopics, updateOnMessageFunction, updateOnBatchFunction, } from '@dydxprotocol-indexer/kafka'; import { KafkaMessage } from 'kafkajs'; @@ -10,11 +10,11 @@ import { onMessage } from '../../lib/on-message'; export async function connect(): Promise { await Promise.all([ - consumer.connect(), + initConsumer(), producer.connect(), ]); - await consumer.subscribe({ + await consumer!.subscribe({ topic: KafkaTopics.TO_VULCAN, // https://kafka.js.org/docs/consuming#a-name-from-beginning-a-frombeginning // fromBeginning is by default set to false, so vulcan will only consume messages produced From 3e94eb45be82cabc4864dbeff7efdf55af413b8c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:48:18 -0400 Subject: [PATCH 12/41] Filter out to single tick per interval. (backport #2403) (#2418) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../api/v4/vault-controller.test.ts | 68 +++++++++++++---- indexer/services/comlink/src/config.ts | 1 + .../controllers/api/v4/vault-controller.ts | 75 ++++++++++++++++--- 3 files changed, 118 insertions(+), 26 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index e53a9b5b76..72ae841616 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -24,13 +24,15 @@ import Big from 'big.js'; describe('vault-controller#V4', () => { const latestBlockHeight: string = '25'; - const currentBlockHeight: string = '7'; - const twoHourBlockHeight: string = '5'; + const currentBlockHeight: string = '9'; + const twoHourBlockHeight: string = '7'; + const almostTwoDayBlockHeight: string = '5'; const twoDayBlockHeight: string = '3'; const currentTime: DateTime = DateTime.utc().startOf('day').minus({ hour: 5 }); const latestTime: DateTime = currentTime.plus({ second: 5 }); const twoHoursAgo: DateTime = currentTime.minus({ hour: 2 }); const twoDaysAgo: DateTime = currentTime.minus({ day: 2 }); + const almostTwoDaysAgo: DateTime = currentTime.minus({ hour: 47 }); const initialFundingIndex: string = '10000'; const vault1Equity: number = 159500; const vault2Equity: number = 10000; @@ -70,6 +72,11 @@ describe('vault-controller#V4', () => { time: latestTime.toISO(), blockHeight: latestBlockHeight, }), + BlockTable.create({ + ...testConstants.defaultBlock, + time: almostTwoDaysAgo.toISO(), + blockHeight: almostTwoDayBlockHeight, + }), ]); await SubaccountTable.create(testConstants.vaultSubaccount); await SubaccountTable.create({ @@ -152,14 +159,27 @@ describe('vault-controller#V4', () => { }); it.each([ - ['no resolution', '', [1, 2]], - ['daily resolution', '?resolution=day', [1, 2]], - ['hourly resolution', '?resolution=hour', [1, 2, 3]], + ['no resolution', '', [1, 2], [undefined, 6], [9, 10]], + ['daily resolution', '?resolution=day', [1, 2], [undefined, 6], [9, 10]], + [ + 'hourly resolution', + '?resolution=hour', + [1, undefined, 2, 3], + [undefined, 5, 6, 7], + [9, undefined, 10, 11], + ], ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s)', async ( _name: string, queryParam: string, - expectedTicksIndex: number[], + expectedTicksIndex1: (number | undefined)[], + expectedTicksIndex2: (number | undefined)[], + expectedTicksIndexMain: (number | undefined)[], ) => { + const expectedTicksArray: (number | undefined)[][] = [ + expectedTicksIndex1, + expectedTicksIndex2, + expectedTicksIndexMain, + ]; await Promise.all([ VaultTable.create({ ...testConstants.defaultVault, @@ -198,15 +218,33 @@ describe('vault-controller#V4', () => { createdAt: latestTime.toISO(), }; - expect(response.body.megavaultPnl).toHaveLength(expectedTicksIndex.length + 1); + expect(response.body.megavaultPnl).toHaveLength(expectedTicksIndex1.length + 1); expect(response.body.megavaultPnl).toEqual( expect.arrayContaining( - expectedTicksIndex.map((index: number) => { + expectedTicksIndex1.map((_: number | undefined, pos: number) => { + const pnlTickBase: any = { + equity: '0', + totalPnl: '0', + netTransfers: '0', + }; + let expectedTick: PnlTicksFromDatabase; + for (const expectedTicks of expectedTicksArray) { + if (expectedTicks[pos] !== undefined) { + expectedTick = createdPnlTicks[expectedTicks[pos]!]; + pnlTickBase.equity = Big(pnlTickBase.equity).add(expectedTick.equity).toFixed(); + pnlTickBase.totalPnl = Big(pnlTickBase.totalPnl) + .add(expectedTick.totalPnl) + .toFixed(); + pnlTickBase.netTransfers = Big(pnlTickBase.netTransfers) + .add(expectedTick.netTransfers) + .toFixed(); + } + } return expect.objectContaining({ - ...expectedPnlTickBase, - createdAt: createdPnlTicks[index].createdAt, - blockHeight: createdPnlTicks[index].blockHeight, - blockTime: createdPnlTicks[index].blockTime, + ...pnlTickBase, + createdAt: expectedTick!.createdAt, + blockHeight: expectedTick!.blockHeight, + blockTime: expectedTick!.blockTime, }); }).concat([expect.objectContaining(finalTick)]), ), @@ -494,9 +532,9 @@ describe('vault-controller#V4', () => { PnlTicksTable.create({ ...testConstants.defaultPnlTick, subaccountId: testConstants.vaultSubaccountId, - blockTime: twoDaysAgo.toISO(), - createdAt: twoDaysAgo.toISO(), - blockHeight: twoDayBlockHeight, + blockTime: almostTwoDaysAgo.toISO(), + createdAt: almostTwoDaysAgo.toISO(), + blockHeight: almostTwoDayBlockHeight, }), PnlTicksTable.create({ ...testConstants.defaultPnlTick, diff --git a/indexer/services/comlink/src/config.ts b/indexer/services/comlink/src/config.ts index 43d2c81222..1b70d73b4b 100644 --- a/indexer/services/comlink/src/config.ts +++ b/indexer/services/comlink/src/config.ts @@ -61,6 +61,7 @@ export const configSchema = { // Vaults config VAULT_PNL_HISTORY_DAYS: parseInteger({ default: 90 }), + VAULT_PNL_HISTORY_HOURS: parseInteger({ default: 72 }), }; //////////////////////////////////////////////////////////////////////////////// diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 6b967f3872..ee4a31fe64 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -30,6 +30,7 @@ import Big from 'big.js'; import express from 'express'; import { checkSchema, matchedData } from 'express-validator'; import _ from 'lodash'; +import { DateTime } from 'luxon'; import { Controller, Get, Query, Route, } from 'tsoa'; @@ -85,7 +86,7 @@ class VaultController extends Controller { BlockFromDatabase, string, ] = await Promise.all([ - getVaultSubaccountPnlTicks(vaultSubaccountIdsWithMainSubaccount, resolution), + getVaultSubaccountPnlTicks(vaultSubaccountIdsWithMainSubaccount, getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), @@ -102,7 +103,7 @@ class VaultController extends Controller { }, mainSubaccountEquity); const pnlTicksWithCurrentTick: PnlTicksFromDatabase[] = getPnlTicksWithCurrentTick( currentEquity, - Array.from(aggregatedPnlTicks.values()), + filterOutIntervalTicks(aggregatedPnlTicks, getResolution(resolution)), latestBlock, ); @@ -128,7 +129,7 @@ class VaultController extends Controller { Map, BlockFromDatabase, ] = await Promise.all([ - getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), resolution), + getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), ]); @@ -294,21 +295,22 @@ router.get( async function getVaultSubaccountPnlTicks( vaultSubaccountIds: string[], - resolution?: PnlTickInterval, + resolution: PnlTickInterval, ): Promise { if (vaultSubaccountIds.length === 0) { return []; } - let pnlTickInterval: PnlTickInterval; - if (resolution === undefined) { - pnlTickInterval = PnlTickInterval.day; + + let windowSeconds: number; + if (resolution === PnlTickInterval.day) { + windowSeconds = config.VAULT_PNL_HISTORY_DAYS * 24 * 60 * 60; // days to seconds } else { - pnlTickInterval = resolution; + windowSeconds = config.VAULT_PNL_HISTORY_HOURS * 60 * 60; // hours to seconds } const pnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.getPnlTicksAtIntervals( - pnlTickInterval, - config.VAULT_PNL_HISTORY_DAYS * 24 * 60 * 60, + resolution, + windowSeconds, vaultSubaccountIds, ); @@ -461,7 +463,7 @@ function getPnlTicksWithCurrentTick( return []; } const currentTick: PnlTicksFromDatabase = { - ...pnlTicks[pnlTicks.length - 1], + ...(_.maxBy(pnlTicks, 'blockTime')!), equity, blockHeight: latestBlock.blockHeight, blockTime: latestBlock.time, @@ -470,6 +472,57 @@ function getPnlTicksWithCurrentTick( return pnlTicks.concat([currentTick]); } +/** + * Takes in a map of block heights to PnlTicks and filters out the closest pnl tick per interval. + * @param pnlTicksByBlock Map of block number to pnl tick. + * @param resolution Resolution of interval. + * @returns Array of PnlTicksFromDatabase, one per interval. + */ +function filterOutIntervalTicks( + pnlTicksByBlock: Map, + resolution: PnlTickInterval, +): PnlTicksFromDatabase[] { + // Track block to block time. + const blockToBlockTime: Map = new Map(); + // Track start of days to closest block by block time. + const blocksPerInterval: Map = new Map(); + // Track start of days to closest Pnl tick. + const ticksPerInterval: Map = new Map(); + pnlTicksByBlock.forEach((pnlTick: PnlTicksFromDatabase, block: number): void => { + const blockTime: DateTime = DateTime.fromISO(pnlTick.blockTime).toUTC(); + blockToBlockTime.set(block, blockTime); + + const startOfInterval: DateTime = blockTime.toUTC().startOf(resolution); + const startOfIntervalStr: string = startOfInterval.toISO(); + const startOfIntervalBlock: number | undefined = blocksPerInterval.get(startOfIntervalStr); + // No block for the start of interval, set this block as the block for the interval. + if (startOfIntervalBlock === undefined) { + blocksPerInterval.set(startOfIntervalStr, block); + ticksPerInterval.set(startOfIntervalStr, pnlTick); + return; + } + + const startOfDayBlockTime: DateTime | undefined = blockToBlockTime.get(startOfIntervalBlock); + // Invalid block set as start of day block, set this block as the block for the day. + if (startOfDayBlockTime === undefined) { + blocksPerInterval.set(startOfIntervalStr, block); + ticksPerInterval.set(startOfIntervalStr, pnlTick); + return; + } + + // This block is closer to the start of the day, set it as the block for the day. + if (blockTime.diff(startOfInterval) < startOfDayBlockTime.diff(startOfInterval)) { + blocksPerInterval.set(startOfIntervalStr, block); + ticksPerInterval.set(startOfIntervalStr, pnlTick); + } + }); + return Array.from(ticksPerInterval.values()); +} + +function getResolution(resolution: PnlTickInterval = PnlTickInterval.day): PnlTickInterval { + return resolution; +} + async function getVaultMapping(): Promise { const vaults: VaultFromDatabase[] = await VaultTable.findAll( {}, From 6005a0d9c0267874937415455d59355ef0246faa Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:55:25 -0400 Subject: [PATCH 13/41] [CT-629] Fix entryPrice calc (backport #2415) (#2417) Co-authored-by: dydxwill <119354122+dydxwill@users.noreply.github.com> --- .../handlers/order-fills/liquidation-handler.test.ts | 4 ++-- .../__tests__/handlers/order-fills/order-handler.test.ts | 4 ++-- .../helpers/dydx_liquidation_fill_handler_per_order.sql | 2 +- .../dydx_update_perpetual_position_aggregate_fields.sql | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts index a5cc41f3d6..805ca676f0 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts @@ -137,7 +137,7 @@ describe('LiquidationHandler', () => { perpetualId: testConstants.defaultPerpetualMarket.id, side: PositionSide.LONG, status: PerpetualPositionStatus.OPEN, - size: '10', + size: '5', maxSize: '25', sumOpen: '10', entryPrice: '15000', @@ -392,7 +392,7 @@ describe('LiquidationHandler', () => { defaultPerpetualPosition.openEventId, ), { - sumOpen: Big(defaultPerpetualPosition.size).plus(totalFilled).toFixed(), + sumOpen: Big(defaultPerpetualPosition.sumOpen!).plus(totalFilled).toFixed(), entryPrice: getWeightedAverage( defaultPerpetualPosition.entryPrice!, defaultPerpetualPosition.size, diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index ba9a62ab34..1e2cdcb90b 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -138,7 +138,7 @@ describe('OrderHandler', () => { perpetualId: testConstants.defaultPerpetualMarket.id, side: PositionSide.LONG, status: PerpetualPositionStatus.OPEN, - size: '10', + size: '5', maxSize: '25', sumOpen: '10', entryPrice: '15000', @@ -439,7 +439,7 @@ describe('OrderHandler', () => { defaultPerpetualPosition.openEventId, ), { - sumOpen: Big(defaultPerpetualPosition.size).plus(totalFilled).toFixed(), + sumOpen: Big(defaultPerpetualPosition.sumOpen!).plus(totalFilled).toFixed(), entryPrice: getWeightedAverage( defaultPerpetualPosition.entryPrice!, defaultPerpetualPosition.size, diff --git a/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql index 493b1257d6..4e1970c8d2 100644 --- a/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql @@ -200,7 +200,7 @@ BEGIN perpetual_position_record."side", order_side) THEN sum_open = dydx_trim_scale(perpetual_position_record."sumOpen" + fill_amount); entry_price = dydx_get_weighted_average( - perpetual_position_record."entryPrice", perpetual_position_record."sumOpen", + perpetual_position_record."entryPrice", perpetual_position_record."size", maker_price, fill_amount); perpetual_position_record."sumOpen" = sum_open; perpetual_position_record."entryPrice" = entry_price; diff --git a/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql b/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql index d021eecf28..deda80b23f 100644 --- a/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql +++ b/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql @@ -44,7 +44,7 @@ BEGIN IF dydx_perpetual_position_and_order_side_matching(perpetual_position_record."side", side) THEN sum_open := dydx_trim_scale(perpetual_position_record."sumOpen" + size); entry_price := dydx_get_weighted_average( - perpetual_position_record."entryPrice", perpetual_position_record."sumOpen", price, size + perpetual_position_record."entryPrice", perpetual_position_record."size", price, size ); perpetual_position_record."sumOpen" = sum_open; perpetual_position_record."entryPrice" = entry_price; From e8af710ce1bec39e10a491382242fa1c1eb8244e Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:16:36 -0400 Subject: [PATCH 14/41] Revert "[CT-629] Fix entryPrice calc" (backport #2425) (#2426) Co-authored-by: dydxwill <119354122+dydxwill@users.noreply.github.com> --- .../handlers/order-fills/liquidation-handler.test.ts | 4 ++-- .../__tests__/handlers/order-fills/order-handler.test.ts | 4 ++-- .../helpers/dydx_liquidation_fill_handler_per_order.sql | 2 +- .../dydx_update_perpetual_position_aggregate_fields.sql | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts index 805ca676f0..a5cc41f3d6 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts @@ -137,7 +137,7 @@ describe('LiquidationHandler', () => { perpetualId: testConstants.defaultPerpetualMarket.id, side: PositionSide.LONG, status: PerpetualPositionStatus.OPEN, - size: '5', + size: '10', maxSize: '25', sumOpen: '10', entryPrice: '15000', @@ -392,7 +392,7 @@ describe('LiquidationHandler', () => { defaultPerpetualPosition.openEventId, ), { - sumOpen: Big(defaultPerpetualPosition.sumOpen!).plus(totalFilled).toFixed(), + sumOpen: Big(defaultPerpetualPosition.size).plus(totalFilled).toFixed(), entryPrice: getWeightedAverage( defaultPerpetualPosition.entryPrice!, defaultPerpetualPosition.size, diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index 1e2cdcb90b..ba9a62ab34 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -138,7 +138,7 @@ describe('OrderHandler', () => { perpetualId: testConstants.defaultPerpetualMarket.id, side: PositionSide.LONG, status: PerpetualPositionStatus.OPEN, - size: '5', + size: '10', maxSize: '25', sumOpen: '10', entryPrice: '15000', @@ -439,7 +439,7 @@ describe('OrderHandler', () => { defaultPerpetualPosition.openEventId, ), { - sumOpen: Big(defaultPerpetualPosition.sumOpen!).plus(totalFilled).toFixed(), + sumOpen: Big(defaultPerpetualPosition.size).plus(totalFilled).toFixed(), entryPrice: getWeightedAverage( defaultPerpetualPosition.entryPrice!, defaultPerpetualPosition.size, diff --git a/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql index 4e1970c8d2..493b1257d6 100644 --- a/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql @@ -200,7 +200,7 @@ BEGIN perpetual_position_record."side", order_side) THEN sum_open = dydx_trim_scale(perpetual_position_record."sumOpen" + fill_amount); entry_price = dydx_get_weighted_average( - perpetual_position_record."entryPrice", perpetual_position_record."size", + perpetual_position_record."entryPrice", perpetual_position_record."sumOpen", maker_price, fill_amount); perpetual_position_record."sumOpen" = sum_open; perpetual_position_record."entryPrice" = entry_price; diff --git a/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql b/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql index deda80b23f..d021eecf28 100644 --- a/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql +++ b/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql @@ -44,7 +44,7 @@ BEGIN IF dydx_perpetual_position_and_order_side_matching(perpetual_position_record."side", side) THEN sum_open := dydx_trim_scale(perpetual_position_record."sumOpen" + size); entry_price := dydx_get_weighted_average( - perpetual_position_record."entryPrice", perpetual_position_record."size", price, size + perpetual_position_record."entryPrice", perpetual_position_record."sumOpen", price, size ); perpetual_position_record."sumOpen" = sum_open; perpetual_position_record."entryPrice" = entry_price; From dbe13a34238d6536711fba5a2d53d3039f5359ba Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:37:03 -0400 Subject: [PATCH 15/41] Fix flaky vault test. (backport #2422) (#2424) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../__tests__/controllers/api/v4/vault-controller.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 72ae841616..b2071c6681 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -21,6 +21,7 @@ import request from 'supertest'; import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers'; import { DateTime } from 'luxon'; import Big from 'big.js'; +import config from '../../../../src/config'; describe('vault-controller#V4', () => { const latestBlockHeight: string = '25'; @@ -37,6 +38,7 @@ describe('vault-controller#V4', () => { const vault1Equity: number = 159500; const vault2Equity: number = 10000; const mainVaultEquity: number = 10000; + const vaultPnlHistoryHoursPrev: number = config.VAULT_PNL_HISTORY_HOURS; beforeAll(async () => { await dbHelpers.migrate(); @@ -48,6 +50,8 @@ describe('vault-controller#V4', () => { describe('GET /v1', () => { beforeEach(async () => { + // Get a week of data for hourly pnl ticks. + config.VAULT_PNL_HISTORY_HOURS = 168; await testMocks.seedData(); await perpetualMarketRefresher.updatePerpetualMarkets(); await liquidityTierRefresher.updateLiquidityTiers(); @@ -109,6 +113,7 @@ describe('vault-controller#V4', () => { afterEach(async () => { await dbHelpers.clearData(); + config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev; }); it('Get /megavault/historicalPnl with no vault subaccounts', async () => { From 0b9e8a2c41f745f3963d583139e51b7c1586def1 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:46:02 -0400 Subject: [PATCH 16/41] Split affiliate info fees by taker and maker (backport #2439) (#2448) Co-authored-by: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> --- .../postgres/__tests__/helpers/constants.ts | 15 +++++---- .../stores/affiliate-info-table.test.ts | 31 +++++++++++-------- ...44813_change_affiliate_info_fee_columns.ts | 25 +++++++++++++++ .../src/models/affiliate-info-model.ts | 25 +++++++++------ .../src/stores/affiliate-info-table.ts | 24 ++++++++------ .../src/types/affiliate-info-types.ts | 10 +++--- .../postgres/src/types/db-model-types.ts | 5 +-- .../api/v4/affiliates-controller.test.ts | 11 +++++-- .../comlink/public/api-documentation.md | 14 +++++++-- indexer/services/comlink/public/swagger.json | 12 ++++++- .../api/v4/affiliates-controller.ts | 11 +++++-- indexer/services/comlink/src/types.ts | 2 ++ .../tasks/update-affiliate-info.test.ts | 20 +++++++----- 13 files changed, 145 insertions(+), 60 deletions(-) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20241002144813_change_affiliate_info_fee_columns.ts diff --git a/indexer/packages/postgres/__tests__/helpers/constants.ts b/indexer/packages/postgres/__tests__/helpers/constants.ts index 56e2865227..5de24209bd 100644 --- a/indexer/packages/postgres/__tests__/helpers/constants.ts +++ b/indexer/packages/postgres/__tests__/helpers/constants.ts @@ -990,9 +990,10 @@ export const defaultAffiliateInfo: AffiliateInfoCreateObject = { affiliateEarnings: '10', referredMakerTrades: 10, referredTakerTrades: 20, - totalReferredFees: '10', + totalReferredMakerFees: '10', + totalReferredTakerFees: '10', + totalReferredMakerRebates: '-10', totalReferredUsers: 5, - referredNetProtocolEarnings: '20', firstReferralBlockHeight: '1', referredTotalVolume: '1000', }; @@ -1002,9 +1003,10 @@ export const defaultAffiliateInfo2: AffiliateInfoCreateObject = { affiliateEarnings: '11', referredMakerTrades: 11, referredTakerTrades: 21, - totalReferredFees: '11', + totalReferredMakerFees: '11', + totalReferredTakerFees: '11', + totalReferredMakerRebates: '-11', totalReferredUsers: 5, - referredNetProtocolEarnings: '21', firstReferralBlockHeight: '11', referredTotalVolume: '1000', }; @@ -1014,9 +1016,10 @@ export const defaultAffiliateInfo3: AffiliateInfoCreateObject = { affiliateEarnings: '12', referredMakerTrades: 12, referredTakerTrades: 22, - totalReferredFees: '12', + totalReferredMakerFees: '12', + totalReferredTakerFees: '12', + totalReferredMakerRebates: '-12', totalReferredUsers: 10, - referredNetProtocolEarnings: '22', firstReferralBlockHeight: '12', referredTotalVolume: '1111111', }; diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index 04790001c8..9c9eddc6eb 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -113,9 +113,10 @@ describe('Affiliate info store', () => { affiliateEarnings: '1000', referredMakerTrades: 1, referredTakerTrades: 1, - totalReferredFees: '2000', + totalReferredMakerFees: '0', + totalReferredTakerFees: '1000', + totalReferredMakerRebates: '-1000', totalReferredUsers: 1, - referredNetProtocolEarnings: '1000', firstReferralBlockHeight: '1', referredTotalVolume: '2', }; @@ -140,9 +141,10 @@ describe('Affiliate info store', () => { affiliateEarnings: '1000', referredMakerTrades: 2, referredTakerTrades: 0, - totalReferredFees: '2000', + totalReferredMakerFees: '2000', + totalReferredTakerFees: '0', + totalReferredMakerRebates: '0', totalReferredUsers: 1, - referredNetProtocolEarnings: '1000', firstReferralBlockHeight: '1', referredTotalVolume: '2', }; @@ -157,14 +159,15 @@ describe('Affiliate info store', () => { const updatedInfo2 = await AffiliateInfoTable.findById( defaultWallet2.address, ); - const expectedAffiliateInfo2 = { + const expectedAffiliateInfo2: AffiliateInfoFromDatabase = { address: defaultWallet2.address, affiliateEarnings: '2000', referredMakerTrades: 3, referredTakerTrades: 1, - totalReferredFees: '4000', + totalReferredMakerFees: '2000', + totalReferredTakerFees: '1000', + totalReferredMakerRebates: '-1000', totalReferredUsers: 1, - referredNetProtocolEarnings: '2000', firstReferralBlockHeight: '1', referredTotalVolume: '4', }; @@ -183,14 +186,15 @@ describe('Affiliate info store', () => { const updatedInfo3 = await AffiliateInfoTable.findById( defaultWallet2.address, ); - const expectedAffiliateInfo3 = { + const expectedAffiliateInfo3: AffiliateInfoFromDatabase = { address: defaultWallet2.address, affiliateEarnings: '2000', referredMakerTrades: 3, referredTakerTrades: 1, - totalReferredFees: '4000', + totalReferredMakerFees: '2000', + totalReferredTakerFees: '1000', + totalReferredMakerRebates: '-1000', totalReferredUsers: 2, - referredNetProtocolEarnings: '2000', firstReferralBlockHeight: '1', referredTotalVolume: '4', }; @@ -236,9 +240,10 @@ describe('Affiliate info store', () => { affiliateEarnings: '0', referredMakerTrades: 0, referredTakerTrades: 0, - totalReferredFees: '0', + totalReferredMakerFees: '0', + totalReferredTakerFees: '0', + totalReferredMakerRebates: '0', totalReferredUsers: 1, - referredNetProtocolEarnings: '0', firstReferralBlockHeight: '2', referredTotalVolume: '0', }; @@ -391,7 +396,7 @@ async function populateFillsAndReferrals(): Promise { eventId: defaultTendermintEventId2, price: '1', size: '1', - fee: '1000', + fee: '-1000', affiliateRevShare: '500', }), FillTable.create({ diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20241002144813_change_affiliate_info_fee_columns.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20241002144813_change_affiliate_info_fee_columns.ts new file mode 100644 index 0000000000..877b2524c0 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20241002144813_change_affiliate_info_fee_columns.ts @@ -0,0 +1,25 @@ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + return knex + .schema + .alterTable('affiliate_info', (table) => { + table.dropColumn('totalReferredFees'); + table.dropColumn('referredNetProtocolEarnings'); + table.decimal('totalReferredTakerFees', null).notNullable().defaultTo(0); + table.decimal('totalReferredMakerFees', null).notNullable().defaultTo(0); + table.decimal('totalReferredMakerRebates', null).notNullable().defaultTo(0); + }); +} + +export async function down(knex: Knex): Promise { + return knex + .schema + .alterTable('affiliate_info', (table) => { + table.decimal('totalReferredFees', null).notNullable().defaultTo(0); + table.decimal('referredNetProtocolEarnings', null).notNullable().defaultTo(0); + table.dropColumn('totalReferredTakerFees'); + table.dropColumn('totalReferredMakerFees'); + table.dropColumn('totalReferredMakerRebates'); + }); +} diff --git a/indexer/packages/postgres/src/models/affiliate-info-model.ts b/indexer/packages/postgres/src/models/affiliate-info-model.ts index 7fcbefac39..49dd0dfdbd 100644 --- a/indexer/packages/postgres/src/models/affiliate-info-model.ts +++ b/indexer/packages/postgres/src/models/affiliate-info-model.ts @@ -1,4 +1,4 @@ -import { NonNegativeNumericPattern } from '../lib/validators'; +import { NonNegativeNumericPattern, NumericPattern } from '../lib/validators'; import UpsertQueryBuilder from '../query-builders/upsert'; import BaseModel from './base-model'; @@ -19,9 +19,10 @@ export default class AffiliateInfoModel extends BaseModel { 'affiliateEarnings', 'referredMakerTrades', 'referredTakerTrades', - 'totalReferredFees', + 'totalReferredMakerFees', + 'totalReferredTakerFees', + 'totalReferredMakerRebates', 'totalReferredUsers', - 'referredNetProtocolEarnings', 'firstReferralBlockHeight', 'referredTotalVolume', ], @@ -30,9 +31,10 @@ export default class AffiliateInfoModel extends BaseModel { affiliateEarnings: { type: 'string', pattern: NonNegativeNumericPattern }, referredMakerTrades: { type: 'int' }, referredTakerTrades: { type: 'int' }, - totalReferredFees: { type: 'string', pattern: NonNegativeNumericPattern }, + totalReferredMakerFees: { type: 'string', pattern: NonNegativeNumericPattern }, + totalReferredTakerFees: { type: 'string', pattern: NonNegativeNumericPattern }, + totalReferredMakerRebates: { type: 'string', pattern: NumericPattern }, totalReferredUsers: { type: 'int' }, - referredNetProtocolEarnings: { type: 'string', pattern: NonNegativeNumericPattern }, firstReferralBlockHeight: { type: 'string', pattern: NonNegativeNumericPattern }, referredTotalVolume: { type: 'string', pattern: NonNegativeNumericPattern }, }, @@ -51,9 +53,10 @@ export default class AffiliateInfoModel extends BaseModel { affiliateEarnings: 'string', referredMakerTrades: 'int', referredTakerTrades: 'int', - totalReferredFees: 'string', + totalReferredMakerFees: 'string', + totalReferredTakerFees: 'string', + totalReferredMakerRebates: 'string', totalReferredUsers: 'int', - referredNetProtocolEarnings: 'string', firstReferralBlockHeight: 'string', referredTotalVolume: 'string', }; @@ -69,11 +72,13 @@ export default class AffiliateInfoModel extends BaseModel { referredTakerTrades!: number; - totalReferredFees!: string; + totalReferredMakerFees!: string; - totalReferredUsers!: number; + totalReferredTakerFees!: string; + + totalReferredMakerRebates!: string; - referredNetProtocolEarnings!: string; + totalReferredUsers!: number; firstReferralBlockHeight!: string; diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 6c2ab2adc4..d661a01d70 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -192,7 +192,9 @@ affiliate_stats AS ( affiliate_fills."affiliateAddress", SUM(affiliate_fills."fee") AS "totalReferredFees", SUM(affiliate_fills."affiliateRevShare") AS "affiliateEarnings", - SUM(affiliate_fills."fee") - SUM(affiliate_fills."affiliateRevShare") AS "referredNetProtocolEarnings", + SUM(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.MAKER}' AND affiliate_fills."fee" > 0 THEN affiliate_fills."fee" ELSE 0 END) AS "totalReferredMakerFees", + SUM(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.TAKER}' THEN affiliate_fills."fee" ELSE 0 END) AS "totalReferredTakerFees", + SUM(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.MAKER}' AND affiliate_fills."fee" < 0 THEN affiliate_fills."fee" ELSE 0 END) AS "totalReferredMakerRebates", COUNT(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.MAKER}' THEN 1 END) AS "referredMakerTrades", COUNT(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.TAKER}' THEN 1 END) AS "referredTakerTrades", SUM(affiliate_fills."price" * affiliate_fills."size") AS "referredTotalVolume" @@ -210,9 +212,10 @@ affiliate_info_update AS ( affiliate_metadata."affiliateAddress", affiliate_metadata."totalReferredUsers", affiliate_metadata."firstReferralBlockHeight", - COALESCE(affiliate_stats."totalReferredFees", 0) AS "totalReferredFees", + COALESCE(affiliate_stats."totalReferredMakerFees", 0) AS "totalReferredMakerFees", + COALESCE(affiliate_stats."totalReferredTakerFees", 0) AS "totalReferredTakerFees", + COALESCE(affiliate_stats."totalReferredMakerRebates", 0) AS "totalReferredMakerRebates", COALESCE(affiliate_stats."affiliateEarnings", 0) AS "affiliateEarnings", - COALESCE(affiliate_stats."referredNetProtocolEarnings", 0) AS "referredNetProtocolEarnings", COALESCE(affiliate_stats."referredMakerTrades", 0) AS "referredMakerTrades", COALESCE(affiliate_stats."referredTakerTrades", 0) AS "referredTakerTrades", COALESCE(affiliate_stats."referredTotalVolume", 0) AS "referredTotalVolume" @@ -231,8 +234,9 @@ INSERT INTO affiliate_info ( "affiliateEarnings", "referredMakerTrades", "referredTakerTrades", - "totalReferredFees", - "referredNetProtocolEarnings", + "totalReferredMakerFees", + "totalReferredTakerFees", + "totalReferredMakerRebates", "referredTotalVolume" ) SELECT @@ -242,8 +246,9 @@ SELECT "affiliateEarnings", "referredMakerTrades", "referredTakerTrades", - "totalReferredFees", - "referredNetProtocolEarnings", + "totalReferredMakerFees", + "totalReferredTakerFees", + "totalReferredMakerRebates", "referredTotalVolume" FROM affiliate_info_update @@ -254,8 +259,9 @@ DO UPDATE SET "affiliateEarnings" = affiliate_info."affiliateEarnings" + EXCLUDED."affiliateEarnings", "referredMakerTrades" = affiliate_info."referredMakerTrades" + EXCLUDED."referredMakerTrades", "referredTakerTrades" = affiliate_info."referredTakerTrades" + EXCLUDED."referredTakerTrades", - "totalReferredFees" = affiliate_info."totalReferredFees" + EXCLUDED."totalReferredFees", - "referredNetProtocolEarnings" = affiliate_info."referredNetProtocolEarnings" + EXCLUDED."referredNetProtocolEarnings", + "totalReferredMakerFees" = affiliate_info."totalReferredMakerFees" + EXCLUDED."totalReferredMakerFees", + "totalReferredTakerFees" = affiliate_info."totalReferredTakerFees" + EXCLUDED."totalReferredTakerFees", + "totalReferredMakerRebates" = affiliate_info."totalReferredMakerRebates" + EXCLUDED."totalReferredMakerRebates", "referredTotalVolume" = affiliate_info."referredTotalVolume" + EXCLUDED."referredTotalVolume"; `; diff --git a/indexer/packages/postgres/src/types/affiliate-info-types.ts b/indexer/packages/postgres/src/types/affiliate-info-types.ts index 885de8b9b7..a1dcc61be6 100644 --- a/indexer/packages/postgres/src/types/affiliate-info-types.ts +++ b/indexer/packages/postgres/src/types/affiliate-info-types.ts @@ -3,9 +3,10 @@ export interface AffiliateInfoCreateObject { affiliateEarnings: string, referredMakerTrades: number, referredTakerTrades: number, - totalReferredFees: string, + totalReferredMakerFees: string, + totalReferredTakerFees: string, + totalReferredMakerRebates: string, totalReferredUsers: number, - referredNetProtocolEarnings: string, firstReferralBlockHeight: string, referredTotalVolume: string, } @@ -15,9 +16,10 @@ export enum AffiliateInfoColumns { affiliateEarnings = 'affiliateEarnings', referredMakerTrades = 'referredMakerTrades', referredTakerTrades = 'referredTakerTrades', - totalReferredFees = 'totalReferredFees', + totalReferredMakerFees = 'totalReferredMakerFees', + totalReferredTakerFees = 'totalReferredTakerFees', + totalReferredMakerRebates = 'totalReferredMakerRebates', totalReferredUsers = 'totalReferredUsers', - referredNetProtocolEarnings = 'referredNetProtocolEarnings', firstReferralBlockHeight = 'firstReferralBlockHeight', referredTotalVolume = 'referredTotalVolume', } diff --git a/indexer/packages/postgres/src/types/db-model-types.ts b/indexer/packages/postgres/src/types/db-model-types.ts index b400153118..9557be8ede 100644 --- a/indexer/packages/postgres/src/types/db-model-types.ts +++ b/indexer/packages/postgres/src/types/db-model-types.ts @@ -288,9 +288,10 @@ export interface AffiliateInfoFromDatabase { affiliateEarnings: string, referredMakerTrades: number, referredTakerTrades: number, - totalReferredFees: string, + totalReferredMakerFees: string, + totalReferredTakerFees: string, + totalReferredMakerRebates: string, totalReferredUsers: number, - referredNetProtocolEarnings: string, firstReferralBlockHeight: string, referredTotalVolume: string, } diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index ea89899e15..30ceebd062 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -369,9 +369,16 @@ function affiliateInfoCreateToResponseObject( affiliateEarnings: Number(info.affiliateEarnings), affiliateReferredTrades: Number(info.referredTakerTrades) + Number(info.referredMakerTrades), - affiliateTotalReferredFees: Number(info.totalReferredFees), + affiliateTotalReferredFees: Number(info.totalReferredMakerFees) + + Number(info.totalReferredTakerFees) + + Number(info.totalReferredMakerRebates), affiliateReferredUsers: Number(info.totalReferredUsers), - affiliateReferredNetProtocolEarnings: Number(info.referredNetProtocolEarnings), + affiliateReferredNetProtocolEarnings: Number(info.totalReferredMakerFees) + + Number(info.totalReferredTakerFees) + + Number(info.totalReferredMakerRebates) - + Number(info.affiliateEarnings), affiliateReferredTotalVolume: Number(info.referredTotalVolume), + affiliateReferredMakerFees: Number(info.totalReferredMakerFees), + affiliateReferredTakerFees: Number(info.totalReferredTakerFees), }; } diff --git a/indexer/services/comlink/public/api-documentation.md b/indexer/services/comlink/public/api-documentation.md index 4d08bf1430..eaec0a2fbe 100644 --- a/indexer/services/comlink/public/api-documentation.md +++ b/indexer/services/comlink/public/api-documentation.md @@ -653,7 +653,9 @@ fetch(`${baseURL}/affiliates/snapshot`, "affiliateTotalReferredFees": 0.1, "affiliateReferredUsers": 0.1, "affiliateReferredNetProtocolEarnings": 0.1, - "affiliateReferredTotalVolume": 0.1 + "affiliateReferredTotalVolume": 0.1, + "affiliateReferredMakerFees": 0.1, + "affiliateReferredTakerFees": 0.1 } ], "total": 0.1, @@ -4244,7 +4246,9 @@ This operation does not require authentication "affiliateTotalReferredFees": 0.1, "affiliateReferredUsers": 0.1, "affiliateReferredNetProtocolEarnings": 0.1, - "affiliateReferredTotalVolume": 0.1 + "affiliateReferredTotalVolume": 0.1, + "affiliateReferredMakerFees": 0.1, + "affiliateReferredTakerFees": 0.1 } ``` @@ -4261,6 +4265,8 @@ This operation does not require authentication |affiliateReferredUsers|number(double)|true|none|none| |affiliateReferredNetProtocolEarnings|number(double)|true|none|none| |affiliateReferredTotalVolume|number(double)|true|none|none| +|affiliateReferredMakerFees|number(double)|true|none|none| +|affiliateReferredTakerFees|number(double)|true|none|none| ## AffiliateSnapshotResponse @@ -4280,7 +4286,9 @@ This operation does not require authentication "affiliateTotalReferredFees": 0.1, "affiliateReferredUsers": 0.1, "affiliateReferredNetProtocolEarnings": 0.1, - "affiliateReferredTotalVolume": 0.1 + "affiliateReferredTotalVolume": 0.1, + "affiliateReferredMakerFees": 0.1, + "affiliateReferredTakerFees": 0.1 } ], "total": 0.1, diff --git a/indexer/services/comlink/public/swagger.json b/indexer/services/comlink/public/swagger.json index 23de12c60c..e94a095d65 100644 --- a/indexer/services/comlink/public/swagger.json +++ b/indexer/services/comlink/public/swagger.json @@ -303,6 +303,14 @@ "affiliateReferredTotalVolume": { "type": "number", "format": "double" + }, + "affiliateReferredMakerFees": { + "type": "number", + "format": "double" + }, + "affiliateReferredTakerFees": { + "type": "number", + "format": "double" } }, "required": [ @@ -313,7 +321,9 @@ "affiliateTotalReferredFees", "affiliateReferredUsers", "affiliateReferredNetProtocolEarnings", - "affiliateReferredTotalVolume" + "affiliateReferredTotalVolume", + "affiliateReferredMakerFees", + "affiliateReferredTakerFees" ], "type": "object", "additionalProperties": false diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index 76e2433255..ac246bbfdf 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -164,10 +164,17 @@ class AffiliatesController extends Controller { info.address in addressUsernameMap ? addressUsernameMap[info.address] : '', affiliateEarnings: Number(info.affiliateEarnings), affiliateReferredTrades: Number(info.referredMakerTrades) + Number(info.referredTakerTrades), - affiliateTotalReferredFees: Number(info.totalReferredFees), + affiliateTotalReferredFees: Number(info.totalReferredMakerFees) + + Number(info.totalReferredTakerFees) + + Number(info.totalReferredMakerRebates), affiliateReferredUsers: Number(info.totalReferredUsers), - affiliateReferredNetProtocolEarnings: Number(info.referredNetProtocolEarnings), + affiliateReferredNetProtocolEarnings: Number(info.totalReferredMakerFees) + + Number(info.totalReferredTakerFees) + + Number(info.totalReferredMakerRebates) - + Number(info.affiliateEarnings), affiliateReferredTotalVolume: Number(info.referredTotalVolume), + affiliateReferredMakerFees: Number(info.totalReferredMakerFees), + affiliateReferredTakerFees: Number(info.totalReferredTakerFees), })); const response: AffiliateSnapshotResponse = { diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index c777521f48..81eace3303 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -731,6 +731,8 @@ export interface AffiliateSnapshotResponseObject { affiliateReferredUsers: number, affiliateReferredNetProtocolEarnings: number, affiliateReferredTotalVolume: number, + affiliateReferredMakerFees: number, + affiliateReferredTakerFees: number, } export interface AffiliateTotalVolumeResponse { diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index adc63c68e3..26b139cdb8 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -72,9 +72,10 @@ describe('update-affiliate-info', () => { affiliateEarnings: '0', referredMakerTrades: 0, referredTakerTrades: 0, - totalReferredFees: '0', + totalReferredMakerFees: '0', + totalReferredTakerFees: '0', + totalReferredMakerRebates: '0', totalReferredUsers: 1, - referredNetProtocolEarnings: '0', firstReferralBlockHeight: '1', referredTotalVolume: '0', }; @@ -121,9 +122,10 @@ describe('update-affiliate-info', () => { affiliateEarnings: '500', referredMakerTrades: 0, referredTakerTrades: 1, - totalReferredFees: '1000', + totalReferredMakerFees: '0', + totalReferredTakerFees: '1000', + totalReferredMakerRebates: '0', totalReferredUsers: 2, - referredNetProtocolEarnings: '500', firstReferralBlockHeight: '1', referredTotalVolume: '1', }; @@ -189,9 +191,10 @@ describe('update-affiliate-info', () => { affiliateEarnings: '1000', referredMakerTrades: 0, referredTakerTrades: 2, - totalReferredFees: '2000', + totalReferredMakerFees: '0', + totalReferredTakerFees: '2000', + totalReferredMakerRebates: '0', totalReferredUsers: 1, - referredNetProtocolEarnings: '1000', firstReferralBlockHeight: '1', referredTotalVolume: '2', }; @@ -255,9 +258,10 @@ describe('update-affiliate-info', () => { affiliateEarnings: '1000', referredMakerTrades: 0, referredTakerTrades: 2, - totalReferredFees: '2000', + totalReferredMakerFees: '0', + totalReferredTakerFees: '2000', + totalReferredMakerRebates: '0', totalReferredUsers: 1, - referredNetProtocolEarnings: '1000', firstReferralBlockHeight: '1', referredTotalVolume: '2', }; From c182cd94e4577551c79a5bf95b0f79be7fbecbba Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:06:31 -0400 Subject: [PATCH 17/41] Fix bug with PnL aggregation. (backport #2446) (#2451) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../comlink/__tests__/lib/helpers.test.ts | 77 ++++++++++++------- .../api/v4/historical-pnl-controller.ts | 6 +- .../controllers/api/v4/vault-controller.ts | 42 ++++------ indexer/services/comlink/src/lib/helpers.ts | 39 ++++++---- 4 files changed, 90 insertions(+), 74 deletions(-) diff --git a/indexer/services/comlink/__tests__/lib/helpers.test.ts b/indexer/services/comlink/__tests__/lib/helpers.test.ts index 8c9e27bebf..d814827235 100644 --- a/indexer/services/comlink/__tests__/lib/helpers.test.ts +++ b/indexer/services/comlink/__tests__/lib/helpers.test.ts @@ -44,7 +44,7 @@ import { getPerpetualPositionsWithUpdatedFunding, initializePerpetualPositionsWithFunding, getChildSubaccountNums, - aggregatePnlTicks, + aggregateHourlyPnlTicks, getSubaccountResponse, } from '../../src/lib/helpers'; import _ from 'lodash'; @@ -60,6 +60,7 @@ import { } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; import { AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject } from '../../src/types'; import { ZERO, ZERO_USDC_POSITION } from '../../src/lib/constants'; +import { DateTime } from 'luxon'; describe('helpers', () => { afterEach(async () => { @@ -833,7 +834,7 @@ describe('helpers', () => { }); }); - describe('aggregatePnlTicks', () => { + describe('aggregateHourlyPnlTicks', () => { it('aggregates single pnl tick', () => { const pnlTick: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, @@ -843,10 +844,12 @@ describe('helpers', () => { ), }; - const aggregatedPnlTicks: Map = aggregatePnlTicks([pnlTick]); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks([pnlTick]); expect( - aggregatedPnlTicks.get(parseInt(pnlTick.blockHeight, 10)), - ).toEqual(expect.objectContaining({ ...testConstants.defaultPnlTick })); + aggregatedPnlTicks, + ).toEqual( + [expect.objectContaining({ ...testConstants.defaultPnlTick })], + ); }); it('aggregates multiple pnl ticks same height', () => { @@ -865,38 +868,58 @@ describe('helpers', () => { ), }; const blockHeight2: string = '80'; + const blockTime2: string = DateTime.fromISO(pnlTick.createdAt).plus({ hour: 1 }).toISO(); const pnlTick3: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, id: PnlTicksTable.uuid( testConstants.defaultPnlTick.subaccountId, - testConstants.defaultPnlTick.createdAt, + blockTime2, ), blockHeight: blockHeight2, + blockTime: blockTime2, + createdAt: blockTime2, + }; + const blockHeight3: string = '81'; + const blockTime3: string = DateTime.fromISO(pnlTick.createdAt).plus({ minute: 61 }).toISO(); + const pnlTick4: PnlTicksFromDatabase = { + ...testConstants.defaultPnlTick, + id: PnlTicksTable.uuid( + testConstants.defaultPnlTick.subaccountId, + blockTime3, + ), + equity: '1', + totalPnl: '2', + netTransfers: '3', + blockHeight: blockHeight3, + blockTime: blockTime3, + createdAt: blockTime3, }; - const aggregatedPnlTicks: Map = aggregatePnlTicks( - [pnlTick, pnlTick2, pnlTick3], + const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks( + [pnlTick, pnlTick2, pnlTick3, pnlTick4], ); - // Combined pnl tick at initial block height. - expect( - aggregatedPnlTicks.get(parseInt(pnlTick.blockHeight, 10)), - ).toEqual(expect.objectContaining({ - equity: (parseFloat(testConstants.defaultPnlTick.equity) + + expect(aggregatedPnlTicks).toEqual( + expect.arrayContaining([ + // Combined pnl tick at initial hour + expect.objectContaining({ + equity: (parseFloat(testConstants.defaultPnlTick.equity) + parseFloat(pnlTick2.equity)).toString(), - totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) + - parseFloat(pnlTick2.totalPnl)).toString(), - netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) + - parseFloat(pnlTick2.netTransfers)).toString(), - createdAt: testConstants.defaultPnlTick.createdAt, - blockHeight: testConstants.defaultPnlTick.blockHeight, - blockTime: testConstants.defaultPnlTick.blockTime, - })); - // Single pnl tick at second block height. - expect( - aggregatedPnlTicks.get(parseInt(blockHeight2, 10)), - ).toEqual(expect.objectContaining({ - ...pnlTick3, - })); + totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) + + parseFloat(pnlTick2.totalPnl)).toString(), + netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) + + parseFloat(pnlTick2.netTransfers)).toString(), + }), + // Combined pnl tick at initial hour + 1 hour and initial hour + 1 hour, 1 minute + expect.objectContaining({ + equity: (parseFloat(pnlTick3.equity) + + parseFloat(pnlTick4.equity)).toString(), + totalPnl: (parseFloat(pnlTick3.totalPnl) + + parseFloat(pnlTick4.totalPnl)).toString(), + netTransfers: (parseFloat(pnlTick3.netTransfers) + + parseFloat(pnlTick4.netTransfers)).toString(), + }), + ]), + ); }); }); }); diff --git a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts index 6ea7a5ce58..96a3131ea9 100644 --- a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts @@ -19,7 +19,7 @@ import { getReqRateLimiter } from '../../../caches/rate-limiters'; import config from '../../../config'; import { complianceAndGeoCheck } from '../../../lib/compliance-and-geo-check'; import { NotFoundError } from '../../../lib/errors'; -import { aggregatePnlTicks, getChildSubaccountIds, handleControllerError } from '../../../lib/helpers'; +import { aggregateHourlyPnlTicks, getChildSubaccountIds, handleControllerError } from '../../../lib/helpers'; import { rateLimiterMiddleware } from '../../../lib/rate-limit'; import { CheckLimitAndCreatedBeforeOrAtAndOnOrAfterSchema, @@ -156,10 +156,10 @@ class HistoricalPnlController extends Controller { } // aggregate pnlTicks for all subaccounts grouped by blockHeight - const aggregatedPnlTicks: Map = aggregatePnlTicks(pnlTicks); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(pnlTicks); return { - historicalPnl: Array.from(aggregatedPnlTicks.values()).map( + historicalPnl: aggregatedPnlTicks.map( (pnlTick: PnlTicksFromDatabase) => { return pnlTicksToResponseObject(pnlTick); }), diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index ee4a31fe64..81f710b315 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -38,7 +38,7 @@ import { import { getReqRateLimiter } from '../../../caches/rate-limiters'; import config from '../../../config'; import { - aggregatePnlTicks, + aggregateHourlyPnlTicks, getSubaccountResponse, handleControllerError, } from '../../../lib/helpers'; @@ -93,7 +93,7 @@ class VaultController extends Controller { ]); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight - const aggregatedPnlTicks: Map = aggregatePnlTicks(vaultPnlTicks); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(vaultPnlTicks); const currentEquity: string = Array.from(vaultPositions.values()) .map((position: VaultPosition): string => { @@ -473,46 +473,34 @@ function getPnlTicksWithCurrentTick( } /** - * Takes in a map of block heights to PnlTicks and filters out the closest pnl tick per interval. - * @param pnlTicksByBlock Map of block number to pnl tick. + * Takes in an array of PnlTicks and filters out the closest pnl tick per interval. + * @param pnlTicks Array of pnl ticks. * @param resolution Resolution of interval. * @returns Array of PnlTicksFromDatabase, one per interval. */ function filterOutIntervalTicks( - pnlTicksByBlock: Map, + pnlTicks: PnlTicksFromDatabase[], resolution: PnlTickInterval, ): PnlTicksFromDatabase[] { - // Track block to block time. - const blockToBlockTime: Map = new Map(); - // Track start of days to closest block by block time. - const blocksPerInterval: Map = new Map(); - // Track start of days to closest Pnl tick. + // Track start of intervals to closest Pnl tick. const ticksPerInterval: Map = new Map(); - pnlTicksByBlock.forEach((pnlTick: PnlTicksFromDatabase, block: number): void => { + pnlTicks.forEach((pnlTick: PnlTicksFromDatabase): void => { const blockTime: DateTime = DateTime.fromISO(pnlTick.blockTime).toUTC(); - blockToBlockTime.set(block, blockTime); const startOfInterval: DateTime = blockTime.toUTC().startOf(resolution); const startOfIntervalStr: string = startOfInterval.toISO(); - const startOfIntervalBlock: number | undefined = blocksPerInterval.get(startOfIntervalStr); - // No block for the start of interval, set this block as the block for the interval. - if (startOfIntervalBlock === undefined) { - blocksPerInterval.set(startOfIntervalStr, block); - ticksPerInterval.set(startOfIntervalStr, pnlTick); - return; - } - - const startOfDayBlockTime: DateTime | undefined = blockToBlockTime.get(startOfIntervalBlock); - // Invalid block set as start of day block, set this block as the block for the day. - if (startOfDayBlockTime === undefined) { - blocksPerInterval.set(startOfIntervalStr, block); + const tickForInterval: PnlTicksFromDatabase | undefined = ticksPerInterval.get( + startOfIntervalStr, + ); + // No tick for the start of interval, set this tick as the block for the interval. + if (tickForInterval === undefined) { ticksPerInterval.set(startOfIntervalStr, pnlTick); return; } + const tickPerIntervalBlockTime: DateTime = DateTime.fromISO(tickForInterval.blockTime); - // This block is closer to the start of the day, set it as the block for the day. - if (blockTime.diff(startOfInterval) < startOfDayBlockTime.diff(startOfInterval)) { - blocksPerInterval.set(startOfIntervalStr, block); + // This tick is closer to the start of the interval, set it as the tick for the interval. + if (blockTime.diff(startOfInterval) < tickPerIntervalBlockTime.diff(startOfInterval)) { ticksPerInterval.set(startOfIntervalStr, pnlTick); } }); diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index 6da4106907..b3f5d3bf19 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -28,6 +28,7 @@ import { import Big from 'big.js'; import express from 'express'; import _ from 'lodash'; +import { DateTime } from 'luxon'; import config from '../config'; import { @@ -672,32 +673,36 @@ export function getSubaccountResponse( /* ------- PNL HELPERS ------- */ /** - * Aggregates a list of PnL ticks, combining any PnL ticks for the same blockheight by summing + * Aggregates a list of PnL ticks, combining any PnL ticks for the same hour by summing * the equity, totalPnl, and net transfers. * Returns a map of block height to the resulting PnL tick. * @param pnlTicks * @returns */ -export function aggregatePnlTicks( +export function aggregateHourlyPnlTicks( pnlTicks: PnlTicksFromDatabase[], -): Map { - const aggregatedPnlTicks: Map = new Map(); +): PnlTicksFromDatabase[] { + const hourlyPnlTicks: Map = new Map(); for (const pnlTick of pnlTicks) { - const blockHeight: number = parseInt(pnlTick.blockHeight, 10); - if (aggregatedPnlTicks.has(blockHeight)) { - const currentPnlTick: PnlTicksFromDatabase = aggregatedPnlTicks.get( - blockHeight, + const truncatedTime: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').toISO(); + if (hourlyPnlTicks.has(truncatedTime)) { + const aggregatedTick: PnlTicksFromDatabase = hourlyPnlTicks.get( + truncatedTime, ) as PnlTicksFromDatabase; - aggregatedPnlTicks.set(blockHeight, { - ...currentPnlTick, - equity: (parseFloat(currentPnlTick.equity) + parseFloat(pnlTick.equity)).toString(), - totalPnl: (parseFloat(currentPnlTick.totalPnl) + parseFloat(pnlTick.totalPnl)).toString(), - netTransfers: (parseFloat(currentPnlTick.netTransfers) + - parseFloat(pnlTick.netTransfers)).toString(), - }); + hourlyPnlTicks.set( + truncatedTime, + { + ...aggregatedTick, + equity: (parseFloat(aggregatedTick.equity) + parseFloat(pnlTick.equity)).toString(), + totalPnl: (parseFloat(aggregatedTick.totalPnl) + parseFloat(pnlTick.totalPnl)).toString(), + netTransfers: ( + parseFloat(aggregatedTick.netTransfers) + parseFloat(pnlTick.netTransfers) + ).toString(), + }, + ); } else { - aggregatedPnlTicks.set(blockHeight, pnlTick); + hourlyPnlTicks.set(truncatedTime, pnlTick); } } - return aggregatedPnlTicks; + return Array.from(hourlyPnlTicks.values()); } From 939cb56830d9907c91f9b967788000160bd3c816 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 19:27:48 -0400 Subject: [PATCH 18/41] Get latest hourly tick to compute final tick for megavault PnL. (backport #2454) (#2466) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../api/v4/vault-controller.test.ts | 96 +++++++++++++------ indexer/services/comlink/src/config.ts | 1 + .../controllers/api/v4/vault-controller.ts | 31 ++++++ 3 files changed, 97 insertions(+), 31 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index b2071c6681..582cb25361 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -25,20 +25,23 @@ import config from '../../../../src/config'; describe('vault-controller#V4', () => { const latestBlockHeight: string = '25'; - const currentBlockHeight: string = '9'; + const currentHourBlockHeight: string = '10'; + const currentDayBlockHeight: string = '9'; const twoHourBlockHeight: string = '7'; const almostTwoDayBlockHeight: string = '5'; const twoDayBlockHeight: string = '3'; - const currentTime: DateTime = DateTime.utc().startOf('day').minus({ hour: 5 }); - const latestTime: DateTime = currentTime.plus({ second: 5 }); - const twoHoursAgo: DateTime = currentTime.minus({ hour: 2 }); - const twoDaysAgo: DateTime = currentTime.minus({ day: 2 }); - const almostTwoDaysAgo: DateTime = currentTime.minus({ hour: 47 }); + const currentDay: DateTime = DateTime.utc().startOf('day').minus({ hour: 5 }); + const currentHour: DateTime = currentDay.plus({ hour: 1 }); + const latestTime: DateTime = currentDay.plus({ minute: 90 }); + const twoHoursAgo: DateTime = currentDay.minus({ hour: 2 }); + const twoDaysAgo: DateTime = currentDay.minus({ day: 2 }); + const almostTwoDaysAgo: DateTime = currentDay.minus({ hour: 47 }); const initialFundingIndex: string = '10000'; const vault1Equity: number = 159500; const vault2Equity: number = 10000; const mainVaultEquity: number = 10000; const vaultPnlHistoryHoursPrev: number = config.VAULT_PNL_HISTORY_HOURS; + const vaultPnlLastPnlWindowPrev: number = config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS; beforeAll(async () => { await dbHelpers.migrate(); @@ -52,6 +55,8 @@ describe('vault-controller#V4', () => { beforeEach(async () => { // Get a week of data for hourly pnl ticks. config.VAULT_PNL_HISTORY_HOURS = 168; + // Use last 48 hours to get latest pnl tick for tests. + config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = 48; await testMocks.seedData(); await perpetualMarketRefresher.updatePerpetualMarkets(); await liquidityTierRefresher.updateLiquidityTiers(); @@ -68,8 +73,8 @@ describe('vault-controller#V4', () => { }), BlockTable.create({ ...testConstants.defaultBlock, - time: currentTime.toISO(), - blockHeight: currentBlockHeight, + time: currentDay.toISO(), + blockHeight: currentDayBlockHeight, }), BlockTable.create({ ...testConstants.defaultBlock, @@ -81,6 +86,11 @@ describe('vault-controller#V4', () => { time: almostTwoDaysAgo.toISO(), blockHeight: almostTwoDayBlockHeight, }), + BlockTable.create({ + ...testConstants.defaultBlock, + time: currentHour.toISO(), + blockHeight: currentHourBlockHeight, + }), ]); await SubaccountTable.create(testConstants.vaultSubaccount); await SubaccountTable.create({ @@ -114,6 +124,7 @@ describe('vault-controller#V4', () => { afterEach(async () => { await dbHelpers.clearData(); config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev; + config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = vaultPnlLastPnlWindowPrev; }); it('Get /megavault/historicalPnl with no vault subaccounts', async () => { @@ -126,13 +137,14 @@ describe('vault-controller#V4', () => { }); it.each([ - ['no resolution', '', [1, 2]], - ['daily resolution', '?resolution=day', [1, 2]], - ['hourly resolution', '?resolution=hour', [1, 2, 3]], + ['no resolution', '', [1, 2], 4], + ['daily resolution', '?resolution=day', [1, 2], 4], + ['hourly resolution', '?resolution=hour', [1, 2, 3, 4], 4], ])('Get /megavault/historicalPnl with single vault subaccount (%s)', async ( _name: string, queryParam: string, expectedTicksIndex: number[], + finalTickIndex: number, ) => { await VaultTable.create({ ...testConstants.defaultVault, @@ -141,7 +153,7 @@ describe('vault-controller#V4', () => { }); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks(); const finalTick: PnlTicksFromDatabase = { - ...createdPnlTicks[expectedTicksIndex[expectedTicksIndex.length - 1]], + ...createdPnlTicks[finalTickIndex], equity: Big(vault1Equity).toFixed(), blockHeight: latestBlockHeight, blockTime: latestTime.toISO(), @@ -164,14 +176,14 @@ describe('vault-controller#V4', () => { }); it.each([ - ['no resolution', '', [1, 2], [undefined, 6], [9, 10]], - ['daily resolution', '?resolution=day', [1, 2], [undefined, 6], [9, 10]], + ['no resolution', '', [1, 2], [undefined, 7], [11, 12]], + ['daily resolution', '?resolution=day', [1, 2], [undefined, 7], [11, 12]], [ 'hourly resolution', '?resolution=hour', - [1, undefined, 2, 3], - [undefined, 5, 6, 7], - [9, undefined, 10, 11], + [1, undefined, 2, 3, 4], + [undefined, 6, 7, 8, 9], + [11, undefined, 12, 13, 14], ], ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s)', async ( _name: string, @@ -212,7 +224,8 @@ describe('vault-controller#V4', () => { const expectedPnlTickBase: any = { equity: (parseFloat(testConstants.defaultPnlTick.equity) * 3).toString(), - totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) * 3).toString(), + // total pnl should be fetched from latest hourly pnl tick. + totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) * 4).toString(), netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) * 3).toString(), }; const finalTick: PnlTicksFromDatabase = { @@ -268,7 +281,7 @@ describe('vault-controller#V4', () => { it.each([ ['no resolution', '', [1, 2]], ['daily resolution', '?resolution=day', [1, 2]], - ['hourly resolution', '?resolution=hour', [1, 2, 3]], + ['hourly resolution', '?resolution=hour', [1, 2, 3, 4]], ])('Get /vaults/historicalPnl with single vault subaccount (%s)', async ( _name: string, queryParam: string, @@ -306,9 +319,9 @@ describe('vault-controller#V4', () => { }); it.each([ - ['no resolution', '', [1, 2], [5, 6]], - ['daily resolution', '?resolution=day', [1, 2], [5, 6]], - ['hourly resolution', '?resolution=hour', [1, 2, 3], [5, 6, 7]], + ['no resolution', '', [1, 2], [6, 7]], + ['daily resolution', '?resolution=day', [1, 2], [6, 7]], + ['hourly resolution', '?resolution=hour', [1, 2, 3, 4], [6, 7, 8, 9]], ])('Get /vaults/historicalPnl with 2 vault subaccounts (%s)', async ( _name: string, queryParam: string, @@ -526,9 +539,16 @@ describe('vault-controller#V4', () => { }), PnlTicksTable.create({ ...testConstants.defaultPnlTick, - blockTime: currentTime.toISO(), - createdAt: currentTime.toISO(), - blockHeight: currentBlockHeight, + blockTime: currentDay.toISO(), + createdAt: currentDay.toISO(), + blockHeight: currentDayBlockHeight, + }), + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + totalPnl: (2 * parseFloat(testConstants.defaultPnlTick.totalPnl)).toString(), + blockTime: currentHour.toISO(), + createdAt: currentHour.toISO(), + blockHeight: currentHourBlockHeight, }), PnlTicksTable.create({ ...testConstants.defaultPnlTick, @@ -551,9 +571,16 @@ describe('vault-controller#V4', () => { PnlTicksTable.create({ ...testConstants.defaultPnlTick, subaccountId: testConstants.vaultSubaccountId, - blockTime: currentTime.toISO(), - createdAt: currentTime.toISO(), - blockHeight: currentBlockHeight, + blockTime: currentDay.toISO(), + createdAt: currentDay.toISO(), + blockHeight: currentDayBlockHeight, + }), + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + subaccountId: testConstants.vaultSubaccountId, + blockTime: currentHour.toISO(), + createdAt: currentHour.toISO(), + blockHeight: currentHourBlockHeight, }), ]); @@ -580,9 +607,16 @@ describe('vault-controller#V4', () => { PnlTicksTable.create({ ...testConstants.defaultPnlTick, subaccountId: MEGAVAULT_SUBACCOUNT_ID, - blockTime: currentTime.toISO(), - createdAt: currentTime.toISO(), - blockHeight: currentBlockHeight, + blockTime: currentDay.toISO(), + createdAt: currentDay.toISO(), + blockHeight: currentDayBlockHeight, + }), + PnlTicksTable.create({ + ...testConstants.defaultPnlTick, + subaccountId: MEGAVAULT_SUBACCOUNT_ID, + blockTime: currentHour.toISO(), + createdAt: currentHour.toISO(), + blockHeight: currentHourBlockHeight, }), ]); createdTicks.push(...mainSubaccountTicks); diff --git a/indexer/services/comlink/src/config.ts b/indexer/services/comlink/src/config.ts index 1b70d73b4b..743bae1dd4 100644 --- a/indexer/services/comlink/src/config.ts +++ b/indexer/services/comlink/src/config.ts @@ -62,6 +62,7 @@ export const configSchema = { // Vaults config VAULT_PNL_HISTORY_DAYS: parseInteger({ default: 90 }), VAULT_PNL_HISTORY_HOURS: parseInteger({ default: 72 }), + VAULT_LATEST_PNL_TICK_WINDOW_HOURS: parseInteger({ default: 1 }), }; //////////////////////////////////////////////////////////////////////////////// diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 81f710b315..d5e8ff5c42 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -80,16 +80,19 @@ class VaultController extends Controller { vaultPositions, latestBlock, mainSubaccountEquity, + latestPnlTick, ] : [ PnlTicksFromDatabase[], Map, BlockFromDatabase, string, + PnlTicksFromDatabase | undefined, ] = await Promise.all([ getVaultSubaccountPnlTicks(vaultSubaccountIdsWithMainSubaccount, getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), + getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount), ]); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight @@ -105,6 +108,7 @@ class VaultController extends Controller { currentEquity, filterOutIntervalTicks(aggregatedPnlTicks, getResolution(resolution)), latestBlock, + latestPnlTick, ); return { @@ -458,7 +462,17 @@ function getPnlTicksWithCurrentTick( equity: string, pnlTicks: PnlTicksFromDatabase[], latestBlock: BlockFromDatabase, + latestTick: PnlTicksFromDatabase | undefined = undefined, ): PnlTicksFromDatabase[] { + if (latestTick !== undefined) { + return pnlTicks.concat({ + ...latestTick, + equity, + blockHeight: latestBlock.blockHeight, + blockTime: latestBlock.time, + createdAt: latestBlock.time, + }); + } if (pnlTicks.length === 0) { return []; } @@ -472,6 +486,23 @@ function getPnlTicksWithCurrentTick( return pnlTicks.concat([currentTick]); } +export async function getLatestPnlTick( + vaultSubaccountIds: string[], +): Promise { + const pnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.getPnlTicksAtIntervals( + PnlTickInterval.hour, + config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60, + vaultSubaccountIds, + ); + // Aggregate and get pnl tick closest to the hour + const aggregatedTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(pnlTicks); + const filteredTicks: PnlTicksFromDatabase[] = filterOutIntervalTicks( + aggregatedTicks, + PnlTickInterval.hour, + ); + return _.maxBy(filteredTicks, 'blockTime'); +} + /** * Takes in an array of PnlTicks and filters out the closest pnl tick per interval. * @param pnlTicks Array of pnl ticks. From 6800a368c2ce72948be59484b701fc221ebff8d3 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 15:02:13 -0400 Subject: [PATCH 19/41] add afflaiteReferredMakerRebates field to response (backport #2473) (#2474) Co-authored-by: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> --- .../controllers/api/v4/affiliates-controller.test.ts | 1 + indexer/services/comlink/public/api-documentation.md | 10 +++++++--- indexer/services/comlink/public/swagger.json | 7 ++++++- .../src/controllers/api/v4/affiliates-controller.ts | 1 + indexer/services/comlink/src/types.ts | 1 + 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index 30ceebd062..ee77111d30 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -380,5 +380,6 @@ function affiliateInfoCreateToResponseObject( affiliateReferredTotalVolume: Number(info.referredTotalVolume), affiliateReferredMakerFees: Number(info.totalReferredMakerFees), affiliateReferredTakerFees: Number(info.totalReferredTakerFees), + affiliateReferredMakerRebates: Number(info.totalReferredMakerRebates), }; } diff --git a/indexer/services/comlink/public/api-documentation.md b/indexer/services/comlink/public/api-documentation.md index eaec0a2fbe..a7ac9f0000 100644 --- a/indexer/services/comlink/public/api-documentation.md +++ b/indexer/services/comlink/public/api-documentation.md @@ -655,7 +655,8 @@ fetch(`${baseURL}/affiliates/snapshot`, "affiliateReferredNetProtocolEarnings": 0.1, "affiliateReferredTotalVolume": 0.1, "affiliateReferredMakerFees": 0.1, - "affiliateReferredTakerFees": 0.1 + "affiliateReferredTakerFees": 0.1, + "affiliateReferredMakerRebates": 0.1 } ], "total": 0.1, @@ -4248,7 +4249,8 @@ This operation does not require authentication "affiliateReferredNetProtocolEarnings": 0.1, "affiliateReferredTotalVolume": 0.1, "affiliateReferredMakerFees": 0.1, - "affiliateReferredTakerFees": 0.1 + "affiliateReferredTakerFees": 0.1, + "affiliateReferredMakerRebates": 0.1 } ``` @@ -4267,6 +4269,7 @@ This operation does not require authentication |affiliateReferredTotalVolume|number(double)|true|none|none| |affiliateReferredMakerFees|number(double)|true|none|none| |affiliateReferredTakerFees|number(double)|true|none|none| +|affiliateReferredMakerRebates|number(double)|true|none|none| ## AffiliateSnapshotResponse @@ -4288,7 +4291,8 @@ This operation does not require authentication "affiliateReferredNetProtocolEarnings": 0.1, "affiliateReferredTotalVolume": 0.1, "affiliateReferredMakerFees": 0.1, - "affiliateReferredTakerFees": 0.1 + "affiliateReferredTakerFees": 0.1, + "affiliateReferredMakerRebates": 0.1 } ], "total": 0.1, diff --git a/indexer/services/comlink/public/swagger.json b/indexer/services/comlink/public/swagger.json index e94a095d65..0737485b43 100644 --- a/indexer/services/comlink/public/swagger.json +++ b/indexer/services/comlink/public/swagger.json @@ -311,6 +311,10 @@ "affiliateReferredTakerFees": { "type": "number", "format": "double" + }, + "affiliateReferredMakerRebates": { + "type": "number", + "format": "double" } }, "required": [ @@ -323,7 +327,8 @@ "affiliateReferredNetProtocolEarnings", "affiliateReferredTotalVolume", "affiliateReferredMakerFees", - "affiliateReferredTakerFees" + "affiliateReferredTakerFees", + "affiliateReferredMakerRebates" ], "type": "object", "additionalProperties": false diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index ac246bbfdf..c87ba90c09 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -175,6 +175,7 @@ class AffiliatesController extends Controller { affiliateReferredTotalVolume: Number(info.referredTotalVolume), affiliateReferredMakerFees: Number(info.totalReferredMakerFees), affiliateReferredTakerFees: Number(info.totalReferredTakerFees), + affiliateReferredMakerRebates: Number(info.totalReferredMakerRebates), })); const response: AffiliateSnapshotResponse = { diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 81eace3303..0e0a57512c 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -733,6 +733,7 @@ export interface AffiliateSnapshotResponseObject { affiliateReferredTotalVolume: number, affiliateReferredMakerFees: number, affiliateReferredTakerFees: number, + affiliateReferredMakerRebates: number, } export interface AffiliateTotalVolumeResponse { From 4b19a59e10eee7b7e6a528ccbafa29574202f3ee Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:03:07 -0400 Subject: [PATCH 20/41] [OTE-846] Bazooka sequential clear (backport #2423) (#2477) Co-authored-by: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> --- indexer/services/bazooka/src/index.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/indexer/services/bazooka/src/index.ts b/indexer/services/bazooka/src/index.ts index 623a97a90b..8329bd69a0 100644 --- a/indexer/services/bazooka/src/index.ts +++ b/indexer/services/bazooka/src/index.ts @@ -264,14 +264,17 @@ async function partitionKafkaTopics(): Promise { async function clearKafkaTopics( existingKafkaTopics: string[], ): Promise { - await Promise.all( - _.map(KAFKA_TOPICS, - clearKafkaTopic.bind(null, - 1, - config.CLEAR_KAFKA_TOPIC_RETRY_MS, - config.CLEAR_KAFKA_TOPIC_MAX_RETRIES, - existingKafkaTopics)), - ); + // Concurrent calls to clear all topics caused the failure: + // TypeError: Cannot destructure property 'partitions' of 'high.pop(...)' as it is undefined. + for (const topic of KAFKA_TOPICS) { + await clearKafkaTopic( + 1, + config.CLEAR_KAFKA_TOPIC_RETRY_MS, + config.CLEAR_KAFKA_TOPIC_MAX_RETRIES, + existingKafkaTopics, + topic, + ); + } } export async function clearKafkaTopic( From ca6c044542c451395ed151243a14f652d2164e32 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:22:41 -0400 Subject: [PATCH 21/41] [OTE-863] update username generation query (backport #2482) (#2483) Co-authored-by: Mohammed Affan --- .../postgres/src/stores/subaccount-usernames-table.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts b/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts index a60b1c6da8..72a894ce58 100644 --- a/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts +++ b/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts @@ -103,10 +103,9 @@ export async function getSubaccountsWithoutUsernames( const queryString: string = ` SELECT id as "subaccountId" FROM subaccounts - WHERE id NOT IN ( - SELECT "subaccountId" FROM subaccount_usernames - ) - AND subaccounts."subaccountNumber"=0 + WHERE subaccounts."subaccountNumber" = 0 + EXCEPT + SELECT "subaccountId" FROM subaccount_usernames; `; const result: { From 071aa313abf014592e8050d9308bf7a4346d9b77 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:20:42 -0400 Subject: [PATCH 22/41] Improve vault endpoint performance. (backport #2475) (#2484) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../funding-index-updates-table.test.ts | 34 +++++++ .../src/stores/funding-index-updates-table.ts | 75 +++++++++++++- .../controllers/api/v4/vault-controller.ts | 99 +++++++++++-------- 3 files changed, 164 insertions(+), 44 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts b/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts index d42df73764..de7daaa34e 100644 --- a/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts @@ -242,4 +242,38 @@ describe('funding index update store', () => { expect(fundingIndexMap[defaultPerpetualMarket2.id]).toEqual(Big(0)); }, ); + + it('Successfully finds funding index maps for multiple effectiveBeforeOrAtHeights', async () => { + const fundingIndexUpdates2: FundingIndexUpdatesCreateObject = { + ...defaultFundingIndexUpdate, + fundingIndex: '124', + effectiveAtHeight: updatedHeight, + effectiveAt: '1982-05-25T00:00:00.000Z', + eventId: defaultTendermintEventId2, + }; + const fundingIndexUpdates3: FundingIndexUpdatesCreateObject = { + ...defaultFundingIndexUpdate, + eventId: defaultTendermintEventId3, + perpetualId: defaultPerpetualMarket2.id, + }; + await Promise.all([ + FundingIndexUpdatesTable.create(defaultFundingIndexUpdate), + FundingIndexUpdatesTable.create(fundingIndexUpdates2), + FundingIndexUpdatesTable.create(fundingIndexUpdates3), + ]); + + const fundingIndexMaps: {[blockHeight:string]: FundingIndexMap} = await FundingIndexUpdatesTable + .findFundingIndexMaps( + ['3', '6'], + ); + + expect(fundingIndexMaps['3'][defaultFundingIndexUpdate.perpetualId]) + .toEqual(Big(defaultFundingIndexUpdate.fundingIndex)); + expect(fundingIndexMaps['3'][fundingIndexUpdates3.perpetualId]) + .toEqual(Big(fundingIndexUpdates3.fundingIndex)); + expect(fundingIndexMaps['6'][defaultFundingIndexUpdate.perpetualId]) + .toEqual(Big(fundingIndexUpdates2.fundingIndex)); + expect(fundingIndexMaps['6'][fundingIndexUpdates3.perpetualId]) + .toEqual(Big(fundingIndexUpdates3.fundingIndex)); + }); }); diff --git a/indexer/packages/postgres/src/stores/funding-index-updates-table.ts b/indexer/packages/postgres/src/stores/funding-index-updates-table.ts index dce9a47028..8ef55a3537 100644 --- a/indexer/packages/postgres/src/stores/funding-index-updates-table.ts +++ b/indexer/packages/postgres/src/stores/funding-index-updates-table.ts @@ -3,6 +3,7 @@ import _ from 'lodash'; import { QueryBuilder } from 'objection'; import { BUFFER_ENCODING_UTF_8, DEFAULT_POSTGRES_OPTIONS } from '../constants'; +import { knexReadReplica } from '../helpers/knex'; import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; import Transaction from '../helpers/transaction'; import { getUuid } from '../helpers/uuid'; @@ -21,6 +22,14 @@ import { } from '../types'; import * as PerpetualMarketTable from './perpetual-market-table'; +// Assuming block time of 1 second, this should be 4 hours of blocks +const FOUR_HOUR_OF_BLOCKS = Big(3600).times(4); +// Type used for querying for funding index maps for multiple effective heights. +interface FundingIndexUpdatesFromDatabaseWithSearchHeight extends FundingIndexUpdatesFromDatabase { + // max effective height being queried for + searchHeight: string, +} + export function uuid( blockHeight: string, eventId: Buffer, @@ -193,8 +202,6 @@ export async function findFundingIndexMap( options, ); - // Assuming block time of 1 second, this should be 4 hours of blocks - const FOUR_HOUR_OF_BLOCKS = Big(3600).times(4); const fundingIndexUpdates: FundingIndexUpdatesFromDatabase[] = await baseQuery .distinctOn(FundingIndexUpdatesColumns.perpetualId) .where(FundingIndexUpdatesColumns.effectiveAtHeight, '<=', effectiveBeforeOrAtHeight) @@ -216,3 +223,67 @@ export async function findFundingIndexMap( initialFundingIndexMap, ); } + +/** + * Finds funding index maps for multiple effective before or at heights. Uses a SQL query unnesting + * an array of effective before or at heights and cross-joining with the funding index updates table + * to find the closest funding index update per effective before or at height. + * @param effectiveBeforeOrAtHeights Heights to get funding index maps for. + * @param options + * @returns Object mapping block heights to the respective funding index maps. + */ +export async function findFundingIndexMaps( + effectiveBeforeOrAtHeights: string[], + options: Options = DEFAULT_POSTGRES_OPTIONS, +): Promise<{[blockHeight: string]: FundingIndexMap}> { + const heightNumbers: number[] = effectiveBeforeOrAtHeights + .map((height: string):number => parseInt(height, 10)) + .filter((parsedHeight: number): boolean => { return !Number.isNaN(parsedHeight); }) + .sort(); + // Get the min height to limit the search to blocks 4 hours or before the min height. + const minHeight: number = heightNumbers[0]; + + const result: { + rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT + DISTINCT ON ("perpetualId", "searchHeight") "perpetualId", "searchHeight", + "funding_index_updates".* + FROM + "funding_index_updates", + unnest(ARRAY[${heightNumbers.join(',')}]) AS "searchHeight" + WHERE + "effectiveAtHeight" > ${Big(minHeight).minus(FOUR_HOUR_OF_BLOCKS).toFixed()} AND + "effectiveAtHeight" <= "searchHeight" + ORDER BY + "perpetualId", + "searchHeight", + "effectiveAtHeight" DESC + `, + ) as unknown as { + rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[], + }; + + const perpetualMarkets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll( + {}, + [], + options, + ); + + const fundingIndexMaps:{[blockHeight: string]: FundingIndexMap} = {}; + for (const height of effectiveBeforeOrAtHeights) { + fundingIndexMaps[height] = _.reduce(perpetualMarkets, + (acc: FundingIndexMap, perpetualMarket: PerpetualMarketFromDatabase): FundingIndexMap => { + acc[perpetualMarket.id] = Big(0); + return acc; + }, + {}, + ); + } + for (const funding of result.rows) { + fundingIndexMaps[funding.searchHeight][funding.perpetualId] = Big(funding.fundingIndex); + } + + return fundingIndexMaps; +} diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index d5e8ff5c42..9480b096f1 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -375,10 +375,23 @@ async function getVaultPositions( BlockTable.getLatest(), ]); - const latestFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable - .findFundingIndexMap( - latestBlock.blockHeight, - ); + const updatedAtHeights: string[] = _(subaccounts).map('updatedAtHeight').uniq().value(); + const [ + latestFundingIndexMap, + fundingIndexMaps, + ]: [ + FundingIndexMap, + {[blockHeight: string]: FundingIndexMap} + ] = await Promise.all([ + FundingIndexUpdatesTable + .findFundingIndexMap( + latestBlock.blockHeight, + ), + FundingIndexUpdatesTable + .findFundingIndexMaps( + updatedAtHeights, + ), + ]); const assetPositionsBySubaccount: { [subaccountId: string]: AssetPositionFromDatabase[] } = _.groupBy( assetPositions, @@ -397,47 +410,49 @@ async function getVaultPositions( const vaultPositionsAndSubaccountId: { position: VaultPosition, subaccountId: string, - }[] = await Promise.all( - subaccounts.map(async (subaccount: SubaccountFromDatabase) => { - const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); - if (perpetualMarket === undefined) { - throw new Error( - `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + + }[] = subaccounts.map((subaccount: SubaccountFromDatabase) => { + const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher + .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); + if (perpetualMarket === undefined) { + throw new Error( + `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + 'perpetual market.'); - } - const lastUpdatedFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable - .findFundingIndexMap( - subaccount.updatedAtHeight, - ); - - const subaccountResponse: SubaccountResponseObject = getSubaccountResponse( - subaccount, - openPerpetualPositionsBySubaccount[subaccount.id] || [], - assetPositionsBySubaccount[subaccount.id] || [], - assets, - markets, - perpetualMarketRefresher.getPerpetualMarketsMap(), - latestBlock.blockHeight, - latestFundingIndexMap, - lastUpdatedFundingIndexMap, + } + const lastUpdatedFundingIndexMap: FundingIndexMap = fundingIndexMaps[ + subaccount.updatedAtHeight + ]; + if (lastUpdatedFundingIndexMap === undefined) { + throw new Error( + `No funding indices could be found for vault with subaccount ${subaccount.id}`, ); + } - return { - position: { - ticker: perpetualMarket.ticker, - assetPosition: subaccountResponse.assetPositions[ - assetIdToAsset[USDC_ASSET_ID].symbol - ], - perpetualPosition: subaccountResponse.openPerpetualPositions[ - perpetualMarket.ticker - ] || undefined, - equity: subaccountResponse.equity, - }, - subaccountId: subaccount.id, - }; - }), - ); + const subaccountResponse: SubaccountResponseObject = getSubaccountResponse( + subaccount, + openPerpetualPositionsBySubaccount[subaccount.id] || [], + assetPositionsBySubaccount[subaccount.id] || [], + assets, + markets, + perpetualMarketRefresher.getPerpetualMarketsMap(), + latestBlock.blockHeight, + latestFundingIndexMap, + lastUpdatedFundingIndexMap, + ); + + return { + position: { + ticker: perpetualMarket.ticker, + assetPosition: subaccountResponse.assetPositions[ + assetIdToAsset[USDC_ASSET_ID].symbol + ], + perpetualPosition: subaccountResponse.openPerpetualPositions[ + perpetualMarket.ticker + ] || undefined, + equity: subaccountResponse.equity, + }, + subaccountId: subaccount.id, + }; + }); return new Map(vaultPositionsAndSubaccountId.map( (obj: { position: VaultPosition, subaccountId: string }) : [string, VaultPosition] => { From 7a2200ec883892cec3862e536c11298b479b85f9 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:24:33 -0400 Subject: [PATCH 23/41] Return undefined from getOrderbookMidPriceMap (backport #2441) (#2486) Co-authored-by: Adam Fraser --- .../__tests__/lib/candles-generator.test.ts | 216 ++---------------- .../ender/src/lib/candles-generator.ts | 8 +- 2 files changed, 14 insertions(+), 210 deletions(-) diff --git a/indexer/services/ender/__tests__/lib/candles-generator.test.ts b/indexer/services/ender/__tests__/lib/candles-generator.test.ts index d58ebd0d2a..cd014d3eaf 100644 --- a/indexer/services/ender/__tests__/lib/candles-generator.test.ts +++ b/indexer/services/ender/__tests__/lib/candles-generator.test.ts @@ -137,8 +137,8 @@ describe('candleHelper', () => { id: CandleTable.uuid(currentStartedAt, defaultCandle.ticker, resolution), startedAt: currentStartedAt, resolution, - orderbookMidPriceClose: '105000', - orderbookMidPriceOpen: '105000', + orderbookMidPriceClose: null, + orderbookMidPriceOpen: null, }; }, ); @@ -187,8 +187,8 @@ describe('candleHelper', () => { startedAt: currentStartedAt, resolution, startingOpenInterest: openInterest, - orderbookMidPriceClose: '80500', - orderbookMidPriceOpen: '80500', + orderbookMidPriceClose: null, + orderbookMidPriceOpen: null, }; }, ); @@ -311,8 +311,8 @@ describe('candleHelper', () => { usdVolume: '0', trades: 0, startingOpenInterest: '100', - orderbookMidPriceClose: '1000', - orderbookMidPriceOpen: '1000', + orderbookMidPriceClose: null, + orderbookMidPriceOpen: null, }, true, 1000, @@ -342,8 +342,8 @@ describe('candleHelper', () => { startedAt, resolution: CandleResolution.ONE_MINUTE, startingOpenInterest: '100', - orderbookMidPriceClose: '1000', - orderbookMidPriceOpen: '1000', + orderbookMidPriceClose: null, + orderbookMidPriceOpen: null, }, true, // contains kafka messages 1000, // orderbook mid price @@ -471,196 +471,6 @@ describe('candleHelper', () => { expectTimingStats(); }); - it('Updates previous candle orderBookMidPriceClose if startTime is past candle resolution', async () => { - // Create existing candles - const existingPrice: string = '7000'; - const startingOpenInterest: string = '200'; - const baseTokenVolume: string = '10'; - const usdVolume: string = Big(existingPrice).times(baseTokenVolume).toString(); - const orderbookMidPriceClose = '7500'; - const orderbookMidPriceOpen = '8000'; - await Promise.all( - _.map(Object.values(CandleResolution), (resolution: CandleResolution) => { - return CandleTable.create({ - startedAt: previousStartedAt, - ticker: testConstants.defaultPerpetualMarket.ticker, - resolution, - low: existingPrice, - high: existingPrice, - open: existingPrice, - close: existingPrice, - baseTokenVolume, - usdVolume, - trades: existingTrades, - startingOpenInterest, - orderbookMidPriceClose, - orderbookMidPriceOpen, - }); - }), - ); - await startCandleCache(); - - await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); - - const publisher: KafkaPublisher = new KafkaPublisher(); - publisher.addEvents([ - defaultTradeKafkaEvent, - defaultTradeKafkaEvent2, - ]); - - // Create new candles, with trades - await runUpdateCandles(publisher).then(async () => { - - // Verify previous candles have orderbookMidPriceClose updated - const previousExpectedCandles: CandleFromDatabase[] = _.map( - Object.values(CandleResolution), - (resolution: CandleResolution) => { - return { - id: CandleTable.uuid(previousStartedAt, defaultCandle.ticker, resolution), - startedAt: previousStartedAt, - ticker: defaultCandle.ticker, - resolution, - low: existingPrice, - high: existingPrice, - open: existingPrice, - close: existingPrice, - baseTokenVolume, - usdVolume, - trades: existingTrades, - startingOpenInterest, - orderbookMidPriceClose: '10005', - orderbookMidPriceOpen, - }; - }, - ); - await verifyCandlesInPostgres(previousExpectedCandles); - }); - - // Verify new candles were created - const expectedCandles: CandleFromDatabase[] = _.map( - Object.values(CandleResolution), - (resolution: CandleResolution) => { - const currentStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( - testConstants.createdDateTime, - resolution, - ).toISO(); - - return { - id: CandleTable.uuid(currentStartedAt, defaultCandle.ticker, resolution), - startedAt: currentStartedAt, - ticker: defaultCandle.ticker, - resolution, - low: '10000', - high: defaultPrice2, - open: '10000', - close: defaultPrice2, - baseTokenVolume: '20', - usdVolume: '250000', - trades: 2, - startingOpenInterest: '0', - orderbookMidPriceClose: '10005', - orderbookMidPriceOpen: '10005', - }; - }, - ); - await verifyCandlesInPostgres(expectedCandles); - await validateCandlesCache(); - expectTimingStats(); - }); - - it('creates an empty candle and updates the previous candle orderBookMidPriceClose if startTime is past candle resolution', async () => { - // Create existing candles - const existingPrice: string = '7000'; - const startingOpenInterest: string = '200'; - const baseTokenVolume: string = '10'; - const usdVolume: string = Big(existingPrice).times(baseTokenVolume).toString(); - const orderbookMidPriceClose = '7500'; - const orderbookMidPriceOpen = '8000'; - - await Promise.all( - _.map(Object.values(CandleResolution), (resolution: CandleResolution) => { - return CandleTable.create({ - startedAt: previousStartedAt, - ticker: testConstants.defaultPerpetualMarket.ticker, - resolution, - low: existingPrice, - high: existingPrice, - open: existingPrice, - close: existingPrice, - baseTokenVolume, - usdVolume, - trades: existingTrades, - startingOpenInterest, - orderbookMidPriceClose, - orderbookMidPriceOpen, - }); - }), - ); - await startCandleCache(); - - await OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '10005'); - - const publisher: KafkaPublisher = new KafkaPublisher(); - publisher.addEvents([]); - - // Create new candles, without trades - await runUpdateCandles(publisher); - - // Verify previous candles have orderbookMidPriceClose updated - const previousExpectedCandles: CandleFromDatabase[] = _.map( - Object.values(CandleResolution), - (resolution: CandleResolution) => { - return { - id: CandleTable.uuid(previousStartedAt, defaultCandle.ticker, resolution), - startedAt: previousStartedAt, - ticker: defaultCandle.ticker, - resolution, - low: existingPrice, - high: existingPrice, - open: existingPrice, - close: existingPrice, - baseTokenVolume, - usdVolume, - trades: existingTrades, - startingOpenInterest, - orderbookMidPriceClose: '10005', - orderbookMidPriceOpen, - }; - }, - ); - await verifyCandlesInPostgres(previousExpectedCandles); - - // Verify new empty candle was created - const expectedCandles: CandleFromDatabase[] = _.map( - Object.values(CandleResolution), - (resolution: CandleResolution) => { - const currentStartedAt: IsoString = helpers.calculateNormalizedCandleStartTime( - testConstants.createdDateTime, - resolution, - ).toISO(); - - return { - id: CandleTable.uuid(currentStartedAt, defaultCandle.ticker, resolution), - startedAt: currentStartedAt, - ticker: defaultCandle.ticker, - resolution, - low: existingPrice, - high: existingPrice, - open: existingPrice, - close: existingPrice, - baseTokenVolume: '0', - usdVolume: '0', - trades: 0, - startingOpenInterest: '0', - orderbookMidPriceClose: '10005', - orderbookMidPriceOpen: '10005', - }; - }, - ); - await verifyCandlesInPostgres(expectedCandles); - - }); - it('successfully creates an orderbook price map for each market', async () => { await Promise.all([ OrderbookMidPricesCache.setPrice(redisClient, 'BTC-USD', '105000'), @@ -670,11 +480,11 @@ describe('candleHelper', () => { const map = await getOrderbookMidPriceMap(); expect(map).toEqual({ - 'BTC-USD': '105000', - 'ETH-USD': '150000', - 'ISO-USD': '115000', - 'ISO2-USD': null, - 'SHIB-USD': null, + 'BTC-USD': undefined, + 'ETH-USD': undefined, + 'ISO-USD': undefined, + 'ISO2-USD': undefined, + 'SHIB-USD': undefined, }); }); }); diff --git a/indexer/services/ender/src/lib/candles-generator.ts b/indexer/services/ender/src/lib/candles-generator.ts index b232a66eb0..f1daa75f06 100644 --- a/indexer/services/ender/src/lib/candles-generator.ts +++ b/indexer/services/ender/src/lib/candles-generator.ts @@ -20,7 +20,6 @@ import { TradeMessageContents, helpers, } from '@dydxprotocol-indexer/postgres'; -import { OrderbookMidPricesCache } from '@dydxprotocol-indexer/redis'; import { CandleMessage } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; import _ from 'lodash'; @@ -28,7 +27,6 @@ import { DateTime } from 'luxon'; import { getCandle } from '../caches/candle-cache'; import config from '../config'; -import { redisClient } from '../helpers/redis/redis-controller'; import { KafkaPublisher } from './kafka-publisher'; import { ConsolidatedKafkaEvent, SingleTradeMessage } from './types'; @@ -538,11 +536,7 @@ export async function getOrderbookMidPriceMap(): Promise<{ [ticker: string]: Ord const perpetualMarkets = Object.values(perpetualMarketRefresher.getPerpetualMarketsMap()); const promises = perpetualMarkets.map(async (perpetualMarket: PerpetualMarketFromDatabase) => { - const price = await OrderbookMidPricesCache.getMedianPrice( - redisClient, - perpetualMarket.ticker, - ); - return { [perpetualMarket.ticker]: price === undefined ? undefined : price }; + return Promise.resolve({ [perpetualMarket.ticker]: undefined }); }); const pricesArray = await Promise.all(promises); From 54b3bced767db99e461a2d8a63e14b7990287ae5 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:25:42 -0400 Subject: [PATCH 24/41] [CT-629] Fix entryPrice calc (backport #2455) (#2496) Co-authored-by: dydxwill <119354122+dydxwill@users.noreply.github.com> --- .../order-fills/liquidation-handler.test.ts | 4 ++-- .../order-fills/order-handler.test.ts | 22 ++++++++++++++++--- ...ydx_liquidation_fill_handler_per_order.sql | 2 +- ...te_perpetual_position_aggregate_fields.sql | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts index a5cc41f3d6..805ca676f0 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts @@ -137,7 +137,7 @@ describe('LiquidationHandler', () => { perpetualId: testConstants.defaultPerpetualMarket.id, side: PositionSide.LONG, status: PerpetualPositionStatus.OPEN, - size: '10', + size: '5', maxSize: '25', sumOpen: '10', entryPrice: '15000', @@ -392,7 +392,7 @@ describe('LiquidationHandler', () => { defaultPerpetualPosition.openEventId, ), { - sumOpen: Big(defaultPerpetualPosition.size).plus(totalFilled).toFixed(), + sumOpen: Big(defaultPerpetualPosition.sumOpen!).plus(totalFilled).toFixed(), entryPrice: getWeightedAverage( defaultPerpetualPosition.entryPrice!, defaultPerpetualPosition.size, diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index ba9a62ab34..65cb1c0422 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -138,7 +138,7 @@ describe('OrderHandler', () => { perpetualId: testConstants.defaultPerpetualMarket.id, side: PositionSide.LONG, status: PerpetualPositionStatus.OPEN, - size: '10', + size: '5', maxSize: '25', sumOpen: '10', entryPrice: '15000', @@ -212,6 +212,7 @@ describe('OrderHandler', () => { { goodTilBlock: 15, }, + false, ], [ 'goodTilBlockTime', @@ -221,6 +222,17 @@ describe('OrderHandler', () => { { goodTilBlockTime: 1_000_005_000, }, + false, + ], + [ + 'goodTilBlock', + { + goodTilBlock: 10, + }, + { + goodTilBlock: 15, + }, + true, ], ])( 'creates fills and orders (with %s), sends vulcan messages for order updates and order ' + @@ -229,6 +241,7 @@ describe('OrderHandler', () => { _name: string, makerGoodTilOneof: Partial, takerGoodTilOneof: Partial, + useNegativeSize: boolean, ) => { const transactionIndex: number = 0; const eventIndex: number = 0; @@ -284,7 +297,10 @@ describe('OrderHandler', () => { // create PerpetualPositions await Promise.all([ - PerpetualPositionTable.create(defaultPerpetualPosition), + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + size: useNegativeSize ? '-5' : defaultPerpetualPosition.size, + }), PerpetualPositionTable.create({ ...defaultPerpetualPosition, subaccountId: testConstants.defaultSubaccountId2, @@ -439,7 +455,7 @@ describe('OrderHandler', () => { defaultPerpetualPosition.openEventId, ), { - sumOpen: Big(defaultPerpetualPosition.size).plus(totalFilled).toFixed(), + sumOpen: Big(defaultPerpetualPosition.sumOpen!).plus(totalFilled).toFixed(), entryPrice: getWeightedAverage( defaultPerpetualPosition.entryPrice!, defaultPerpetualPosition.size, diff --git a/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql index 493b1257d6..dc3dfa5e4c 100644 --- a/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/helpers/dydx_liquidation_fill_handler_per_order.sql @@ -200,7 +200,7 @@ BEGIN perpetual_position_record."side", order_side) THEN sum_open = dydx_trim_scale(perpetual_position_record."sumOpen" + fill_amount); entry_price = dydx_get_weighted_average( - perpetual_position_record."entryPrice", perpetual_position_record."sumOpen", + perpetual_position_record."entryPrice", ABS(perpetual_position_record."size"), maker_price, fill_amount); perpetual_position_record."sumOpen" = sum_open; perpetual_position_record."entryPrice" = entry_price; diff --git a/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql b/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql index d021eecf28..f6e269a0b1 100644 --- a/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql +++ b/indexer/services/ender/src/scripts/helpers/dydx_update_perpetual_position_aggregate_fields.sql @@ -44,7 +44,7 @@ BEGIN IF dydx_perpetual_position_and_order_side_matching(perpetual_position_record."side", side) THEN sum_open := dydx_trim_scale(perpetual_position_record."sumOpen" + size); entry_price := dydx_get_weighted_average( - perpetual_position_record."entryPrice", perpetual_position_record."sumOpen", price, size + perpetual_position_record."entryPrice", ABS(perpetual_position_record."size"), price, size ); perpetual_position_record."sumOpen" = sum_open; perpetual_position_record."entryPrice" = entry_price; From 666898fc6fefd2c785d9294b9752e1a1548a74ad Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:27:21 -0400 Subject: [PATCH 25/41] Don't increment messageId for custom ping messages (backport #2493) (#2497) Co-authored-by: dydxwill <119354122+dydxwill@users.noreply.github.com> --- indexer/services/socks/src/websocket/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/indexer/services/socks/src/websocket/index.ts b/indexer/services/socks/src/websocket/index.ts index 795f6a7527..1906d3a72e 100644 --- a/indexer/services/socks/src/websocket/index.ts +++ b/indexer/services/socks/src/websocket/index.ts @@ -243,8 +243,6 @@ export class Index { return; } - this.connections[connectionId].messageId += 1; - const messageStr = message.toString(); let parsed: IncomingMessage; @@ -282,6 +280,8 @@ export class Index { messageContents: safeJsonStringify(message), }); + this.connections[connectionId].messageId += 1; + // Do not wait for this. this.subscriptions.subscribe( this.connections[connectionId].ws, @@ -311,6 +311,8 @@ export class Index { unsubscribeMessage.id, ); + this.connections[connectionId].messageId += 1; + sendMessage( this.connections[connectionId].ws, connectionId, From 5042763fa0b8ba6cbbdddf8fc18432eb5b251abc Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:08:13 -0400 Subject: [PATCH 26/41] [OTE-880] Emit log in case of collisions (backport #2500) (#2504) Co-authored-by: Mohammed Affan --- .../tasks/subaccount-username-generator.ts | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/indexer/services/roundtable/src/tasks/subaccount-username-generator.ts b/indexer/services/roundtable/src/tasks/subaccount-username-generator.ts index af0312481c..819afa5bfd 100644 --- a/indexer/services/roundtable/src/tasks/subaccount-username-generator.ts +++ b/indexer/services/roundtable/src/tasks/subaccount-username-generator.ts @@ -1,9 +1,10 @@ -import { logger } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { SubaccountUsernamesTable, SubaccountsWithoutUsernamesResult, } from '@dydxprotocol-indexer/postgres'; +import config from '../config'; import { generateUsername } from '../helpers/usernames-helper'; export default async function runTask(): Promise { @@ -21,13 +22,18 @@ export default async function runTask(): Promise { subaccountId: subaccount.subaccountId, }); } catch (e) { - logger.error({ - at: 'subaccount-username-generator#runTask', - message: 'Failed to insert username for subaccount', - subaccountId: subaccount.subaccountId, - username, - error: e, - }); + if (e instanceof Error && e.name === 'UniqueViolationError') { + stats.increment( + `${config.SERVICE_NAME}.subaccount-username-generator.collision`, 1); + } else { + logger.error({ + at: 'subaccount-username-generator#runTask', + message: 'Failed to insert username for subaccount', + subaccountId: subaccount.subaccountId, + username, + error: e, + }); + } } } } From 3339df5e1a80e30a75f0dd83f99d6980cfde9833 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:25:01 -0400 Subject: [PATCH 27/41] [OTE-876] update roundtable loop timings for instrumentation and uncrossing (backport #2494) (#2506) Co-authored-by: Mohammed Affan --- indexer/services/roundtable/src/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index 42b66a4882..34dafba44b 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -70,7 +70,7 @@ export const configSchema = { default: 2 * ONE_MINUTE_IN_MILLISECONDS, }), LOOPS_INTERVAL_MS_UNCROSS_ORDERBOOK: parseInteger({ - default: THIRTY_SECONDS_IN_MILLISECONDS, + default: 15 * ONE_SECOND_IN_MILLISECONDS, }), LOOPS_INTERVAL_MS_PNL_TICKS: parseInteger({ default: THIRTY_SECONDS_IN_MILLISECONDS, @@ -79,7 +79,7 @@ export const configSchema = { default: 2 * ONE_MINUTE_IN_MILLISECONDS, }), LOOPS_INTERVAL_MS_ORDERBOOK_INSTRUMENTATION: parseInteger({ - default: 5 * ONE_SECOND_IN_MILLISECONDS, + default: 1 * ONE_MINUTE_IN_MILLISECONDS, }), LOOPS_INTERVAL_MS_PNL_INSTRUMENTATION: parseInteger({ default: ONE_HOUR_IN_MILLISECONDS, From e2298c430bbdd6db080116fb0400f9fc3fa7f3ff Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:58:02 -0400 Subject: [PATCH 28/41] Remove orderbook cache roundtable job (backport #2510) (#2511) Co-authored-by: Adam Fraser --- .../tasks/cache-orderbook-mid-prices.test.ts | 98 ------------------- indexer/services/roundtable/src/config.ts | 1 - indexer/services/roundtable/src/index.ts | 9 -- .../src/tasks/cache-orderbook-mid-prices.ts | 40 -------- 4 files changed, 148 deletions(-) delete mode 100644 indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts delete mode 100644 indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts diff --git a/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts b/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts deleted file mode 100644 index cd0eee3970..0000000000 --- a/indexer/services/roundtable/__tests__/tasks/cache-orderbook-mid-prices.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { - dbHelpers, - testConstants, - testMocks, -} from '@dydxprotocol-indexer/postgres'; -import { - OrderbookMidPricesCache, - OrderbookLevelsCache, - redis, -} from '@dydxprotocol-indexer/redis'; -import { redisClient } from '../../src/helpers/redis'; -import runTask from '../../src/tasks/cache-orderbook-mid-prices'; - -jest.mock('@dydxprotocol-indexer/base', () => ({ - ...jest.requireActual('@dydxprotocol-indexer/base'), - logger: { - info: jest.fn(), - error: jest.fn(), - }, -})); - -jest.mock('@dydxprotocol-indexer/redis', () => ({ - ...jest.requireActual('@dydxprotocol-indexer/redis'), - OrderbookLevelsCache: { - getOrderBookMidPrice: jest.fn(), - }, -})); - -describe('cache-orderbook-mid-prices', () => { - beforeAll(async () => { - await dbHelpers.migrate(); - }); - - beforeEach(async () => { - await dbHelpers.clearData(); - await redis.deleteAllAsync(redisClient); - await testMocks.seedData(); - }); - - afterAll(async () => { - await dbHelpers.teardown(); - jest.resetAllMocks(); - }); - - it('caches mid prices for all markets', async () => { - const market1 = testConstants.defaultPerpetualMarket; - const market2 = testConstants.defaultPerpetualMarket2; - - const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); - mockGetOrderBookMidPrice.mockResolvedValueOnce('100.5'); // For market1 - mockGetOrderBookMidPrice.mockResolvedValueOnce('200.75'); // For market2 - - await runTask(); - - // Check if the mock was called with the correct arguments - expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market1.ticker, redisClient); - expect(mockGetOrderBookMidPrice).toHaveBeenCalledWith(market2.ticker, redisClient); - - // Check if prices were cached correctly - const price1 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market1.ticker); - const price2 = await OrderbookMidPricesCache.getMedianPrice(redisClient, market2.ticker); - - expect(price1).toBe('100.5'); - expect(price2).toBe('200.75'); - }); - - it('handles undefined prices', async () => { - const market = testConstants.defaultPerpetualMarket; - - const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); - mockGetOrderBookMidPrice.mockResolvedValueOnce(undefined); - - await runTask(); - - const price = await OrderbookMidPricesCache.getMedianPrice(redisClient, market.ticker); - expect(price).toBeNull(); - - // Check that a log message was created - expect(jest.requireMock('@dydxprotocol-indexer/base').logger.info).toHaveBeenCalledWith({ - at: 'cache-orderbook-mid-prices#runTask', - message: `undefined price for ${market.ticker}`, - }); - }); - - it('handles errors', async () => { - // Mock OrderbookLevelsCache.getOrderBookMidPrice to throw an error - const mockGetOrderBookMidPrice = jest.spyOn(OrderbookLevelsCache, 'getOrderBookMidPrice'); - mockGetOrderBookMidPrice.mockRejectedValueOnce(new Error('Test error')); - - await runTask(); - - expect(jest.requireMock('@dydxprotocol-indexer/base').logger.error).toHaveBeenCalledWith({ - at: 'cache-orderbook-mid-prices#runTask', - message: 'Test error', - error: expect.any(Error), - }); - }); -}); diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index 34dafba44b..bdc8877e5c 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -60,7 +60,6 @@ export const configSchema = { LOOPS_ENABLED_UPDATE_WALLET_TOTAL_VOLUME: parseBoolean({ default: true }), LOOPS_ENABLED_UPDATE_AFFILIATE_INFO: parseBoolean({ default: true }), LOOPS_ENABLED_DELETE_OLD_FIREBASE_NOTIFICATION_TOKENS: parseBoolean({ default: true }), - LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES: parseBoolean({ default: true }), // Loop Timing LOOPS_INTERVAL_MS_MARKET_UPDATER: parseInteger({ diff --git a/indexer/services/roundtable/src/index.ts b/indexer/services/roundtable/src/index.ts index bfdee334c7..f52903ac19 100644 --- a/indexer/services/roundtable/src/index.ts +++ b/indexer/services/roundtable/src/index.ts @@ -10,7 +10,6 @@ import { connect as connectToRedis, } from './helpers/redis'; import aggregateTradingRewardsTasks from './tasks/aggregate-trading-rewards'; -import cacheOrderbookMidPrices from './tasks/cache-orderbook-mid-prices'; import cancelStaleOrdersTask from './tasks/cancel-stale-orders'; import createLeaderboardTask from './tasks/create-leaderboard'; import createPnlTicksTask from './tasks/create-pnl-ticks'; @@ -273,14 +272,6 @@ async function start(): Promise { ); } - if (config.LOOPS_ENABLED_CACHE_ORDERBOOK_MID_PRICES) { - startLoop( - cacheOrderbookMidPrices, - 'cache_orderbook_mid_prices', - config.LOOPS_INTERVAL_MS_CACHE_ORDERBOOK_MID_PRICES, - ); - } - logger.info({ at: 'index', message: 'Successfully started', diff --git a/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts b/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts deleted file mode 100644 index 644f50df6f..0000000000 --- a/indexer/services/roundtable/src/tasks/cache-orderbook-mid-prices.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - logger, -} from '@dydxprotocol-indexer/base'; -import { - PerpetualMarketFromDatabase, - PerpetualMarketTable, -} from '@dydxprotocol-indexer/postgres'; -import { - OrderbookMidPricesCache, - OrderbookLevelsCache, -} from '@dydxprotocol-indexer/redis'; - -import { redisClient } from '../helpers/redis'; - -/** - * Updates OrderbookMidPricesCache with current orderbook mid price for each market - */ -export default async function runTask(): Promise { - const markets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll({}, []); - - for (const market of markets) { - try { - const price = await OrderbookLevelsCache.getOrderBookMidPrice(market.ticker, redisClient); - if (price) { - await OrderbookMidPricesCache.setPrice(redisClient, market.ticker, price); - } else { - logger.info({ - at: 'cache-orderbook-mid-prices#runTask', - message: `undefined price for ${market.ticker}`, - }); - } - } catch (error) { - logger.error({ - at: 'cache-orderbook-mid-prices#runTask', - message: error.message, - error, - }); - } - } -} From ec8b9195ab9d5d9c70ce57667114e48f2bfc0235 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:44:33 -0400 Subject: [PATCH 29/41] Add config var to exclude specific stateful order ids from being processed. (backport #2513) (#2514) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- ...onditional-order-placement-handler.test.ts | 22 ++++++++++ ...onditional-order-triggered-handler.test.ts | 40 +++++++++++++++++++ .../stateful-order-placement-handler.test.ts | 26 ++++++++++++ .../stateful-order-removal-handler.test.ts | 35 ++++++++++++++++ indexer/services/ender/src/config.ts | 8 ++++ .../services/ender/src/lib/block-processor.ts | 20 +++++++++- indexer/services/ender/src/lib/types.ts | 2 + .../dydx_block_processor_ordered_handlers.sql | 2 + ...ydx_block_processor_unordered_handlers.sql | 2 + .../validators/stateful-order-validator.ts | 38 ++++++++++++++++++ .../ender/src/validators/validator.ts | 9 +++++ 11 files changed, 202 insertions(+), 2 deletions(-) diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts index edc47dfd76..ca5fb27041 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-placement-handler.test.ts @@ -44,8 +44,11 @@ import Long from 'long'; import { producer } from '@dydxprotocol-indexer/kafka'; import { ConditionalOrderPlacementHandler } from '../../../src/handlers/stateful-order/conditional-order-placement-handler'; import { createPostgresFunctions } from '../../../src/helpers/postgres/postgres-functions'; +import config from '../../../src/config'; describe('conditionalOrderPlacementHandler', () => { + const prevSkippedOrderUUIDs: string = config.SKIP_STATEFUL_ORDER_UUIDS; + beforeAll(async () => { await dbHelpers.migrate(); await createPostgresFunctions(); @@ -59,6 +62,7 @@ describe('conditionalOrderPlacementHandler', () => { }); afterEach(async () => { + config.SKIP_STATEFUL_ORDER_UUIDS = prevSkippedOrderUUIDs; await dbHelpers.clearData(); jest.clearAllMocks(); }); @@ -226,4 +230,22 @@ describe('conditionalOrderPlacementHandler', () => { order!, ); }); + + it.each([ + ['transaction event', 0], + ['block event', -1], + ])('successfully skips order (as %s)', async ( + _name: string, + transactionIndex: number, + ) => { + config.SKIP_STATEFUL_ORDER_UUIDS = OrderTable.orderIdToUuid(defaultOrder.orderId!); + const kafkaMessage: KafkaMessage = createKafkaMessageFromStatefulOrderEvent( + defaultStatefulOrderEvent, + transactionIndex, + ); + + await onMessage(kafkaMessage); + const order: OrderFromDatabase | undefined = await OrderTable.findById(orderId); + expect(order).toBeUndefined(); + }); }); diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts index cb5ad98721..9e17cbe521 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/conditional-order-triggered-handler.test.ts @@ -38,8 +38,11 @@ import { ORDER_FLAG_CONDITIONAL } from '@dydxprotocol-indexer/v4-proto-parser'; import { ConditionalOrderTriggeredHandler } from '../../../src/handlers/stateful-order/conditional-order-triggered-handler'; import { defaultPerpetualMarket } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; import { createPostgresFunctions } from '../../../src/helpers/postgres/postgres-functions'; +import config from '../../../src/config'; describe('conditionalOrderTriggeredHandler', () => { + const prevSkippedOrderUUIDs: string = config.SKIP_STATEFUL_ORDER_UUIDS; + beforeAll(async () => { await dbHelpers.migrate(); await createPostgresFunctions(); @@ -53,6 +56,7 @@ describe('conditionalOrderTriggeredHandler', () => { }); afterEach(async () => { + config.SKIP_STATEFUL_ORDER_UUIDS = prevSkippedOrderUUIDs; await dbHelpers.clearData(); jest.clearAllMocks(); }); @@ -163,4 +167,40 @@ describe('conditionalOrderTriggeredHandler', () => { `Unable to update order status with orderId: ${orderId}`, ); }); + + it.each([ + ['transaction event', 0], + ['block event', -1], + ])('successfully skips order trigger event (as %s)', async ( + _name: string, + transactionIndex: number, + ) => { + config.SKIP_STATEFUL_ORDER_UUIDS = OrderTable.uuid( + testConstants.defaultOrderGoodTilBlockTime.subaccountId, + '0', + testConstants.defaultOrderGoodTilBlockTime.clobPairId, + testConstants.defaultOrderGoodTilBlockTime.orderFlags, + ); + await OrderTable.create({ + ...testConstants.defaultOrderGoodTilBlockTime, + orderFlags: conditionalOrderId.orderFlags.toString(), + status: OrderStatus.UNTRIGGERED, + triggerPrice: '1000', + clientId: '0', + }); + const kafkaMessage: KafkaMessage = createKafkaMessageFromStatefulOrderEvent( + defaultStatefulOrderEvent, + transactionIndex, + ); + + await onMessage(kafkaMessage); + const order: OrderFromDatabase | undefined = await OrderTable.findById(orderId); + + expect(order).toBeDefined(); + expect(order).toEqual(expect.objectContaining({ + status: OrderStatus.OPEN, + updatedAt: defaultDateTime.toISO(), + updatedAtHeight: defaultHeight.toString(), + })); + }); }); diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts index b65e6fe6f6..daa3a746a1 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-placement-handler.test.ts @@ -44,8 +44,11 @@ import { STATEFUL_ORDER_ORDER_FILL_EVENT_TYPE } from '../../../src/constants'; import { producer } from '@dydxprotocol-indexer/kafka'; import { ORDER_FLAG_LONG_TERM } from '@dydxprotocol-indexer/v4-proto-parser'; import { createPostgresFunctions } from '../../../src/helpers/postgres/postgres-functions'; +import config from '../../../src/config'; describe('statefulOrderPlacementHandler', () => { + const prevSkippedOrderUUIDs: string = config.SKIP_STATEFUL_ORDER_UUIDS; + beforeAll(async () => { await dbHelpers.migrate(); await createPostgresFunctions(); @@ -59,6 +62,7 @@ describe('statefulOrderPlacementHandler', () => { }); afterEach(async () => { + config.SKIP_STATEFUL_ORDER_UUIDS = prevSkippedOrderUUIDs; await dbHelpers.clearData(); jest.clearAllMocks(); }); @@ -250,4 +254,26 @@ describe('statefulOrderPlacementHandler', () => { }); // TODO[IND-20]: Add tests for vulcan messages }); + + it.each([ + // TODO(IND-334): Remove after deprecating StatefulOrderPlacementEvent + ['stateful order placement as txn event', defaultStatefulOrderEvent, 0], + ['stateful long term order placement as txn event', defaultStatefulOrderLongTermEvent, 0], + ['stateful order placement as block event', defaultStatefulOrderEvent, -1], + ['stateful long term order placement as block event', defaultStatefulOrderLongTermEvent, -1], + ])('successfully skips order with %s', async ( + _name: string, + statefulOrderEvent: StatefulOrderEventV1, + transactionIndex: number, + ) => { + config.SKIP_STATEFUL_ORDER_UUIDS = OrderTable.orderIdToUuid(defaultOrder.orderId!); + const kafkaMessage: KafkaMessage = createKafkaMessageFromStatefulOrderEvent( + statefulOrderEvent, + transactionIndex, + ); + + await onMessage(kafkaMessage); + const order: OrderFromDatabase | undefined = await OrderTable.findById(orderId); + expect(order).toBeUndefined(); + }); }); diff --git a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts index 565b361057..135f745256 100644 --- a/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/stateful-order/stateful-order-removal-handler.test.ts @@ -37,8 +37,11 @@ import { StatefulOrderRemovalHandler } from '../../../src/handlers/stateful-orde import { STATEFUL_ORDER_ORDER_FILL_EVENT_TYPE } from '../../../src/constants'; import { producer } from '@dydxprotocol-indexer/kafka'; import { createPostgresFunctions } from '../../../src/helpers/postgres/postgres-functions'; +import config from '../../../src/config'; describe('statefulOrderRemovalHandler', () => { + const prevSkippedOrderUUIDs: string = config.SKIP_STATEFUL_ORDER_UUIDS; + beforeAll(async () => { await dbHelpers.migrate(); await createPostgresFunctions(); @@ -52,6 +55,7 @@ describe('statefulOrderRemovalHandler', () => { }); afterEach(async () => { + config.SKIP_STATEFUL_ORDER_UUIDS = prevSkippedOrderUUIDs; await dbHelpers.clearData(); jest.clearAllMocks(); }); @@ -153,4 +157,35 @@ describe('statefulOrderRemovalHandler', () => { `Unable to update order status with orderId: ${orderId}`, ); }); + + it.each([ + ['transaction event', 0], + ['block event', -1], + ])('successfully skips order removal event (as %s)', async ( + _name: string, + transactionIndex: number, + ) => { + config.SKIP_STATEFUL_ORDER_UUIDS = OrderTable.uuid( + testConstants.defaultOrder.subaccountId, + '0', + testConstants.defaultOrder.clobPairId, + testConstants.defaultOrder.orderFlags, + ); + await OrderTable.create({ + ...testConstants.defaultOrder, + clientId: '0', + }); + const kafkaMessage: KafkaMessage = createKafkaMessageFromStatefulOrderEvent( + defaultStatefulOrderEvent, + transactionIndex, + ); + + await onMessage(kafkaMessage); + const order: OrderFromDatabase | undefined = await OrderTable.findById(orderId); + expect(order).toBeDefined(); + expect(order).toEqual(expect.objectContaining({ + ...testConstants.defaultOrder, + clientId: '0', + })); + }); }); diff --git a/indexer/services/ender/src/config.ts b/indexer/services/ender/src/config.ts index c22122b318..201f7807ee 100644 --- a/indexer/services/ender/src/config.ts +++ b/indexer/services/ender/src/config.ts @@ -6,6 +6,7 @@ import { parseSchema, baseConfigSchema, parseBoolean, + parseString, } from '@dydxprotocol-indexer/base'; import { kafkaConfigSchema, @@ -23,6 +24,13 @@ export const configSchema = { SEND_WEBSOCKET_MESSAGES: parseBoolean({ default: true, }), + // Config var to skip processing stateful order events with specific uuids. + // Order UUIDs should be in a string delimited by commas. + // Only set if invalid order events are being included in a block and preventing ender from + // progressing. + SKIP_STATEFUL_ORDER_UUIDS: parseString({ + default: '', + }), }; export default parseSchema(configSchema); diff --git a/indexer/services/ender/src/lib/block-processor.ts b/indexer/services/ender/src/lib/block-processor.ts index f3ab8ba1e3..6051425b8e 100644 --- a/indexer/services/ender/src/lib/block-processor.ts +++ b/indexer/services/ender/src/lib/block-processor.ts @@ -37,7 +37,7 @@ import { KafkaPublisher } from './kafka-publisher'; import { SyncHandlers, SYNCHRONOUS_SUBTYPES } from './sync-handlers'; import { ConsolidatedKafkaEvent, - DydxIndexerSubtypes, EventMessage, EventProtoWithTypeAndVersion, GroupedEvents, + DydxIndexerSubtypes, EventMessage, EventProtoWithTypeAndVersion, GroupedEvents, SKIPPED_EVENT_SUBTYPE, } from './types'; const TXN_EVENT_SUBTYPE_VERSION_TO_VALIDATOR_MAPPING: Record = { @@ -216,12 +216,28 @@ export class BlockProcessor { ); validator.validate(); this.sqlEventPromises[eventProto.blockEventIndex] = validator.getEventForBlockProcessor(); - const handlers: Handler[] = validator.createHandlers( + let handlers: Handler[] = validator.createHandlers( eventProto.indexerTendermintEvent, this.txId, this.messageReceivedTimestamp, ); + if (validator.shouldExcludeEvent()) { + // If the event should be excluded from being processed, set the subtype to a special value + // for skipped events. + this.block.events[eventProto.blockEventIndex] = { + ...this.block.events[eventProto.blockEventIndex], + subtype: SKIPPED_EVENT_SUBTYPE, + }; + // Set handlers to empty array if event is to be skipped. + handlers = []; + logger.info({ + at: 'onMessage#shouldExcludeEvent', + message: 'Excluded event from processing', + eventProto, + }); + } + _.map(handlers, (handler: Handler) => { if (SYNCHRONOUS_SUBTYPES.includes(eventProto.type as DydxIndexerSubtypes)) { this.syncHandlers.addHandler(eventProto.type, handler); diff --git a/indexer/services/ender/src/lib/types.ts b/indexer/services/ender/src/lib/types.ts index 3150c464fc..b13797bf63 100644 --- a/indexer/services/ender/src/lib/types.ts +++ b/indexer/services/ender/src/lib/types.ts @@ -61,6 +61,8 @@ export enum DydxIndexerSubtypes { UPSERT_VAULT = 'upsert_vault', } +export const SKIPPED_EVENT_SUBTYPE = 'skipped_event'; + // Generic interface used for creating the Handler objects // eslint-disable-next-line @typescript-eslint/no-explicit-any export type EventMessage = any; diff --git a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql index 1a6235d850..8d75ea9fe3 100644 --- a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql +++ b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_ordered_handlers.sql @@ -69,6 +69,8 @@ BEGIN rval[i] = dydx_funding_handler(block_height, block_time, event_data, event_index, transaction_index); WHEN '"upsert_vault"'::jsonb THEN rval[i] = dydx_vault_upsert_handler(block_time, event_data); + WHEN '"skipped_event"'::jsonb THEN + rval[i] = jsonb_build_object(); ELSE NULL; END CASE; diff --git a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql index 9c985c79ca..4b82d32b5a 100644 --- a/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql +++ b/indexer/services/ender/src/scripts/handlers/dydx_block_processor_unordered_handlers.sql @@ -67,6 +67,8 @@ BEGIN rval[i] = dydx_trading_rewards_handler(block_height, block_time, event_data, event_index, transaction_index, jsonb_array_element_text(block->'txHashes', transaction_index)); WHEN '"register_affiliate"'::jsonb THEN rval[i] = dydx_register_affiliate_handler(block_height, event_data); + WHEN '"skipped_event"'::jsonb THEN + rval[i] = jsonb_build_object(); ELSE NULL; END CASE; diff --git a/indexer/services/ender/src/validators/stateful-order-validator.ts b/indexer/services/ender/src/validators/stateful-order-validator.ts index 8507fb6ca7..4561c3525b 100644 --- a/indexer/services/ender/src/validators/stateful-order-validator.ts +++ b/indexer/services/ender/src/validators/stateful-order-validator.ts @@ -1,3 +1,4 @@ +import { OrderTable } from '@dydxprotocol-indexer/postgres'; import { ORDER_FLAG_CONDITIONAL, ORDER_FLAG_LONG_TERM } from '@dydxprotocol-indexer/v4-proto-parser'; import { IndexerTendermintEvent, @@ -13,6 +14,7 @@ import { } from '@dydxprotocol-indexer/v4-protos'; import Long from 'long'; +import config from '../config'; import { Handler, HandlerInitializer } from '../handlers/handler'; import { ConditionalOrderPlacementHandler } from '../handlers/stateful-order/conditional-order-placement-handler'; import { ConditionalOrderTriggeredHandler } from '../handlers/stateful-order/conditional-order-triggered-handler'; @@ -233,4 +235,40 @@ export class StatefulOrderValidator extends Validator { return [handler]; } + + /** + * Skip order uuids in config env var. + */ + public shouldExcludeEvent(): boolean { + const orderUUIDsToSkip: string[] = config.SKIP_STATEFUL_ORDER_UUIDS.split(','); + if (orderUUIDsToSkip.length === 0) { + return false; + } + + const orderUUIDStoSkipSet: Set = new Set(orderUUIDsToSkip); + if (orderUUIDStoSkipSet.has(this.getOrderUUId())) { + return true; + } + + return false; + } + + /** + * Gets order uuid for the event being validated. + * Assumes events are valid. + */ + private getOrderUUId(): string { + if (this.event.orderPlace !== undefined) { + return OrderTable.orderIdToUuid(this.event.orderPlace.order!.orderId!); + } else if (this.event.orderRemoval !== undefined) { + return OrderTable.orderIdToUuid(this.event.orderRemoval.removedOrderId!); + } else if (this.event.conditionalOrderPlacement !== undefined) { + return OrderTable.orderIdToUuid(this.event.conditionalOrderPlacement.order!.orderId!); + } else if (this.event.conditionalOrderTriggered !== undefined) { + return OrderTable.orderIdToUuid(this.event.conditionalOrderTriggered.triggeredOrderId!); + } else if (this.event.longTermOrderPlacement !== undefined) { + return OrderTable.orderIdToUuid(this.event.longTermOrderPlacement.order!.orderId!); + } + return ''; + } } diff --git a/indexer/services/ender/src/validators/validator.ts b/indexer/services/ender/src/validators/validator.ts index f9c0a53662..e3235fb96f 100644 --- a/indexer/services/ender/src/validators/validator.ts +++ b/indexer/services/ender/src/validators/validator.ts @@ -58,4 +58,13 @@ export abstract class Validator { txId: number, messageReceivedTimestamp: string, ): Handler[]; + + /** + * Allows aribtrary logic to exclude events from being processed. + * Defaults to no events being excluded. + * @returns + */ + public shouldExcludeEvent(): boolean { + return false; + } } From bba2a123a6b67c222c231895127397d9feea35fa Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 18:15:09 -0400 Subject: [PATCH 30/41] add wallet when transfer to subaccount (backport #2519) (#2520) Co-authored-by: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> --- .../__tests__/handlers/transfer-handler.test.ts | 15 +++++++++++---- .../scripts/handlers/dydx_transfer_handler.sql | 5 +++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts b/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts index 4f7f959070..cd8c7e8005 100644 --- a/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts @@ -311,7 +311,7 @@ describe('transferHandler', () => { expect(wallet).toEqual(defaultWallet); }); - it('creates new deposit for previously non-existent subaccount', async () => { + it('creates new deposit for previously non-existent subaccount (also non-existent recipient wallet)', async () => { const transactionIndex: number = 0; const depositEvent: TransferEventV1 = defaultDepositEvent; @@ -348,16 +348,23 @@ describe('transferHandler', () => { newTransfer, asset, ); - // Confirm the wallet was created - const wallet: WalletFromDatabase | undefined = await WalletTable.findById( + // Confirm the wallet was created for the sender and recipient + const walletSender: WalletFromDatabase | undefined = await WalletTable.findById( defaultWalletAddress, ); + const walletRecipient: WalletFromDatabase | undefined = await WalletTable.findById( + defaultDepositEvent.recipient!.subaccountId!.owner, + ); const newRecipientSubaccount: SubaccountFromDatabase | undefined = await SubaccountTable.findById( defaultRecipientSubaccountId, ); expect(newRecipientSubaccount).toBeDefined(); - expect(wallet).toEqual(defaultWallet); + expect(walletSender).toEqual(defaultWallet); + expect(walletRecipient).toEqual({ + ...defaultWallet, + address: defaultDepositEvent.recipient!.subaccountId!.owner, + }); }); it('creates new withdrawal for existing subaccount', async () => { diff --git a/indexer/services/ender/src/scripts/handlers/dydx_transfer_handler.sql b/indexer/services/ender/src/scripts/handlers/dydx_transfer_handler.sql index 396a0075f8..8e9b251548 100644 --- a/indexer/services/ender/src/scripts/handlers/dydx_transfer_handler.sql +++ b/indexer/services/ender/src/scripts/handlers/dydx_transfer_handler.sql @@ -46,6 +46,11 @@ BEGIN SET "updatedAtHeight" = recipient_subaccount_record."updatedAtHeight", "updatedAt" = recipient_subaccount_record."updatedAt"; + + recipient_wallet_record."address" = event_data->'recipient'->'subaccountId'->>'owner'; + recipient_wallet_record."totalTradingRewards" = '0'; + recipient_wallet_record."totalVolume" = '0'; + INSERT INTO wallets VALUES (recipient_wallet_record.*) ON CONFLICT DO NOTHING; END IF; IF event_data->'sender'->'subaccountId' IS NOT NULL THEN From daddb134173cf236efc4b4a4978d03a472bd3ca6 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 19:31:15 -0400 Subject: [PATCH 31/41] update migration to stop excessive consumption of computation (backport #2521) (#2522) Co-authored-by: Mohammed Affan --- ...1410_change_fills_affiliaterevshare_type.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts index dd61af5d3b..96d7b48083 100644 --- a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts @@ -2,15 +2,21 @@ import * as Knex from 'knex'; // No data has been stored added at time of commit export async function up(knex: Knex): Promise { - return knex.schema.alterTable('fills', (table) => { - // decimal('columnName') has is 8,2 precision and scale - // decimal('columnName', null) has variable precision and scale - table.decimal('affiliateRevShare', null).notNullable().defaultTo(0).alter(); + // decimal('columnName') has is 8,2 precision and scale + // decimal('columnName', null) has variable precision and scale + await knex.schema.alterTable('fills', (table) => { + table.dropColumn('affiliateRevShare'); + }); + await knex.schema.alterTable('fills', (table) => { + table.decimal('affiliateRevShare', null).notNullable().defaultTo(0); }); } export async function down(knex: Knex): Promise { - return knex.schema.alterTable('fills', (table) => { - table.string('affiliateRevShare').notNullable().defaultTo('0').alter(); + await knex.schema.alterTable('fills', (table) => { + table.dropColumn('affiliateRevShare'); + }); + await knex.schema.alterTable('fills', (table) => { + table.string('affiliateRevShare').notNullable().defaultTo('0'); }); } From bb809c27b74c4e81c11b6ee674da5901aba7e892 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:00:25 -0400 Subject: [PATCH 32/41] Get funding index maps for vault positions in chunks. (backport #2525) (#2527) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../src/stores/funding-index-updates-table.ts | 2 + indexer/services/comlink/src/config.ts | 1 + .../controllers/api/v4/vault-controller.ts | 104 +++++++++++++++++- 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/indexer/packages/postgres/src/stores/funding-index-updates-table.ts b/indexer/packages/postgres/src/stores/funding-index-updates-table.ts index 8ef55a3537..785431ff93 100644 --- a/indexer/packages/postgres/src/stores/funding-index-updates-table.ts +++ b/indexer/packages/postgres/src/stores/funding-index-updates-table.ts @@ -242,6 +242,7 @@ export async function findFundingIndexMaps( .sort(); // Get the min height to limit the search to blocks 4 hours or before the min height. const minHeight: number = heightNumbers[0]; + const maxheight: number = heightNumbers[heightNumbers.length - 1]; const result: { rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[], @@ -255,6 +256,7 @@ export async function findFundingIndexMaps( unnest(ARRAY[${heightNumbers.join(',')}]) AS "searchHeight" WHERE "effectiveAtHeight" > ${Big(minHeight).minus(FOUR_HOUR_OF_BLOCKS).toFixed()} AND + "effectiveAtHeight" <= ${Big(maxheight)} AND "effectiveAtHeight" <= "searchHeight" ORDER BY "perpetualId", diff --git a/indexer/services/comlink/src/config.ts b/indexer/services/comlink/src/config.ts index 743bae1dd4..bfb702abcc 100644 --- a/indexer/services/comlink/src/config.ts +++ b/indexer/services/comlink/src/config.ts @@ -63,6 +63,7 @@ export const configSchema = { VAULT_PNL_HISTORY_DAYS: parseInteger({ default: 90 }), VAULT_PNL_HISTORY_HOURS: parseInteger({ default: 72 }), VAULT_LATEST_PNL_TICK_WINDOW_HOURS: parseInteger({ default: 1 }), + VAULT_FETCH_FUNDING_INDEX_BLOCK_WINDOWS: parseInteger({ default: 250_000 }), }; //////////////////////////////////////////////////////////////////////////////// diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 9480b096f1..e4855b016a 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -1,4 +1,4 @@ -import { stats } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { PnlTicksFromDatabase, PnlTicksTable, @@ -71,7 +71,14 @@ class VaultController extends Controller { async getMegavaultHistoricalPnl( @Query() resolution?: PnlTickInterval, ): Promise { + const start: number = Date.now(); const vaultSubaccounts: VaultMapping = await getVaultMapping(); + stats.timing( + `${config.SERVICE_NAME}.${controllerName}.fetch_vaults.timing`, + Date.now() - start, + ); + + const startTicksPositions: number = Date.now(); const vaultSubaccountIdsWithMainSubaccount: string[] = _ .keys(vaultSubaccounts) .concat([MEGAVAULT_SUBACCOUNT_ID]); @@ -94,6 +101,10 @@ class VaultController extends Controller { getMainSubaccountEquity(), getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount), ]); + stats.timing( + `${config.SERVICE_NAME}.${controllerName}.fetch_ticks_positions_equity.timing`, + Date.now() - startTicksPositions, + ); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(vaultPnlTicks); @@ -324,6 +335,7 @@ async function getVaultSubaccountPnlTicks( async function getVaultPositions( vaultSubaccounts: VaultMapping, ): Promise> { + const start: number = Date.now(); const vaultSubaccountIds: string[] = _.keys(vaultSubaccounts); if (vaultSubaccountIds.length === 0) { return new Map(); @@ -374,7 +386,12 @@ async function getVaultPositions( ), BlockTable.getLatest(), ]); + stats.timing( + `${config.SERVICE_NAME}.${controllerName}.positions.fetch_subaccounts_positions.timing`, + Date.now() - start, + ); + const startFunding: number = Date.now(); const updatedAtHeights: string[] = _(subaccounts).map('updatedAtHeight').uniq().value(); const [ latestFundingIndexMap, @@ -387,11 +404,13 @@ async function getVaultPositions( .findFundingIndexMap( latestBlock.blockHeight, ), - FundingIndexUpdatesTable - .findFundingIndexMaps( - updatedAtHeights, - ), + getFundingIndexMapsChunked(updatedAtHeights), ]); + stats.timing( + `${config.SERVICE_NAME}.${controllerName}.positions.fetch_funding.timing`, + Date.now() - startFunding, + ); + const assetPositionsBySubaccount: { [subaccountId: string]: AssetPositionFromDatabase[] } = _.groupBy( assetPositions, @@ -557,13 +576,68 @@ function getResolution(resolution: PnlTickInterval = PnlTickInterval.day): PnlTi return resolution; } +/** + * Gets funding index maps in a chunked fashion to reduce database load and aggregates into a + * a map of funding index maps. + * @param updatedAtHeights + * @returns + */ +async function getFundingIndexMapsChunked( + updatedAtHeights: string[], +): Promise<{[blockHeight: string]: FundingIndexMap}> { + const updatedAtHeightsNum: number[] = updatedAtHeights.map((height: string): number => { + return parseInt(height, 10); + }).sort(); + const aggregateFundingIndexMaps: {[blockHeight: string]: FundingIndexMap} = {}; + await Promise.all(getHeightWindows(updatedAtHeightsNum).map( + async (heightWindow: number[]): Promise => { + const fundingIndexMaps: {[blockHeight: string]: FundingIndexMap} = await + FundingIndexUpdatesTable + .findFundingIndexMaps( + heightWindow.map((heightNum: number): string => { return heightNum.toString(); }), + ); + for (const height of _.keys(fundingIndexMaps)) { + aggregateFundingIndexMaps[height] = fundingIndexMaps[height]; + } + })); + return aggregateFundingIndexMaps; +} + +/** + * Separates an array of heights into a chunks based on a window size. Each chunk should only + * contain heights within a certain number of blocks of each other. + * @param heights + * @returns + */ +function getHeightWindows( + heights: number[], +): number[][] { + if (heights.length === 0) { + return []; + } + const windows: number[][] = []; + let windowStart: number = heights[0]; + let currentWindow: number[] = []; + for (const height of heights) { + if (height - windowStart < config.VAULT_FETCH_FUNDING_INDEX_BLOCK_WINDOWS) { + currentWindow.push(height); + } else { + windows.push(currentWindow); + currentWindow = [height]; + windowStart = height; + } + } + windows.push(currentWindow); + return windows; +} + async function getVaultMapping(): Promise { const vaults: VaultFromDatabase[] = await VaultTable.findAll( {}, [], {}, ); - return _.zipObject( + const vaultMapping: VaultMapping = _.zipObject( vaults.map((vault: VaultFromDatabase): string => { return SubaccountTable.uuid(vault.address, 0); }), @@ -571,6 +645,24 @@ async function getVaultMapping(): Promise { return vault.clobPairId; }), ); + const validVaultMapping: VaultMapping = {}; + for (const subaccountId of _.keys(vaultMapping)) { + const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher + .getPerpetualMarketFromClobPairId( + vaultMapping[subaccountId], + ); + if (perpetual === undefined) { + logger.warning({ + at: 'VaultController#getVaultPositions', + message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` + + 'perpetual market.', + subaccountId, + }); + continue; + } + validVaultMapping[subaccountId] = vaultMapping[subaccountId]; + } + return vaultMapping; } export default router; From 32cbd9d9a5ab0c5f03d7d29ab6c1387f23919ed5 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:16:35 -0400 Subject: [PATCH 33/41] Vulcan topic to 210 partitions (backport #2528) (#2530) Co-authored-by: roy-dydx <133032749+roy-dydx@users.noreply.github.com> --- indexer/services/bazooka/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/services/bazooka/src/index.ts b/indexer/services/bazooka/src/index.ts index 8329bd69a0..efd30af255 100644 --- a/indexer/services/bazooka/src/index.ts +++ b/indexer/services/bazooka/src/index.ts @@ -25,7 +25,7 @@ const DEFAULT_NUM_REPLICAS: number = 3; const KAFKA_TOPICS_TO_PARTITIONS: { [key in KafkaTopics]: number } = { [KafkaTopics.TO_ENDER]: 1, - [KafkaTopics.TO_VULCAN]: 150, + [KafkaTopics.TO_VULCAN]: 210, [KafkaTopics.TO_WEBSOCKETS_ORDERBOOKS]: 1, [KafkaTopics.TO_WEBSOCKETS_SUBACCOUNTS]: 3, [KafkaTopics.TO_WEBSOCKETS_TRADES]: 1, From 38a2d36d80f344bb197a4f9637c664be68175b31 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:36:44 -0400 Subject: [PATCH 34/41] Fix typo and add test for invalid vaults. (backport #2535) (#2536) Co-authored-by: vincentwschau <99756290+vincentwschau@users.noreply.github.com> --- .../__tests__/controllers/api/v4/vault-controller.test.ts | 7 ++++++- .../comlink/src/controllers/api/v4/vault-controller.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 582cb25361..1686311cc1 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -451,7 +451,7 @@ describe('vault-controller#V4', () => { }); }); - it('Get /megavault/positions with 2 vault subaccount, 1 with no perpetual', async () => { + it('Get /megavault/positions with 2 vault subaccount, 1 with no perpetual, 1 invalid', async () => { await Promise.all([ VaultTable.create({ ...testConstants.defaultVault, @@ -463,6 +463,11 @@ describe('vault-controller#V4', () => { address: testConstants.vaultAddress, clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, }), + VaultTable.create({ + ...testConstants.defaultVault, + address: 'invalid', + clobPairId: '999', + }), ]); const response: request.Response = await sendRequest({ type: RequestMethod.GET, diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index e4855b016a..10342c28e4 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -662,7 +662,7 @@ async function getVaultMapping(): Promise { } validVaultMapping[subaccountId] = vaultMapping[subaccountId]; } - return vaultMapping; + return validVaultMapping; } export default router; From a207ab160d11e9bdfb12cf54f4750e5b02c691a4 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Thu, 24 Oct 2024 23:46:49 -0400 Subject: [PATCH 35/41] De-dupe and filter out aggregated ticks with missing data. --- .../indexer-build-and-push-testnet.yml | 1 + indexer/pnpm-lock.yaml | 6 ++ .../comlink/__tests__/lib/helpers.test.ts | 64 ++++++++++++++----- indexer/services/comlink/package.json | 3 +- .../api/v4/historical-pnl-controller.ts | 6 +- .../controllers/api/v4/vault-controller.ts | 51 +++++++++++---- indexer/services/comlink/src/lib/helpers.ts | 20 +++++- indexer/services/comlink/src/types.ts | 5 ++ 8 files changed, 123 insertions(+), 33 deletions(-) diff --git a/.github/workflows/indexer-build-and-push-testnet.yml b/.github/workflows/indexer-build-and-push-testnet.yml index 3cee1bf285..34d78ccec6 100644 --- a/.github/workflows/indexer-build-and-push-testnet.yml +++ b/.github/workflows/indexer-build-and-push-testnet.yml @@ -3,6 +3,7 @@ name: Indexer Build & Push Images to AWS ECR for Testnet Branch on: # yamllint disable-line rule:truthy push: branches: + - vincentc/dedup-filter-ticks - main - 'release/indexer/v[0-9]+.[0-9]+.x' # e.g. release/indexer/v0.1.x - 'release/indexer/v[0-9]+.x' # e.g. release/indexer/v1.x diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 3f7362293d..0b961578cc 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -471,6 +471,7 @@ importers: '@types/supertest': ^2.0.12 '@types/swagger-ui-express': ^4.1.3 big.js: ^6.2.1 + binary-searching: ^2.0.5 body-parser: ^1.20.0 concurrently: ^7.6.0 cors: ^2.8.5 @@ -510,6 +511,7 @@ importers: '@keplr-wallet/cosmos': 0.12.122 '@tsoa/runtime': 5.0.0 big.js: 6.2.1 + binary-searching: 2.0.5 body-parser: 1.20.0 cors: 2.8.5 dd-trace: 3.32.1 @@ -7939,6 +7941,10 @@ packages: resolution: {integrity: sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw==} dev: false + /binary-searching/2.0.5: + resolution: {integrity: sha512-v4N2l3RxL+m4zDxyxz3Ne2aTmiPn8ZUpKFpdPtO+ItW1NcTCXA7JeHG5GMBSvoKSkQZ9ycS+EouDVxYB9ufKWA==} + dev: false + /bindings/1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: diff --git a/indexer/services/comlink/__tests__/lib/helpers.test.ts b/indexer/services/comlink/__tests__/lib/helpers.test.ts index d814827235..939ab78472 100644 --- a/indexer/services/comlink/__tests__/lib/helpers.test.ts +++ b/indexer/services/comlink/__tests__/lib/helpers.test.ts @@ -58,7 +58,7 @@ import { defaultTendermintEventId2, defaultTendermintEventId3, } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; -import { AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject } from '../../src/types'; +import { AggregatedPnlTick, AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject } from '../../src/types'; import { ZERO, ZERO_USDC_POSITION } from '../../src/lib/constants'; import { DateTime } from 'luxon'; @@ -844,11 +844,16 @@ describe('helpers', () => { ), }; - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks([pnlTick]); + const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks([pnlTick]); expect( aggregatedPnlTicks, ).toEqual( - [expect.objectContaining({ ...testConstants.defaultPnlTick })], + [expect.objectContaining( + { + pnlTick: expect.objectContaining(testConstants.defaultPnlTick), + numTicks: 1, + }, + )], ); }); @@ -862,6 +867,7 @@ describe('helpers', () => { }; const pnlTick2: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, + subaccountId: testConstants.defaultSubaccountId2, id: PnlTicksTable.uuid( testConstants.defaultSubaccountId2, testConstants.defaultPnlTick.createdAt, @@ -883,8 +889,9 @@ describe('helpers', () => { const blockTime3: string = DateTime.fromISO(pnlTick.createdAt).plus({ minute: 61 }).toISO(); const pnlTick4: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, + subaccountId: testConstants.defaultSubaccountId2, id: PnlTicksTable.uuid( - testConstants.defaultPnlTick.subaccountId, + testConstants.defaultSubaccountId2, blockTime3, ), equity: '1', @@ -894,29 +901,52 @@ describe('helpers', () => { blockTime: blockTime3, createdAt: blockTime3, }; + const blockHeight4: string = '82'; + const blockTime4: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').plus({ minute: 63 }).toISO(); + // should be de-duped + const pnlTick5: PnlTicksFromDatabase = { + ...testConstants.defaultPnlTick, + subaccountId: testConstants.defaultSubaccountId2, + id: PnlTicksTable.uuid( + testConstants.defaultSubaccountId2, + blockTime4, + ), + equity: '1', + totalPnl: '2', + netTransfers: '3', + blockHeight: blockHeight4, + blockTime: blockTime4, + createdAt: blockTime4, + } - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks( + const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks( [pnlTick, pnlTick2, pnlTick3, pnlTick4], ); expect(aggregatedPnlTicks).toEqual( expect.arrayContaining([ // Combined pnl tick at initial hour expect.objectContaining({ - equity: (parseFloat(testConstants.defaultPnlTick.equity) + - parseFloat(pnlTick2.equity)).toString(), - totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) + - parseFloat(pnlTick2.totalPnl)).toString(), - netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) + - parseFloat(pnlTick2.netTransfers)).toString(), + pnlTick: expect.objectContaining({ + equity: (parseFloat(testConstants.defaultPnlTick.equity) + + parseFloat(pnlTick2.equity)).toString(), + totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) + + parseFloat(pnlTick2.totalPnl)).toString(), + netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) + + parseFloat(pnlTick2.netTransfers)).toString(), + }), + numTicks: 2, }), // Combined pnl tick at initial hour + 1 hour and initial hour + 1 hour, 1 minute expect.objectContaining({ - equity: (parseFloat(pnlTick3.equity) + - parseFloat(pnlTick4.equity)).toString(), - totalPnl: (parseFloat(pnlTick3.totalPnl) + - parseFloat(pnlTick4.totalPnl)).toString(), - netTransfers: (parseFloat(pnlTick3.netTransfers) + - parseFloat(pnlTick4.netTransfers)).toString(), + pnlTick: expect.objectContaining({ + equity: (parseFloat(pnlTick3.equity) + + parseFloat(pnlTick4.equity)).toString(), + totalPnl: (parseFloat(pnlTick3.totalPnl) + + parseFloat(pnlTick4.totalPnl)).toString(), + netTransfers: (parseFloat(pnlTick3.netTransfers) + + parseFloat(pnlTick4.netTransfers)).toString(), + }), + numTicks: 2, }), ]), ); diff --git a/indexer/services/comlink/package.json b/indexer/services/comlink/package.json index 29ea1def60..b4ac31ae38 100644 --- a/indexer/services/comlink/package.json +++ b/indexer/services/comlink/package.json @@ -28,14 +28,15 @@ "@cosmjs/encoding": "^0.32.3", "@dydxprotocol-indexer/base": "workspace:^0.0.1", "@dydxprotocol-indexer/compliance": "workspace:^0.0.1", + "@dydxprotocol-indexer/notifications": "workspace:^0.0.1", "@dydxprotocol-indexer/postgres": "workspace:^0.0.1", "@dydxprotocol-indexer/redis": "workspace:^0.0.1", "@dydxprotocol-indexer/v4-proto-parser": "workspace:^0.0.1", "@dydxprotocol-indexer/v4-protos": "workspace:^0.0.1", "@keplr-wallet/cosmos": "^0.12.122", - "@dydxprotocol-indexer/notifications": "workspace:^0.0.1", "@tsoa/runtime": "^5.0.0", "big.js": "^6.2.1", + "binary-searching": "^2.0.5", "body-parser": "^1.20.0", "cors": "^2.8.5", "dd-trace": "^3.32.1", diff --git a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts index 96a3131ea9..ff726fa6a5 100644 --- a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts @@ -31,6 +31,7 @@ import { handleValidationErrors } from '../../../request-helpers/error-handler'; import ExportResponseCodeStats from '../../../request-helpers/export-response-code-stats'; import { pnlTicksToResponseObject } from '../../../request-helpers/request-transformer'; import { PnlTicksRequest, HistoricalPnlResponse, ParentSubaccountPnlTicksRequest } from '../../../types'; +import _ from 'lodash'; const router: express.Router = express.Router(); const controllerName: string = 'historical-pnl-controller'; @@ -156,7 +157,10 @@ class HistoricalPnlController extends Controller { } // aggregate pnlTicks for all subaccounts grouped by blockHeight - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(pnlTicks); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = _.map( + aggregateHourlyPnlTicks(pnlTicks), + 'pnlTick', + ); return { historicalPnl: aggregatedPnlTicks.map( diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 10342c28e4..e0cc6a0f03 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -56,13 +56,15 @@ import { SubaccountResponseObject, MegavaultHistoricalPnlRequest, VaultsHistoricalPnlRequest, + AggregatedPnlTick, } from '../../../types'; +import bounds from 'binary-searching'; const router: express.Router = express.Router(); const controllerName: string = 'vault-controller'; interface VaultMapping { - [subaccountId: string]: string, + [subaccountId: string]: VaultFromDatabase, } @Route('vault/v1') @@ -99,7 +101,7 @@ class VaultController extends Controller { getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), - getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount), + getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)), ]); stats.timing( `${config.SERVICE_NAME}.${controllerName}.fetch_ticks_positions_equity.timing`, @@ -107,7 +109,10 @@ class VaultController extends Controller { ); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(vaultPnlTicks); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( + vaultPnlTicks, + _.values(vaultSubaccounts), + ); const currentEquity: string = Array.from(vaultPositions.values()) .map((position: VaultPosition): string => { @@ -154,7 +159,7 @@ class VaultController extends Controller { .mapValues((pnlTicks: PnlTicksFromDatabase[], subaccountId: string): VaultHistoricalPnl => { const market: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher .getPerpetualMarketFromClobPairId( - vaultSubaccounts[subaccountId], + vaultSubaccounts[subaccountId].clobPairId, ); if (market === undefined) { @@ -306,7 +311,8 @@ router.get( Date.now() - start, ); } - }); + } +); async function getVaultSubaccountPnlTicks( vaultSubaccountIds: string[], @@ -431,7 +437,7 @@ async function getVaultPositions( subaccountId: string, }[] = subaccounts.map((subaccount: SubaccountFromDatabase) => { const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); + .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id].clobPairId); if (perpetualMarket === undefined) { throw new Error( `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + @@ -522,6 +528,7 @@ function getPnlTicksWithCurrentTick( export async function getLatestPnlTick( vaultSubaccountIds: string[], + vaults: VaultFromDatabase[], ): Promise { const pnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.getPnlTicksAtIntervals( PnlTickInterval.hour, @@ -529,7 +536,10 @@ export async function getLatestPnlTick( vaultSubaccountIds, ); // Aggregate and get pnl tick closest to the hour - const aggregatedTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(pnlTicks); + const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( + pnlTicks, + vaults, + ); const filteredTicks: PnlTicksFromDatabase[] = filterOutIntervalTicks( aggregatedTicks, PnlTickInterval.hour, @@ -631,6 +641,27 @@ function getHeightWindows( return windows; } +function aggregateVaultPnlTicks( + vaultPnlTicks: PnlTicksFromDatabase[], + vaults: VaultFromDatabase[], +): PnlTicksFromDatabase[] { + // aggregate pnlTicks for all vault subaccounts grouped by blockHeight + const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks(vaultPnlTicks); + const vaultCreationTimes: DateTime[] = _.map(vaults, 'createdAt').map( + (createdAt: string) => { return DateTime.fromISO(createdAt); } + ).sort((a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }); + return aggregatedPnlTicks.filter((aggregatedTick: AggregatedPnlTick) => { + const numVaultsCreated: number = bounds.le( + vaultCreationTimes, + DateTime.fromISO(aggregatedTick.pnlTick.createdAt), + (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; } + ); + // Number of ticks should be strictly greater than number of vaults created before it + // as there should be a tick for the main vault subaccount. + return aggregatedTick.numTicks > numVaultsCreated; + }).map((aggregatedPnlTick: AggregatedPnlTick) => { return aggregatedPnlTick.pnlTick }); +} + async function getVaultMapping(): Promise { const vaults: VaultFromDatabase[] = await VaultTable.findAll( {}, @@ -641,15 +672,13 @@ async function getVaultMapping(): Promise { vaults.map((vault: VaultFromDatabase): string => { return SubaccountTable.uuid(vault.address, 0); }), - vaults.map((vault: VaultFromDatabase): string => { - return vault.clobPairId; - }), + vaults, ); const validVaultMapping: VaultMapping = {}; for (const subaccountId of _.keys(vaultMapping)) { const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher .getPerpetualMarketFromClobPairId( - vaultMapping[subaccountId], + vaultMapping[subaccountId].clobPairId, ); if (perpetual === undefined) { logger.warning({ diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index b3f5d3bf19..0dd923e792 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -37,6 +37,7 @@ import { subaccountToResponseObject, } from '../request-helpers/request-transformer'; import { + AggregatedPnlTick, AssetById, AssetPositionResponseObject, AssetPositionsMap, @@ -675,17 +676,23 @@ export function getSubaccountResponse( /** * Aggregates a list of PnL ticks, combining any PnL ticks for the same hour by summing * the equity, totalPnl, and net transfers. - * Returns a map of block height to the resulting PnL tick. + * Returns a map of aggregated pnl ticks and the number of ticks the aggreated tick is made up of. * @param pnlTicks * @returns */ export function aggregateHourlyPnlTicks( pnlTicks: PnlTicksFromDatabase[], -): PnlTicksFromDatabase[] { +): AggregatedPnlTick[] { const hourlyPnlTicks: Map = new Map(); + const hourlySubaccountIds: Map> = new Map(); for (const pnlTick of pnlTicks) { const truncatedTime: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').toISO(); if (hourlyPnlTicks.has(truncatedTime)) { + const subaccountIds: Set = hourlySubaccountIds.get(truncatedTime) as Set; + if (subaccountIds.has(pnlTick.subaccountId)) { + continue; + } + subaccountIds.add(pnlTick.subaccountId); const aggregatedTick: PnlTicksFromDatabase = hourlyPnlTicks.get( truncatedTime, ) as PnlTicksFromDatabase; @@ -700,9 +707,16 @@ export function aggregateHourlyPnlTicks( ).toString(), }, ); + hourlySubaccountIds.set(truncatedTime, subaccountIds); } else { hourlyPnlTicks.set(truncatedTime, pnlTick); + hourlySubaccountIds.set(truncatedTime, new Set([pnlTick.subaccountId])); } } - return Array.from(hourlyPnlTicks.values()); + return Array.from(hourlyPnlTicks.keys()).map((hour: string): AggregatedPnlTick => { + return { + pnlTick: hourlyPnlTicks.get(hour) as PnlTicksFromDatabase, + numTicks: (hourlySubaccountIds.get(hour) as Set).size, + } + }); } diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 0e0a57512c..fc83a92e29 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -264,6 +264,11 @@ export interface PnlTicksResponseObject { blockTime: IsoString, } +export interface AggregatedPnlTick{ + pnlTick: PnlTicksResponseObject, + numTicks: number, +} + /* ------- TRADE TYPES ------- */ export interface TradeResponse extends PaginationResponse { From d72898dbff7dbd7eb4f1968a8d01d532f1b1c274 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 25 Oct 2024 03:05:01 -0400 Subject: [PATCH 36/41] Add tests. --- .../controllers/api/v4/vault-controller.test.ts | 13 +++++++++---- .../services/comlink/__tests__/lib/helpers.test.ts | 2 +- .../src/controllers/api/v4/vault-controller.ts | 12 +++++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 1686311cc1..f89ce9b641 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -181,11 +181,12 @@ describe('vault-controller#V4', () => { [ 'hourly resolution', '?resolution=hour', - [1, undefined, 2, 3, 4], - [undefined, 6, 7, 8, 9], - [11, undefined, 12, 13, 14], + [1, 2, 3, 4], + [undefined, 7, 8, 9], + [11, 12, 13, 14], ], - ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s)', async ( + ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s), ' + + 'excludes tick with missing vault ticks', async ( _name: string, queryParam: string, expectedTicksIndex1: (number | undefined)[], @@ -202,11 +203,14 @@ describe('vault-controller#V4', () => { ...testConstants.defaultVault, address: testConstants.defaultAddress, clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + createdAt: twoDaysAgo.toISO(), }), + // Single tick for this vault will be excluded from result. VaultTable.create({ ...testConstants.defaultVault, address: testConstants.vaultAddress, clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, + createdAt: almostTwoDaysAgo.toISO(), }), AssetPositionTable.upsert({ ...testConstants.defaultAssetPosition, @@ -559,6 +563,7 @@ describe('vault-controller#V4', () => { ...testConstants.defaultPnlTick, subaccountId: testConstants.vaultSubaccountId, }), + // Invalid pnl tick to be excluded as only a single pnl tick but 2 pnl ticks should exist. PnlTicksTable.create({ ...testConstants.defaultPnlTick, subaccountId: testConstants.vaultSubaccountId, diff --git a/indexer/services/comlink/__tests__/lib/helpers.test.ts b/indexer/services/comlink/__tests__/lib/helpers.test.ts index 939ab78472..b7c0f0b663 100644 --- a/indexer/services/comlink/__tests__/lib/helpers.test.ts +++ b/indexer/services/comlink/__tests__/lib/helpers.test.ts @@ -857,7 +857,7 @@ describe('helpers', () => { ); }); - it('aggregates multiple pnl ticks same height', () => { + it('aggregates multiple pnl ticks same height and de-dupes ticks', () => { const pnlTick: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, id: PnlTicksTable.uuid( diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index e0cc6a0f03..721a4f09c5 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -641,6 +641,14 @@ function getHeightWindows( return windows; } +/** + * Aggregates vault pnl ticks per hour, filtering out pnl ticks made up of less ticks than expected. + * Expected number of pnl ticks is calculated from the number of vaults that were created before + * the pnl tick was created. + * @param vaultPnlTicks Pnl ticks to aggregate. + * @param vaults List of all valid vaults. + * @returns + */ function aggregateVaultPnlTicks( vaultPnlTicks: PnlTicksFromDatabase[], vaults: VaultFromDatabase[], @@ -651,11 +659,13 @@ function aggregateVaultPnlTicks( (createdAt: string) => { return DateTime.fromISO(createdAt); } ).sort((a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }); return aggregatedPnlTicks.filter((aggregatedTick: AggregatedPnlTick) => { + // Get number of vaults created before the pnl tick was created by binary-searching for the + // index of the pnl ticks createdAt in a sorted array of vault createdAt times. const numVaultsCreated: number = bounds.le( vaultCreationTimes, DateTime.fromISO(aggregatedTick.pnlTick.createdAt), (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; } - ); + ) + 1; // Number of ticks should be strictly greater than number of vaults created before it // as there should be a tick for the main vault subaccount. return aggregatedTick.numTicks > numVaultsCreated; From 7af7388642e6af69464c8584e95762ed8fb15d10 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:17:23 -0400 Subject: [PATCH 37/41] Update gh workflow. --- .../workflows/indexer-build-and-push-dev-staging.yml | 1 + .../services/comlink/__tests__/lib/helpers.test.ts | 10 ++++++---- .../controllers/api/v4/historical-pnl-controller.ts | 2 +- .../src/controllers/api/v4/vault-controller.ts | 12 ++++++------ indexer/services/comlink/src/lib/helpers.ts | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/indexer-build-and-push-dev-staging.yml b/.github/workflows/indexer-build-and-push-dev-staging.yml index 5a98b72552..3e536ca01f 100644 --- a/.github/workflows/indexer-build-and-push-dev-staging.yml +++ b/.github/workflows/indexer-build-and-push-dev-staging.yml @@ -3,6 +3,7 @@ name: Indexer Build & Push Images to AWS ECR for Dev / Staging branches on: # yamllint disable-line rule:truthy push: branches: + - vincentc/dedup-filter-ticks - main - 'release/indexer/v[0-9]+.[0-9]+.x' # e.g. release/indexer/v0.1.x - 'release/indexer/v[0-9]+.x' # e.g. release/indexer/v1.x diff --git a/indexer/services/comlink/__tests__/lib/helpers.test.ts b/indexer/services/comlink/__tests__/lib/helpers.test.ts index b7c0f0b663..03d169db35 100644 --- a/indexer/services/comlink/__tests__/lib/helpers.test.ts +++ b/indexer/services/comlink/__tests__/lib/helpers.test.ts @@ -58,7 +58,9 @@ import { defaultTendermintEventId2, defaultTendermintEventId3, } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; -import { AggregatedPnlTick, AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject } from '../../src/types'; +import { + AggregatedPnlTick, AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject, +} from '../../src/types'; import { ZERO, ZERO_USDC_POSITION } from '../../src/lib/constants'; import { DateTime } from 'luxon'; @@ -849,7 +851,7 @@ describe('helpers', () => { aggregatedPnlTicks, ).toEqual( [expect.objectContaining( - { + { pnlTick: expect.objectContaining(testConstants.defaultPnlTick), numTicks: 1, }, @@ -917,10 +919,10 @@ describe('helpers', () => { blockHeight: blockHeight4, blockTime: blockTime4, createdAt: blockTime4, - } + }; const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks( - [pnlTick, pnlTick2, pnlTick3, pnlTick4], + [pnlTick, pnlTick2, pnlTick3, pnlTick4, pnlTick5], ); expect(aggregatedPnlTicks).toEqual( expect.arrayContaining([ diff --git a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts index ff726fa6a5..a1682eec94 100644 --- a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts @@ -11,6 +11,7 @@ import { } from '@dydxprotocol-indexer/postgres'; import express from 'express'; import { matchedData } from 'express-validator'; +import _ from 'lodash'; import { Controller, Get, Query, Route, } from 'tsoa'; @@ -31,7 +32,6 @@ import { handleValidationErrors } from '../../../request-helpers/error-handler'; import ExportResponseCodeStats from '../../../request-helpers/export-response-code-stats'; import { pnlTicksToResponseObject } from '../../../request-helpers/request-transformer'; import { PnlTicksRequest, HistoricalPnlResponse, ParentSubaccountPnlTicksRequest } from '../../../types'; -import _ from 'lodash'; const router: express.Router = express.Router(); const controllerName: string = 'historical-pnl-controller'; diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 721a4f09c5..a7e84f1f6d 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -27,6 +27,7 @@ import { MEGAVAULT_SUBACCOUNT_ID, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; +import bounds from 'binary-searching'; import express from 'express'; import { checkSchema, matchedData } from 'express-validator'; import _ from 'lodash'; @@ -58,7 +59,6 @@ import { VaultsHistoricalPnlRequest, AggregatedPnlTick, } from '../../../types'; -import bounds from 'binary-searching'; const router: express.Router = express.Router(); const controllerName: string = 'vault-controller'; @@ -311,7 +311,7 @@ router.get( Date.now() - start, ); } - } + }, ); async function getVaultSubaccountPnlTicks( @@ -647,7 +647,7 @@ function getHeightWindows( * the pnl tick was created. * @param vaultPnlTicks Pnl ticks to aggregate. * @param vaults List of all valid vaults. - * @returns + * @returns */ function aggregateVaultPnlTicks( vaultPnlTicks: PnlTicksFromDatabase[], @@ -656,7 +656,7 @@ function aggregateVaultPnlTicks( // aggregate pnlTicks for all vault subaccounts grouped by blockHeight const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks(vaultPnlTicks); const vaultCreationTimes: DateTime[] = _.map(vaults, 'createdAt').map( - (createdAt: string) => { return DateTime.fromISO(createdAt); } + (createdAt: string) => { return DateTime.fromISO(createdAt); }, ).sort((a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }); return aggregatedPnlTicks.filter((aggregatedTick: AggregatedPnlTick) => { // Get number of vaults created before the pnl tick was created by binary-searching for the @@ -664,12 +664,12 @@ function aggregateVaultPnlTicks( const numVaultsCreated: number = bounds.le( vaultCreationTimes, DateTime.fromISO(aggregatedTick.pnlTick.createdAt), - (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; } + (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }, ) + 1; // Number of ticks should be strictly greater than number of vaults created before it // as there should be a tick for the main vault subaccount. return aggregatedTick.numTicks > numVaultsCreated; - }).map((aggregatedPnlTick: AggregatedPnlTick) => { return aggregatedPnlTick.pnlTick }); + }).map((aggregatedPnlTick: AggregatedPnlTick) => { return aggregatedPnlTick.pnlTick; }); } async function getVaultMapping(): Promise { diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index 0dd923e792..16d27bf624 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -717,6 +717,6 @@ export function aggregateHourlyPnlTicks( return { pnlTick: hourlyPnlTicks.get(hour) as PnlTicksFromDatabase, numTicks: (hourlySubaccountIds.get(hour) as Set).size, - } + }; }); } From eefff3e786c21678df9e0cd5bb575f5d13381fb4 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:49:49 -0400 Subject: [PATCH 38/41] Logging. --- .../services/comlink/src/controllers/api/v4/vault-controller.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index a7e84f1f6d..7904c093e8 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -658,6 +658,7 @@ function aggregateVaultPnlTicks( const vaultCreationTimes: DateTime[] = _.map(vaults, 'createdAt').map( (createdAt: string) => { return DateTime.fromISO(createdAt); }, ).sort((a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }); + console.log(JSON.stringify(vaultCreationTimes)); return aggregatedPnlTicks.filter((aggregatedTick: AggregatedPnlTick) => { // Get number of vaults created before the pnl tick was created by binary-searching for the // index of the pnl ticks createdAt in a sorted array of vault createdAt times. @@ -666,6 +667,7 @@ function aggregateVaultPnlTicks( DateTime.fromISO(aggregatedTick.pnlTick.createdAt), (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }, ) + 1; + console.log(`Num vaults created: ${numVaultsCreated}, num ticks: ${aggregatedTick.numTicks}, createdAt: ${aggregatedTick.pnlTick.createdAt}`) // Number of ticks should be strictly greater than number of vaults created before it // as there should be a tick for the main vault subaccount. return aggregatedTick.numTicks > numVaultsCreated; From 18f5b73d2f00e238ff6e3960bd42bb60f930d350 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:52:16 -0400 Subject: [PATCH 39/41] Off-by-one --- .../services/comlink/src/controllers/api/v4/vault-controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 7904c093e8..e59e3770c2 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -666,7 +666,7 @@ function aggregateVaultPnlTicks( vaultCreationTimes, DateTime.fromISO(aggregatedTick.pnlTick.createdAt), (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }, - ) + 1; + ); console.log(`Num vaults created: ${numVaultsCreated}, num ticks: ${aggregatedTick.numTicks}, createdAt: ${aggregatedTick.pnlTick.createdAt}`) // Number of ticks should be strictly greater than number of vaults created before it // as there should be a tick for the main vault subaccount. From 8ee7ab58304c33d3ded8c8841699583ccd8d466d Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 25 Oct 2024 21:05:11 -0400 Subject: [PATCH 40/41] Add correct logic to account for main vault subaccount. --- .../api/v4/vault-controller.test.ts | 6 +++ .../controllers/api/v4/vault-controller.ts | 46 ++++++++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index f89ce9b641..960e5c57b6 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -15,6 +15,7 @@ import { VaultTable, MEGAVAULT_MODULE_ADDRESS, MEGAVAULT_SUBACCOUNT_ID, + TransferTable, } from '@dydxprotocol-indexer/postgres'; import { RequestMethod, VaultHistoricalPnl } from '../../../../src/types'; import request from 'supertest'; @@ -216,6 +217,11 @@ describe('vault-controller#V4', () => { ...testConstants.defaultAssetPosition, subaccountId: MEGAVAULT_SUBACCOUNT_ID, }), + TransferTable.create({ + ...testConstants.defaultTransfer, + recipientSubaccountId: MEGAVAULT_SUBACCOUNT_ID, + createdAt: twoDaysAgo.toISO(), + }), ]); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks( diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index e59e3770c2..a249b50b23 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -25,6 +25,10 @@ import { VaultTable, VaultFromDatabase, MEGAVAULT_SUBACCOUNT_ID, + TransferFromDatabase, + TransferTable, + TransferColumns, + Ordering, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; import bounds from 'binary-searching'; @@ -90,18 +94,21 @@ class VaultController extends Controller { latestBlock, mainSubaccountEquity, latestPnlTick, + firstMainVaultTransferTimestamp, ] : [ PnlTicksFromDatabase[], Map, BlockFromDatabase, string, PnlTicksFromDatabase | undefined, + DateTime | undefined ] = await Promise.all([ getVaultSubaccountPnlTicks(vaultSubaccountIdsWithMainSubaccount, getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)), + getFirstMainVaultTransferDateTime(), ]); stats.timing( `${config.SERVICE_NAME}.${controllerName}.fetch_ticks_positions_equity.timing`, @@ -112,6 +119,7 @@ class VaultController extends Controller { const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( vaultPnlTicks, _.values(vaultSubaccounts), + firstMainVaultTransferTimestamp, ); const currentEquity: string = Array.from(vaultPositions.values()) @@ -641,24 +649,51 @@ function getHeightWindows( return windows; } +async function getFirstMainVaultTransferDateTime(): Promise { + const { results }: { + results: TransferFromDatabase[], + } = await TransferTable.findAllToOrFromSubaccountId( + { + subaccountId: [MEGAVAULT_SUBACCOUNT_ID], + limit: 1, + }, + [], + { + orderBy: [[TransferColumns.createdAt, Ordering.ASC]], + }, + ); + if (results.length === 0) { + return undefined; + } + return DateTime.fromISO(results[0].createdAt); +} + /** * Aggregates vault pnl ticks per hour, filtering out pnl ticks made up of less ticks than expected. * Expected number of pnl ticks is calculated from the number of vaults that were created before * the pnl tick was created. * @param vaultPnlTicks Pnl ticks to aggregate. * @param vaults List of all valid vaults. + * @param mainVaultCreatedAt Date time when the main vault was created or undefined if it does not + * exist yet. * @returns */ function aggregateVaultPnlTicks( vaultPnlTicks: PnlTicksFromDatabase[], vaults: VaultFromDatabase[], + mainVaultCreatedAt?: DateTime, ): PnlTicksFromDatabase[] { // aggregate pnlTicks for all vault subaccounts grouped by blockHeight const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks(vaultPnlTicks); const vaultCreationTimes: DateTime[] = _.map(vaults, 'createdAt').map( (createdAt: string) => { return DateTime.fromISO(createdAt); }, - ).sort((a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }); - console.log(JSON.stringify(vaultCreationTimes)); + ).concat( + mainVaultCreatedAt === undefined ? [] : [mainVaultCreatedAt], + ).sort( + (a: DateTime, b: DateTime) => { + return a.diff(b).milliseconds; + }, + ); return aggregatedPnlTicks.filter((aggregatedTick: AggregatedPnlTick) => { // Get number of vaults created before the pnl tick was created by binary-searching for the // index of the pnl ticks createdAt in a sorted array of vault createdAt times. @@ -667,10 +702,9 @@ function aggregateVaultPnlTicks( DateTime.fromISO(aggregatedTick.pnlTick.createdAt), (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }, ); - console.log(`Num vaults created: ${numVaultsCreated}, num ticks: ${aggregatedTick.numTicks}, createdAt: ${aggregatedTick.pnlTick.createdAt}`) - // Number of ticks should be strictly greater than number of vaults created before it - // as there should be a tick for the main vault subaccount. - return aggregatedTick.numTicks > numVaultsCreated; + // Number of ticks should be greater than number of vaults created before it as there should be + // a tick for the main vault subaccount. + return aggregatedTick.numTicks >= numVaultsCreated; }).map((aggregatedPnlTick: AggregatedPnlTick) => { return aggregatedPnlTick.pnlTick; }); } From 0a9cdcf7cdef1530198c5e1d1c6acf71e435981b Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:02:27 -0400 Subject: [PATCH 41/41] Remove gh workflow changes. --- .github/workflows/indexer-build-and-push-dev-staging.yml | 1 - .github/workflows/indexer-build-and-push-testnet.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/indexer-build-and-push-dev-staging.yml b/.github/workflows/indexer-build-and-push-dev-staging.yml index 3e536ca01f..5a98b72552 100644 --- a/.github/workflows/indexer-build-and-push-dev-staging.yml +++ b/.github/workflows/indexer-build-and-push-dev-staging.yml @@ -3,7 +3,6 @@ name: Indexer Build & Push Images to AWS ECR for Dev / Staging branches on: # yamllint disable-line rule:truthy push: branches: - - vincentc/dedup-filter-ticks - main - 'release/indexer/v[0-9]+.[0-9]+.x' # e.g. release/indexer/v0.1.x - 'release/indexer/v[0-9]+.x' # e.g. release/indexer/v1.x diff --git a/.github/workflows/indexer-build-and-push-testnet.yml b/.github/workflows/indexer-build-and-push-testnet.yml index 34d78ccec6..3cee1bf285 100644 --- a/.github/workflows/indexer-build-and-push-testnet.yml +++ b/.github/workflows/indexer-build-and-push-testnet.yml @@ -3,7 +3,6 @@ name: Indexer Build & Push Images to AWS ECR for Testnet Branch on: # yamllint disable-line rule:truthy push: branches: - - vincentc/dedup-filter-ticks - main - 'release/indexer/v[0-9]+.[0-9]+.x' # e.g. release/indexer/v0.1.x - 'release/indexer/v[0-9]+.x' # e.g. release/indexer/v1.x