diff --git a/indexer/pnpm-lock.yaml b/indexer/pnpm-lock.yaml index 8dce665aea..98acd5cf3f 100644 --- a/indexer/pnpm-lock.yaml +++ b/indexer/pnpm-lock.yaml @@ -471,6 +471,7 @@ importers: '@types/supertest': ^2.0.12 '@types/swagger-ui-express': ^4.1.3 big.js: ^6.2.1 + binary-searching: ^2.0.5 body-parser: ^1.20.0 concurrently: ^7.6.0 cors: ^2.8.5 @@ -510,6 +511,7 @@ importers: '@keplr-wallet/cosmos': 0.12.122 '@tsoa/runtime': 5.0.0 big.js: 6.2.1 + binary-searching: 2.0.5 body-parser: 1.20.0 cors: 2.8.5 dd-trace: 3.32.1 @@ -7939,6 +7941,10 @@ packages: resolution: {integrity: sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw==} dev: false + /binary-searching/2.0.5: + resolution: {integrity: sha512-v4N2l3RxL+m4zDxyxz3Ne2aTmiPn8ZUpKFpdPtO+ItW1NcTCXA7JeHG5GMBSvoKSkQZ9ycS+EouDVxYB9ufKWA==} + dev: false + /bindings/1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 1686311cc1..960e5c57b6 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -15,6 +15,7 @@ import { VaultTable, MEGAVAULT_MODULE_ADDRESS, MEGAVAULT_SUBACCOUNT_ID, + TransferTable, } from '@dydxprotocol-indexer/postgres'; import { RequestMethod, VaultHistoricalPnl } from '../../../../src/types'; import request from 'supertest'; @@ -181,11 +182,12 @@ describe('vault-controller#V4', () => { [ 'hourly resolution', '?resolution=hour', - [1, undefined, 2, 3, 4], - [undefined, 6, 7, 8, 9], - [11, undefined, 12, 13, 14], + [1, 2, 3, 4], + [undefined, 7, 8, 9], + [11, 12, 13, 14], ], - ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s)', async ( + ])('Get /megavault/historicalPnl with 2 vault subaccounts and main subaccount (%s), ' + + 'excludes tick with missing vault ticks', async ( _name: string, queryParam: string, expectedTicksIndex1: (number | undefined)[], @@ -202,16 +204,24 @@ describe('vault-controller#V4', () => { ...testConstants.defaultVault, address: testConstants.defaultAddress, clobPairId: testConstants.defaultPerpetualMarket.clobPairId, + createdAt: twoDaysAgo.toISO(), }), + // Single tick for this vault will be excluded from result. VaultTable.create({ ...testConstants.defaultVault, address: testConstants.vaultAddress, clobPairId: testConstants.defaultPerpetualMarket2.clobPairId, + createdAt: almostTwoDaysAgo.toISO(), }), AssetPositionTable.upsert({ ...testConstants.defaultAssetPosition, subaccountId: MEGAVAULT_SUBACCOUNT_ID, }), + TransferTable.create({ + ...testConstants.defaultTransfer, + recipientSubaccountId: MEGAVAULT_SUBACCOUNT_ID, + createdAt: twoDaysAgo.toISO(), + }), ]); const createdPnlTicks: PnlTicksFromDatabase[] = await createPnlTicks( @@ -559,6 +569,7 @@ describe('vault-controller#V4', () => { ...testConstants.defaultPnlTick, subaccountId: testConstants.vaultSubaccountId, }), + // Invalid pnl tick to be excluded as only a single pnl tick but 2 pnl ticks should exist. PnlTicksTable.create({ ...testConstants.defaultPnlTick, subaccountId: testConstants.vaultSubaccountId, diff --git a/indexer/services/comlink/__tests__/lib/helpers.test.ts b/indexer/services/comlink/__tests__/lib/helpers.test.ts index bd342e1754..03d169db35 100644 --- a/indexer/services/comlink/__tests__/lib/helpers.test.ts +++ b/indexer/services/comlink/__tests__/lib/helpers.test.ts @@ -58,7 +58,9 @@ import { defaultTendermintEventId2, defaultTendermintEventId3, } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; -import { AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject } from '../../src/types'; +import { + AggregatedPnlTick, AssetPositionsMap, PerpetualPositionWithFunding, SubaccountResponseObject, +} from '../../src/types'; import { ZERO, ZERO_USDC_POSITION } from '../../src/lib/constants'; import { DateTime } from 'luxon'; @@ -844,15 +846,20 @@ describe('helpers', () => { ), }; - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks([pnlTick]); + const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks([pnlTick]); expect( aggregatedPnlTicks, ).toEqual( - [expect.objectContaining({ ...testConstants.defaultPnlTick })], + [expect.objectContaining( + { + pnlTick: expect.objectContaining(testConstants.defaultPnlTick), + numTicks: 1, + }, + )], ); }); - it('aggregates multiple pnl ticks same height', () => { + it('aggregates multiple pnl ticks same height and de-dupes ticks', () => { const pnlTick: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, id: PnlTicksTable.uuid( @@ -862,13 +869,14 @@ describe('helpers', () => { }; const pnlTick2: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, + subaccountId: testConstants.defaultSubaccountId2, id: PnlTicksTable.uuid( testConstants.defaultSubaccountId2, testConstants.defaultPnlTick.createdAt, ), }; const blockHeight2: string = '80'; - const blockTime2: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').plus({ minute: 61 }).toISO(); + const blockTime2: string = DateTime.fromISO(pnlTick.createdAt).plus({ hour: 1 }).toISO(); const pnlTick3: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, id: PnlTicksTable.uuid( @@ -880,11 +888,12 @@ describe('helpers', () => { createdAt: blockTime2, }; const blockHeight3: string = '81'; - const blockTime3: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').plus({ minute: 62 }).toISO(); + const blockTime3: string = DateTime.fromISO(pnlTick.createdAt).plus({ minute: 61 }).toISO(); const pnlTick4: PnlTicksFromDatabase = { ...testConstants.defaultPnlTick, + subaccountId: testConstants.defaultSubaccountId2, id: PnlTicksTable.uuid( - testConstants.defaultPnlTick.subaccountId, + testConstants.defaultSubaccountId2, blockTime3, ), equity: '1', @@ -894,29 +903,52 @@ describe('helpers', () => { blockTime: blockTime3, createdAt: blockTime3, }; + const blockHeight4: string = '82'; + const blockTime4: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').plus({ minute: 63 }).toISO(); + // should be de-duped + const pnlTick5: PnlTicksFromDatabase = { + ...testConstants.defaultPnlTick, + subaccountId: testConstants.defaultSubaccountId2, + id: PnlTicksTable.uuid( + testConstants.defaultSubaccountId2, + blockTime4, + ), + equity: '1', + totalPnl: '2', + netTransfers: '3', + blockHeight: blockHeight4, + blockTime: blockTime4, + createdAt: blockTime4, + }; - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks( - [pnlTick, pnlTick2, pnlTick3, pnlTick4], + const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks( + [pnlTick, pnlTick2, pnlTick3, pnlTick4, pnlTick5], ); expect(aggregatedPnlTicks).toEqual( expect.arrayContaining([ // Combined pnl tick at initial hour expect.objectContaining({ - equity: (parseFloat(testConstants.defaultPnlTick.equity) + - parseFloat(pnlTick2.equity)).toString(), - totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) + - parseFloat(pnlTick2.totalPnl)).toString(), - netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) + - parseFloat(pnlTick2.netTransfers)).toString(), + pnlTick: expect.objectContaining({ + equity: (parseFloat(testConstants.defaultPnlTick.equity) + + parseFloat(pnlTick2.equity)).toString(), + totalPnl: (parseFloat(testConstants.defaultPnlTick.totalPnl) + + parseFloat(pnlTick2.totalPnl)).toString(), + netTransfers: (parseFloat(testConstants.defaultPnlTick.netTransfers) + + parseFloat(pnlTick2.netTransfers)).toString(), + }), + numTicks: 2, }), // Combined pnl tick at initial hour + 1 hour and initial hour + 1 hour, 1 minute expect.objectContaining({ - equity: (parseFloat(pnlTick3.equity) + - parseFloat(pnlTick4.equity)).toString(), - totalPnl: (parseFloat(pnlTick3.totalPnl) + - parseFloat(pnlTick4.totalPnl)).toString(), - netTransfers: (parseFloat(pnlTick3.netTransfers) + - parseFloat(pnlTick4.netTransfers)).toString(), + pnlTick: expect.objectContaining({ + equity: (parseFloat(pnlTick3.equity) + + parseFloat(pnlTick4.equity)).toString(), + totalPnl: (parseFloat(pnlTick3.totalPnl) + + parseFloat(pnlTick4.totalPnl)).toString(), + netTransfers: (parseFloat(pnlTick3.netTransfers) + + parseFloat(pnlTick4.netTransfers)).toString(), + }), + numTicks: 2, }), ]), ); diff --git a/indexer/services/comlink/package.json b/indexer/services/comlink/package.json index 5217b59b6e..448ac57866 100644 --- a/indexer/services/comlink/package.json +++ b/indexer/services/comlink/package.json @@ -28,14 +28,15 @@ "@cosmjs/encoding": "^0.32.3", "@dydxprotocol-indexer/base": "workspace:^0.0.1", "@dydxprotocol-indexer/compliance": "workspace:^0.0.1", + "@dydxprotocol-indexer/notifications": "workspace:^0.0.1", "@dydxprotocol-indexer/postgres": "workspace:^0.0.1", "@dydxprotocol-indexer/redis": "workspace:^0.0.1", "@dydxprotocol-indexer/v4-proto-parser": "workspace:^0.0.1", "@dydxprotocol-indexer/v4-protos": "workspace:^0.0.1", "@keplr-wallet/cosmos": "^0.12.122", - "@dydxprotocol-indexer/notifications": "workspace:^0.0.1", "@tsoa/runtime": "^5.0.0", "big.js": "^6.2.1", + "binary-searching": "^2.0.5", "body-parser": "^1.20.0", "cors": "^2.8.5", "dd-trace": "^3.32.1", diff --git a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts index 96a3131ea9..a1682eec94 100644 --- a/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/historical-pnl-controller.ts @@ -11,6 +11,7 @@ import { } from '@dydxprotocol-indexer/postgres'; import express from 'express'; import { matchedData } from 'express-validator'; +import _ from 'lodash'; import { Controller, Get, Query, Route, } from 'tsoa'; @@ -156,7 +157,10 @@ class HistoricalPnlController extends Controller { } // aggregate pnlTicks for all subaccounts grouped by blockHeight - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(pnlTicks); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = _.map( + aggregateHourlyPnlTicks(pnlTicks), + 'pnlTick', + ); return { historicalPnl: aggregatedPnlTicks.map( diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 10342c28e4..a249b50b23 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -25,8 +25,13 @@ import { VaultTable, VaultFromDatabase, MEGAVAULT_SUBACCOUNT_ID, + TransferFromDatabase, + TransferTable, + TransferColumns, + Ordering, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; +import bounds from 'binary-searching'; import express from 'express'; import { checkSchema, matchedData } from 'express-validator'; import _ from 'lodash'; @@ -56,13 +61,14 @@ import { SubaccountResponseObject, MegavaultHistoricalPnlRequest, VaultsHistoricalPnlRequest, + AggregatedPnlTick, } from '../../../types'; const router: express.Router = express.Router(); const controllerName: string = 'vault-controller'; interface VaultMapping { - [subaccountId: string]: string, + [subaccountId: string]: VaultFromDatabase, } @Route('vault/v1') @@ -88,18 +94,21 @@ class VaultController extends Controller { latestBlock, mainSubaccountEquity, latestPnlTick, + firstMainVaultTransferTimestamp, ] : [ PnlTicksFromDatabase[], Map, BlockFromDatabase, string, PnlTicksFromDatabase | undefined, + DateTime | undefined ] = await Promise.all([ getVaultSubaccountPnlTicks(vaultSubaccountIdsWithMainSubaccount, getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), - getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount), + getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)), + getFirstMainVaultTransferDateTime(), ]); stats.timing( `${config.SERVICE_NAME}.${controllerName}.fetch_ticks_positions_equity.timing`, @@ -107,7 +116,11 @@ class VaultController extends Controller { ); // aggregate pnlTicks for all vault subaccounts grouped by blockHeight - const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(vaultPnlTicks); + const aggregatedPnlTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( + vaultPnlTicks, + _.values(vaultSubaccounts), + firstMainVaultTransferTimestamp, + ); const currentEquity: string = Array.from(vaultPositions.values()) .map((position: VaultPosition): string => { @@ -154,7 +167,7 @@ class VaultController extends Controller { .mapValues((pnlTicks: PnlTicksFromDatabase[], subaccountId: string): VaultHistoricalPnl => { const market: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher .getPerpetualMarketFromClobPairId( - vaultSubaccounts[subaccountId], + vaultSubaccounts[subaccountId].clobPairId, ); if (market === undefined) { @@ -306,7 +319,8 @@ router.get( Date.now() - start, ); } - }); + }, +); async function getVaultSubaccountPnlTicks( vaultSubaccountIds: string[], @@ -431,7 +445,7 @@ async function getVaultPositions( subaccountId: string, }[] = subaccounts.map((subaccount: SubaccountFromDatabase) => { const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); + .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id].clobPairId); if (perpetualMarket === undefined) { throw new Error( `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + @@ -522,6 +536,7 @@ function getPnlTicksWithCurrentTick( export async function getLatestPnlTick( vaultSubaccountIds: string[], + vaults: VaultFromDatabase[], ): Promise { const pnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.getPnlTicksAtIntervals( PnlTickInterval.hour, @@ -529,7 +544,10 @@ export async function getLatestPnlTick( vaultSubaccountIds, ); // Aggregate and get pnl tick closest to the hour - const aggregatedTicks: PnlTicksFromDatabase[] = aggregateHourlyPnlTicks(pnlTicks); + const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( + pnlTicks, + vaults, + ); const filteredTicks: PnlTicksFromDatabase[] = filterOutIntervalTicks( aggregatedTicks, PnlTickInterval.hour, @@ -631,6 +649,65 @@ function getHeightWindows( return windows; } +async function getFirstMainVaultTransferDateTime(): Promise { + const { results }: { + results: TransferFromDatabase[], + } = await TransferTable.findAllToOrFromSubaccountId( + { + subaccountId: [MEGAVAULT_SUBACCOUNT_ID], + limit: 1, + }, + [], + { + orderBy: [[TransferColumns.createdAt, Ordering.ASC]], + }, + ); + if (results.length === 0) { + return undefined; + } + return DateTime.fromISO(results[0].createdAt); +} + +/** + * Aggregates vault pnl ticks per hour, filtering out pnl ticks made up of less ticks than expected. + * Expected number of pnl ticks is calculated from the number of vaults that were created before + * the pnl tick was created. + * @param vaultPnlTicks Pnl ticks to aggregate. + * @param vaults List of all valid vaults. + * @param mainVaultCreatedAt Date time when the main vault was created or undefined if it does not + * exist yet. + * @returns + */ +function aggregateVaultPnlTicks( + vaultPnlTicks: PnlTicksFromDatabase[], + vaults: VaultFromDatabase[], + mainVaultCreatedAt?: DateTime, +): PnlTicksFromDatabase[] { + // aggregate pnlTicks for all vault subaccounts grouped by blockHeight + const aggregatedPnlTicks: AggregatedPnlTick[] = aggregateHourlyPnlTicks(vaultPnlTicks); + const vaultCreationTimes: DateTime[] = _.map(vaults, 'createdAt').map( + (createdAt: string) => { return DateTime.fromISO(createdAt); }, + ).concat( + mainVaultCreatedAt === undefined ? [] : [mainVaultCreatedAt], + ).sort( + (a: DateTime, b: DateTime) => { + return a.diff(b).milliseconds; + }, + ); + return aggregatedPnlTicks.filter((aggregatedTick: AggregatedPnlTick) => { + // Get number of vaults created before the pnl tick was created by binary-searching for the + // index of the pnl ticks createdAt in a sorted array of vault createdAt times. + const numVaultsCreated: number = bounds.le( + vaultCreationTimes, + DateTime.fromISO(aggregatedTick.pnlTick.createdAt), + (a: DateTime, b: DateTime) => { return a.diff(b).milliseconds; }, + ); + // Number of ticks should be greater than number of vaults created before it as there should be + // a tick for the main vault subaccount. + return aggregatedTick.numTicks >= numVaultsCreated; + }).map((aggregatedPnlTick: AggregatedPnlTick) => { return aggregatedPnlTick.pnlTick; }); +} + async function getVaultMapping(): Promise { const vaults: VaultFromDatabase[] = await VaultTable.findAll( {}, @@ -641,15 +718,13 @@ async function getVaultMapping(): Promise { vaults.map((vault: VaultFromDatabase): string => { return SubaccountTable.uuid(vault.address, 0); }), - vaults.map((vault: VaultFromDatabase): string => { - return vault.clobPairId; - }), + vaults, ); const validVaultMapping: VaultMapping = {}; for (const subaccountId of _.keys(vaultMapping)) { const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher .getPerpetualMarketFromClobPairId( - vaultMapping[subaccountId], + vaultMapping[subaccountId].clobPairId, ); if (perpetual === undefined) { logger.warning({ diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index b3f5d3bf19..16d27bf624 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -37,6 +37,7 @@ import { subaccountToResponseObject, } from '../request-helpers/request-transformer'; import { + AggregatedPnlTick, AssetById, AssetPositionResponseObject, AssetPositionsMap, @@ -675,17 +676,23 @@ export function getSubaccountResponse( /** * Aggregates a list of PnL ticks, combining any PnL ticks for the same hour by summing * the equity, totalPnl, and net transfers. - * Returns a map of block height to the resulting PnL tick. + * Returns a map of aggregated pnl ticks and the number of ticks the aggreated tick is made up of. * @param pnlTicks * @returns */ export function aggregateHourlyPnlTicks( pnlTicks: PnlTicksFromDatabase[], -): PnlTicksFromDatabase[] { +): AggregatedPnlTick[] { const hourlyPnlTicks: Map = new Map(); + const hourlySubaccountIds: Map> = new Map(); for (const pnlTick of pnlTicks) { const truncatedTime: string = DateTime.fromISO(pnlTick.createdAt).startOf('hour').toISO(); if (hourlyPnlTicks.has(truncatedTime)) { + const subaccountIds: Set = hourlySubaccountIds.get(truncatedTime) as Set; + if (subaccountIds.has(pnlTick.subaccountId)) { + continue; + } + subaccountIds.add(pnlTick.subaccountId); const aggregatedTick: PnlTicksFromDatabase = hourlyPnlTicks.get( truncatedTime, ) as PnlTicksFromDatabase; @@ -700,9 +707,16 @@ export function aggregateHourlyPnlTicks( ).toString(), }, ); + hourlySubaccountIds.set(truncatedTime, subaccountIds); } else { hourlyPnlTicks.set(truncatedTime, pnlTick); + hourlySubaccountIds.set(truncatedTime, new Set([pnlTick.subaccountId])); } } - return Array.from(hourlyPnlTicks.values()); + return Array.from(hourlyPnlTicks.keys()).map((hour: string): AggregatedPnlTick => { + return { + pnlTick: hourlyPnlTicks.get(hour) as PnlTicksFromDatabase, + numTicks: (hourlySubaccountIds.get(hour) as Set).size, + }; + }); } diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 0e0a57512c..fc83a92e29 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -264,6 +264,11 @@ export interface PnlTicksResponseObject { blockTime: IsoString, } +export interface AggregatedPnlTick{ + pnlTick: PnlTicksResponseObject, + numTicks: number, +} + /* ------- TRADE TYPES ------- */ export interface TradeResponse extends PaginationResponse {