Improve vault endpoint performance. (backport #2475)

indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts

@@ -242,4 +242,38 @@ describe('funding index update store', () => {
       expect(fundingIndexMap[defaultPerpetualMarket2.id]).toEqual(Big(0));
     },
   );
+
+  it('Successfully finds funding index maps for multiple effectiveBeforeOrAtHeights', async () => {
+    const fundingIndexUpdates2: FundingIndexUpdatesCreateObject = {
+      ...defaultFundingIndexUpdate,
+      fundingIndex: '124',
+      effectiveAtHeight: updatedHeight,
+      effectiveAt: '1982-05-25T00:00:00.000Z',
+      eventId: defaultTendermintEventId2,
+    };
+    const fundingIndexUpdates3: FundingIndexUpdatesCreateObject = {
+      ...defaultFundingIndexUpdate,
+      eventId: defaultTendermintEventId3,
+      perpetualId: defaultPerpetualMarket2.id,
+    };
+    await Promise.all([
+      FundingIndexUpdatesTable.create(defaultFundingIndexUpdate),
+      FundingIndexUpdatesTable.create(fundingIndexUpdates2),
+      FundingIndexUpdatesTable.create(fundingIndexUpdates3),
+    ]);
+
+    const fundingIndexMaps: {[blockHeight:string]: FundingIndexMap} = await FundingIndexUpdatesTable
+      .findFundingIndexMaps(
+        ['3', '6'],
+      );
+
+    expect(fundingIndexMaps['3'][defaultFundingIndexUpdate.perpetualId])
+      .toEqual(Big(defaultFundingIndexUpdate.fundingIndex));
+    expect(fundingIndexMaps['3'][fundingIndexUpdates3.perpetualId])
+      .toEqual(Big(fundingIndexUpdates3.fundingIndex));
+    expect(fundingIndexMaps['6'][defaultFundingIndexUpdate.perpetualId])
+      .toEqual(Big(fundingIndexUpdates2.fundingIndex));
+    expect(fundingIndexMaps['6'][fundingIndexUpdates3.perpetualId])
+      .toEqual(Big(fundingIndexUpdates3.fundingIndex));
+  });
 });

indexer/packages/postgres/src/stores/funding-index-updates-table.ts

@@ -3,6 +3,7 @@ import _ from 'lodash';
 import { QueryBuilder } from 'objection';
 
 import { BUFFER_ENCODING_UTF_8, DEFAULT_POSTGRES_OPTIONS } from '../constants';
+import { knexReadReplica } from '../helpers/knex';
 import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers';
 import Transaction from '../helpers/transaction';
 import { getUuid } from '../helpers/uuid';
@@ -21,6 +22,14 @@ import {
 } from '../types';
 import * as PerpetualMarketTable from './perpetual-market-table';
 
+// Assuming block time of 1 second, this should be 4 hours of blocks
+const FOUR_HOUR_OF_BLOCKS = Big(3600).times(4);
+// Type used for querying for funding index maps for multiple effective heights.
+interface FundingIndexUpdatesFromDatabaseWithSearchHeight extends FundingIndexUpdatesFromDatabase {
+  // max effective height being queried for
+  searchHeight: string,
+}
+
 export function uuid(
   blockHeight: string,
   eventId: Buffer,
@@ -193,8 +202,6 @@ export async function findFundingIndexMap(
       options,
     );
 
-  // Assuming block time of 1 second, this should be 4 hours of blocks
-  const FOUR_HOUR_OF_BLOCKS = Big(3600).times(4);
   const fundingIndexUpdates: FundingIndexUpdatesFromDatabase[] = await baseQuery
     .distinctOn(FundingIndexUpdatesColumns.perpetualId)
     .where(FundingIndexUpdatesColumns.effectiveAtHeight, '<=', effectiveBeforeOrAtHeight)
@@ -216,3 +223,67 @@ export async function findFundingIndexMap(
     initialFundingIndexMap,
   );
 }
+
+/**
+ * Finds funding index maps for multiple effective before or at heights. Uses a SQL query unnesting
+ * an array of effective before or at heights and cross-joining with the funding index updates table
+ * to find the closest funding index update per effective before or at height.
+ * @param effectiveBeforeOrAtHeights Heights to get funding index maps for.
+ * @param options
+ * @returns Object mapping block heights to the respective funding index maps.
+ */
+export async function findFundingIndexMaps(
+  effectiveBeforeOrAtHeights: string[],
+  options: Options = DEFAULT_POSTGRES_OPTIONS,
+): Promise<{[blockHeight: string]: FundingIndexMap}> {
+  const heightNumbers: number[] = effectiveBeforeOrAtHeights
+    .map((height: string):number => parseInt(height, 10))
+    .filter((parsedHeight: number): boolean => { return !Number.isNaN(parsedHeight); })
+    .sort();
+  // Get the min height to limit the search to blocks 4 hours or before the min height.
+  const minHeight: number = heightNumbers[0];
+
+  const result: {
+    rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[],
+  } = await knexReadReplica.getConnection().raw(
+    `
+    SELECT
+      DISTINCT ON ("perpetualId", "searchHeight") "perpetualId", "searchHeight",
+      "funding_index_updates".*
+    FROM
+      "funding_index_updates",
+      unnest(ARRAY[${heightNumbers.join(',')}]) AS "searchHeight"
+    WHERE
+      "effectiveAtHeight" > ${Big(minHeight).minus(FOUR_HOUR_OF_BLOCKS).toFixed()} AND
+      "effectiveAtHeight" <= "searchHeight"
+    ORDER BY
+      "perpetualId",
+      "searchHeight",
+      "effectiveAtHeight" DESC
+    `,
+  ) as unknown as {
+    rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[],
+  };
+
+  const perpetualMarkets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll(
+    {},
+    [],
+    options,
+  );
+
+  const fundingIndexMaps:{[blockHeight: string]: FundingIndexMap} = {};
+  for (const height of effectiveBeforeOrAtHeights) {
+    fundingIndexMaps[height] = _.reduce(perpetualMarkets,
+      (acc: FundingIndexMap, perpetualMarket: PerpetualMarketFromDatabase): FundingIndexMap => {
+        acc[perpetualMarket.id] = Big(0);
+        return acc;
+      },
+      {},
+    );
+  }
+  for (const funding of result.rows) {
+    fundingIndexMaps[funding.searchHeight][funding.perpetualId] = Big(funding.fundingIndex);
+  }
+
+  return fundingIndexMaps;
+}

indexer/services/comlink/src/controllers/api/v4/vault-controller.ts

@@ -375,10 +375,23 @@ async function getVaultPositions(
     BlockTable.getLatest(),
   ]);
 
-  const latestFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable
-    .findFundingIndexMap(
-      latestBlock.blockHeight,
-    );
+  const updatedAtHeights: string[] = _(subaccounts).map('updatedAtHeight').uniq().value();
+  const [
+    latestFundingIndexMap,
+    fundingIndexMaps,
+  ]: [
+    FundingIndexMap,
+    {[blockHeight: string]: FundingIndexMap}
+  ] = await Promise.all([
+    FundingIndexUpdatesTable
+      .findFundingIndexMap(
+        latestBlock.blockHeight,
+      ),
+    FundingIndexUpdatesTable
+      .findFundingIndexMaps(
+        updatedAtHeights,
+      ),
+  ]);
   const assetPositionsBySubaccount:
   { [subaccountId: string]: AssetPositionFromDatabase[] } = _.groupBy(
     assetPositions,
@@ -397,47 +410,49 @@ async function getVaultPositions(
   const vaultPositionsAndSubaccountId: {
     position: VaultPosition,
     subaccountId: string,
-  }[] = await Promise.all(
-    subaccounts.map(async (subaccount: SubaccountFromDatabase) => {
-      const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
-        .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]);
-      if (perpetualMarket === undefined) {
-        throw new Error(
-          `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` +
+  }[] = subaccounts.map((subaccount: SubaccountFromDatabase) => {
+    const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
+      .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]);
+    if (perpetualMarket === undefined) {
+      throw new Error(
+        `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` +
           'perpetual market.');
-      }
-      const lastUpdatedFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable
-        .findFundingIndexMap(
-          subaccount.updatedAtHeight,
-        );
-
-      const subaccountResponse: SubaccountResponseObject = getSubaccountResponse(
-        subaccount,
-        openPerpetualPositionsBySubaccount[subaccount.id] || [],
-        assetPositionsBySubaccount[subaccount.id] || [],
-        assets,
-        markets,
-        perpetualMarketRefresher.getPerpetualMarketsMap(),
-        latestBlock.blockHeight,
-        latestFundingIndexMap,
-        lastUpdatedFundingIndexMap,
+    }
+    const lastUpdatedFundingIndexMap: FundingIndexMap = fundingIndexMaps[
+      subaccount.updatedAtHeight
+    ];
+    if (lastUpdatedFundingIndexMap === undefined) {
+      throw new Error(
+        `No funding indices could be found for vault with subaccount ${subaccount.id}`,
       );
+    }
 
-      return {
-        position: {
-          ticker: perpetualMarket.ticker,
-          assetPosition: subaccountResponse.assetPositions[
-            assetIdToAsset[USDC_ASSET_ID].symbol
-          ],
-          perpetualPosition: subaccountResponse.openPerpetualPositions[
-            perpetualMarket.ticker
-          ] || undefined,
-          equity: subaccountResponse.equity,
-        },
-        subaccountId: subaccount.id,
-      };
-    }),
-  );
+    const subaccountResponse: SubaccountResponseObject = getSubaccountResponse(
+      subaccount,
+      openPerpetualPositionsBySubaccount[subaccount.id] || [],
+      assetPositionsBySubaccount[subaccount.id] || [],
+      assets,
+      markets,
+      perpetualMarketRefresher.getPerpetualMarketsMap(),
+      latestBlock.blockHeight,
+      latestFundingIndexMap,
+      lastUpdatedFundingIndexMap,
+    );
+
+    return {
+      position: {
+        ticker: perpetualMarket.ticker,
+        assetPosition: subaccountResponse.assetPositions[
+          assetIdToAsset[USDC_ASSET_ID].symbol
+        ],
+        perpetualPosition: subaccountResponse.openPerpetualPositions[
+          perpetualMarket.ticker
+        ] || undefined,
+        equity: subaccountResponse.equity,
+      },
+      subaccountId: subaccount.id,
+    };
+  });
 
   return new Map(vaultPositionsAndSubaccountId.map(
     (obj: { position: VaultPosition, subaccountId: string }) : [string, VaultPosition] => {