Dictionary V2 Ethereum (#225)

* move node-core * working but need tidy up poc with indexing tidy up * use EnqueueBlock * draft * Update packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts Co-authored-by: Scott Twiname <[email protected]> * draft * generalization draft * tidy up * Fix runtime issue * working * rename update before fix types fix types tidy up * update interface * fix v1 tests * update tests * update interface and tests * fix test and change enqueue type * fix modulo and bypass blocks * tidy up * tidy up and more fixes * Update packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts Co-authored-by: Scott Twiname <[email protected]> * Update packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts Co-authored-by: Scott Twiname <[email protected]> * Fix part 1 * Fix part 2 * Update packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts Co-authored-by: Scott Twiname <[email protected]> * Update packages/node/src/indexer/dictionary/v2/types.ts Co-authored-by: Scott Twiname <[email protected]> * Update packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts Co-authored-by: Scott Twiname <[email protected]> * Fix part 3 * Fix part 4 * Fix part 5 * Fix worker fetching * Add feature disable dictionary, timeout inspection query * private getDictionary * Fix part 6 * Fix update v2 metadata when getData * other fixes * other fixes * remove node-core, update dependencies * update * remove FAT * fix formatter * Update packages/node/src/ethereum/utils.ethereum.ts Co-authored-by: Scott Twiname <[email protected]> * tidy up * fix isFullBlock * fix node-core version * Update packages/node/src/indexer/dictionary/v2/ethDictionaryV2.ts Co-authored-by: Scott Twiname <[email protected]> * Update packages/node/src/indexer/dictionary/v1/ethDictionaryV1.ts Co-authored-by: Scott Twiname <[email protected]> * Fix1 * tests * Fix2 * tidy up * fix dependencies * fix dependencies * fix tests * Fix converting dict block to eth block, add tests (#265) * Fix converting dict block to eth block, add tests * Fix tx being empty --------- Co-authored-by: Scott Twiname <[email protected]>
subquery · Mar 21, 2024 · 8324c35 · 8324c35
1 parent 77a2e72
commit 8324c35
Show file tree

Hide file tree

Showing 38 changed files with 1,692 additions and 1,220 deletions.
diff --git a/.prettierignore b/.prettierignore
@@ -9,3 +9,5 @@
 *.tgz
 *.cmd
 *.sh
+generate-project*.yaml
+*.proto
diff --git a/jest.config.js b/jest.config.js
@@ -81,7 +81,7 @@ module.exports = {
   // moduleNameMapper: {},
   moduleNameMapper: {
     '@subql/common-ethereum': '<rootDir>/packages/common-ethereum/src',
-    '@subql/types-ethereum': '<rootDir>/packages/types/src',
+    '@subql/types-ethereum': '<rootDir>/packages/types/src'
   },
 
   // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader

diff --git a/packages/common-ethereum/package.json b/packages/common-ethereum/package.json
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.4.1",
+    "@subql/common": "^3.4.2-1",
     "@subql/types-ethereum": "workspace:*",
     "@typechain/ethers-v5": "^11.1.1",
     "@zilliqa-js/crypto": "^3.5.0",

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -25,10 +25,10 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.4.1",
+    "@subql/common": "^3.4.2-1",
     "@subql/common-ethereum": "workspace:*",
-    "@subql/node-core": "^7.5.0",
-    "@subql/testing": "^2.1.0",
+    "@subql/node-core": "^7.5.1-3",
+    "@subql/testing": "^2.1.1-0",
     "@subql/types-ethereum": "workspace:*",
     "cacheable-lookup": "6",
     "cron-converter": "^1.0.2",

diff --git a/packages/node/src/ethereum/api.connection.ts b/packages/node/src/ethereum/api.connection.ts
@@ -11,14 +11,18 @@ import {
   RateLimitError,
   TimeoutError,
   IApiConnectionSpecific,
+  IBlock,
 } from '@subql/node-core';
 import { EthereumBlock, LightEthereumBlock } from '@subql/types-ethereum';
 import { EthereumApi } from './api.ethereum';
 import SafeEthProvider from './safe-api';
 
 export type FetchFunc =
-  | ((api: EthereumApi, batch: number[]) => Promise<EthereumBlock[]>)
-  | ((api: EthereumApi, batch: number[]) => Promise<LightEthereumBlock[]>);
+  | ((api: EthereumApi, batch: number[]) => Promise<IBlock<EthereumBlock>[]>)
+  | ((
+      api: EthereumApi,
+      batch: number[],
+    ) => Promise<IBlock<LightEthereumBlock>[]>);
 
 // We use a function to get the fetch function because it can change depending on the skipBlocks feature
 export type GetFetchFunc = () => FetchFunc;
@@ -28,7 +32,7 @@ export class EthereumApiConnection
     IApiConnectionSpecific<
       EthereumApi,
       SafeEthProvider,
-      EthereumBlock[] | LightEthereumBlock[]
+      IBlock<EthereumBlock>[] | IBlock<LightEthereumBlock>[]
     >
 {
   readonly networkMeta: NetworkMetadataPayload;
@@ -77,7 +81,7 @@ export class EthereumApiConnection
 
   async fetchBlocks(
     heights: number[],
-  ): Promise<EthereumBlock[] | LightEthereumBlock[]> {
+  ): Promise<IBlock<EthereumBlock>[] | IBlock<LightEthereumBlock>[]> {
     const blocks = await this.fetchBlocksBatches()(this.unsafeApi, heights);
     return blocks;
   }

diff --git a/packages/node/src/ethereum/api.ethereum.test.ts b/packages/node/src/ethereum/api.ethereum.test.ts
@@ -14,6 +14,7 @@ import { EthereumApi } from './api.ethereum';
 import {
   filterLogsProcessor,
   filterTransactionsProcessor,
+  isFullBlock,
 } from './block.ethereum';
 
 // Add api key to work
@@ -49,7 +50,7 @@ describe('Api.ethereum', () => {
   const fetchBlock = async (height: number) => {
     const block = await ethApi.fetchBlock(height);
 
-    return block as EthereumBlock;
+    return block.block as EthereumBlock;
   };
 
   beforeEach(async () => {
@@ -356,4 +357,18 @@ describe('Api.ethereum', () => {
       `Log BlockHash does not match block: ${mockBlockNumber}`,
     );
   });
+
+  it('Should able to check is fullBlock', async () => {
+    // block with transactions
+    const lightBlock = (await (ethApi as any).fetchLightBlock(16258633)).block;
+    expect(isFullBlock(blockData)).toBeTruthy();
+    expect(isFullBlock(lightBlock)).toBeFalsy();
+
+    // block without transaction
+    const block10001 = (await (ethApi as any).fetchBlock(10001)).block;
+    const lightBlock10001 = (await (ethApi as any).fetchLightBlock(10001))
+      .block;
+    expect(isFullBlock(block10001)).toBeFalsy();
+    expect(isFullBlock(lightBlock10001)).toBeFalsy();
+  });
 });
diff --git a/packages/node/src/ethereum/api.ethereum.ts b/packages/node/src/ethereum/api.ethereum.ts
@@ -14,7 +14,7 @@ import {
 } from '@ethersproject/abstract-provider';
 import { WebSocketProvider } from '@ethersproject/providers';
 import { EventEmitter2 } from '@nestjs/event-emitter';
-import { getLogger, timeout } from '@subql/node-core';
+import { getLogger, IBlock, timeout } from '@subql/node-core';
 import {
   ApiWrapper,
   EthereumBlock,
@@ -40,6 +40,7 @@ import { ConnectionInfo } from './ethers/web';
 import SafeEthProvider from './safe-api';
 import {
   formatBlock,
+  formatBlockUtil,
   formatLog,
   formatReceipt,
   formatTransaction,
@@ -295,7 +296,6 @@ export class EthereumApi implements ApiWrapper {
 
     return block;
   }
-
   async getTransactionReceipt(
     transactionHash: string | Promise<string>,
   ): Promise<TransactionReceipt> {
@@ -304,10 +304,12 @@ export class EthereumApi implements ApiWrapper {
     );
   }
 
-  async fetchBlock(blockNumber: number): Promise<EthereumBlock> {
+  async fetchBlock(blockNumber: number): Promise<IBlock<EthereumBlock>> {
     try {
       const block = await this.getBlockPromise(blockNumber, true);
-      const logsRaw = await this.client.getLogs({ blockHash: block.hash });
+      const logsRaw = await this.client.getLogs({
+        blockHash: block.hash,
+      });
 
       // Certain RPC may not accommodate for blockHash, and would return wrong logs
       if (logsRaw.length) {
@@ -328,31 +330,32 @@ export class EthereumApi implements ApiWrapper {
       }));
 
       this.eventEmitter.emit('fetchBlock');
-      return block;
+      return formatBlockUtil(block);
     } catch (e) {
       throw this.handleError(e);
     }
   }
 
   private async fetchLightBlock(
     blockNumber: number,
-  ): Promise<LightEthereumBlock> {
+  ): Promise<IBlock<LightEthereumBlock>> {
     const block = await this.getBlockPromise(blockNumber, false);
     const logs = await this.client.getLogs({ blockHash: block.hash });
 
-    return {
+    const lightBlock: LightEthereumBlock = {
       ...block,
       logs: logs.map((l) => formatLog(l, block)),
     };
+    return formatBlockUtil<LightEthereumBlock>(lightBlock);
   }
 
-  async fetchBlocks(bufferBlocks: number[]): Promise<EthereumBlock[]> {
+  async fetchBlocks(bufferBlocks: number[]): Promise<IBlock<EthereumBlock>[]> {
     return Promise.all(bufferBlocks.map(async (num) => this.fetchBlock(num)));
   }
 
   async fetchBlocksLight(
     bufferBlocks: number[],
-  ): Promise<LightEthereumBlock[]> {
+  ): Promise<IBlock<LightEthereumBlock>[]> {
     return Promise.all(
       bufferBlocks.map(async (num) => this.fetchLightBlock(num)),
     );
@@ -449,7 +452,6 @@ export class EthereumApi implements ApiWrapper {
         )) as Array<EthereumLog | EthereumLog<T>>);
 
       transaction.args = args;
-
       return transaction;
     } catch (e) {
       logger.warn(`Failed to parse transaction data: ${e.message}`);

diff --git a/packages/node/src/ethereum/api.service.ethereum.ts b/packages/node/src/ethereum/api.service.ethereum.ts
@@ -9,6 +9,7 @@ import {
   getLogger,
   NodeConfig,
   profilerWrap,
+  IBlock,
 } from '@subql/node-core';
 import {
   EthereumBlock,
@@ -32,7 +33,7 @@ const logger = getLogger('api');
 export class EthereumApiService extends ApiService<
   EthereumApi,
   SafeEthProvider,
-  EthereumBlock[] | LightEthereumBlock[]
+  IBlock<EthereumBlock>[] | IBlock<LightEthereumBlock>[]
 > {
   private fetchBlocksFunction: FetchFunc;
   private fetchBlocksBatches: GetFetchFunc = () => this.fetchBlocksFunction;
@@ -159,14 +160,14 @@ export class EthereumApiService extends ApiService<
   private async fetchFullBlocksBatch(
     api: EthereumApi,
     batch: number[],
-  ): Promise<EthereumBlock[]> {
+  ): Promise<IBlock<EthereumBlock>[]> {
     return api.fetchBlocks(batch);
   }
 
   private async fetchLightBlocksBatch(
     api: EthereumApi,
     batch: number[],
-  ): Promise<LightEthereumBlock[]> {
+  ): Promise<IBlock<LightEthereumBlock>[]> {
     return api.fetchBlocksLight(batch);
   }
 

diff --git a/packages/node/src/ethereum/block.ethereum.ts b/packages/node/src/ethereum/block.ethereum.ts
@@ -136,6 +136,11 @@ export function filterLogsProcessor(
 }
 
 export function isFullBlock(block: BlockContent): block is EthereumBlock {
-  // Light etherum block just contains transaction hashes for transactions. If the block has no transactions then both types would be the same
-  return typeof (block as EthereumBlock).transactions[0] !== 'string';
+  // Light etherum block just contains transaction hashes for transactions.
+  // If the block has no transactions then both types would be the same
+
+  if (block.transactions.length && block.logs.length) {
+    return typeof (block as EthereumBlock).transactions[0] !== 'string';
+  }
+  return false;
 }
diff --git a/packages/node/src/ethereum/utils.ethereum.ts b/packages/node/src/ethereum/utils.ethereum.ts
@@ -1,18 +1,22 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import { Block } from '@ethersproject/abstract-provider';
 import { getAddress } from '@ethersproject/address';
 import { BigNumber } from '@ethersproject/bignumber';
 import { Zero } from '@ethersproject/constants';
+import { Header, IBlock } from '@subql/node-core';
 import {
   ApiWrapper,
   EthereumBlock,
   EthereumLog,
   EthereumReceipt,
   EthereumResult,
   EthereumTransaction,
+  LightEthereumBlock,
 } from '@subql/types-ethereum';
 import { omit } from 'lodash';
+import { BlockContent } from '../indexer/types';
 
 export function calcInterval(api: ApiWrapper): number {
   // TODO find a way to get this from the blockchain
@@ -26,7 +30,7 @@ function handleAddress(value: string): string | null {
   return getAddress(value);
 }
 
-function handleNumber(value: string | number): BigNumber {
+export function handleNumber(value: string | number): BigNumber {
   if (value === undefined) {
     return Zero;
   }
@@ -58,6 +62,16 @@ export function formatBlock(block: Record<string, any>): EthereumBlock {
     logs: [], // Filled in at AvalancheBlockWrapped constructor
   } as EthereumBlock;
 }
+
+export function formatBlockUtil<
+  B extends EthereumBlock | LightEthereumBlock = EthereumBlock,
+>(block: B): IBlock<B> {
+  return {
+    block,
+    getHeader: () => ethereumBlockToHeader(block),
+  };
+}
+
 export function formatLog(
   log: Omit<
     EthereumLog<EthereumResult> | EthereumLog,
@@ -139,3 +153,11 @@ export function formatReceipt<R extends EthereumReceipt = EthereumReceipt>(
     },
   } as unknown as R;
 }
+
+export function ethereumBlockToHeader(block: BlockContent | Block): Header {
+  return {
+    blockHeight: block.number,
+    blockHash: block.hash,
+    parentHash: block.parentHash,
+  };
+}
diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts
@@ -9,12 +9,12 @@ import {
   StoreCacheService,
   StoreService,
   IProjectService,
-  PoiService,
   BlockDispatcher,
   ProcessBlockResponse,
   ApiService,
   IProjectUpgradeService,
   PoiSyncService,
+  IBlock,
 } from '@subql/node-core';
 import {
   EthereumProjectDs,
@@ -63,16 +63,12 @@ export class BlockDispatcherService
     );
   }
 
-  protected getBlockHeight(block: BlockContent): number {
-    return block.number;
-  }
-
   protected async indexBlock(
-    block: BlockContent,
+    block: IBlock<BlockContent>,
   ): Promise<ProcessBlockResponse> {
     return this.indexerManager.indexBlock(
       block,
-      await this.projectService.getDataSources(this.getBlockHeight(block)),
+      await this.projectService.getDataSources(block.getHeader().blockHeight),
     );
   }
 }
diff --git a/packages/node/src/indexer/blockDispatcher/ethereum-block-dispatcher.ts b/packages/node/src/indexer/blockDispatcher/ethereum-block-dispatcher.ts
@@ -1,8 +1,10 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import { IBlockDispatcher } from '@subql/node-core';
+import { IBlockDispatcher, IBlock } from '@subql/node-core';
+import { EthereumBlock } from '@subql/types-ethereum';
 
-export interface IEthereumBlockDispatcher extends IBlockDispatcher {
+export interface IEthereumBlockDispatcher
+  extends IBlockDispatcher<EthereumBlock> {
   init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
 }
diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -8,7 +8,6 @@ import {
   NodeConfig,
   SmartBatchService,
   StoreService,
-  PoiService,
   StoreCacheService,
   IProjectService,
   WorkerBlockDispatcher,
@@ -18,6 +17,7 @@ import {
   InMemoryCacheService,
   createIndexerWorker,
 } from '@subql/node-core';
+import { EthereumBlock } from '@subql/types-ethereum';
 import {
   EthereumProjectDs,
   SubqueryProject,
@@ -34,7 +34,7 @@ type IndexerWorker = IIndexerWorker & {
 
 @Injectable()
 export class WorkerBlockDispatcherService
-  extends WorkerBlockDispatcher<EthereumProjectDs, IndexerWorker>
+  extends WorkerBlockDispatcher<EthereumProjectDs, IndexerWorker, EthereumBlock>
   implements OnApplicationShutdown
 {
   constructor(
@@ -89,19 +89,6 @@ export class WorkerBlockDispatcherService
     worker: IndexerWorker,
     height: number,
   ): Promise<void> {
-    const start = new Date();
     await worker.fetchBlock(height, null);
-    const end = new Date();
-
-    // const waitTime = end.getTime() - start.getTime();
-    // if (waitTime > 1000) {
-    //   logger.info(
-    //     `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,
-    //   );
-    // } else if (waitTime > 200) {
-    //   logger.info(
-    //     `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`,
-    //   );
-    // }
   }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,3 +9,5 @@ @@
     *.tgz
     *.cmd
     *.sh
+    generate-project*.yaml
+    *.proto