Add support for endpoint config headers and batch size

packages/common-ethereum/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Suport for network endpoint config (#334)
 
 ## [4.4.0] - 2024-07-26
 ### Changed

packages/common-ethereum/package.json

@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^4.1.0",
+    "@subql/common": "^5.0.0",
     "@subql/types-ethereum": "workspace:*",
     "@typechain/ethers-v5": "^11.1.1",
     "@zilliqa-js/crypto": "^3.5.0",

packages/common-ethereum/src/project/versioned/v1_0_0/model.ts

@@ -3,7 +3,10 @@
 
 import {
   BaseDeploymentV1_0_0,
+  CommonEndpointConfig,
+  CommonProjectNetworkV1_0_0,
   FileType,
+  IsNetworkEndpoint,
   ParentProjectModel,
   ProjectManifestBaseImpl,
   RunnerNodeImpl,
@@ -13,6 +16,7 @@ import {BaseMapping, NodeSpec, RunnerSpecs, QuerySpec, ParentProject} from '@sub
 import {
   CustomDatasourceTemplate,
   EthereumProjectManifestV1_0_0,
+  IEthereumEndpointConfig,
   RuntimeDatasourceTemplate,
   SubqlCustomDatasource,
   SubqlMapping,
@@ -26,6 +30,7 @@ import {
   IsNotEmpty,
   IsObject,
   IsOptional,
+  IsPositive,
   IsString,
   ValidateNested,
   validateSync,
@@ -94,21 +99,21 @@ export class ProjectNetworkDeploymentV1_0_0 {
   @Transform(({value}: TransformFnParams) => value.trim())
   @IsString()
   chainId!: string;
-  @ValidateNested()
-  @Type(() => FileType)
-  @IsOptional()
-  chaintypes?: FileType;
   @IsOptional()
   @IsArray()
   bypassBlocks?: (number | `${number}-${number}`)[];
 }
 
-export class ProjectNetworkV1_0_0 extends ProjectNetworkDeploymentV1_0_0 {
-  @IsString({each: true})
-  endpoint!: string | string[];
-  @IsString()
+export class EthereumEndpointConfig extends CommonEndpointConfig implements IEthereumEndpointConfig {
+  @IsOptional()
+  @IsPositive()
+  batchSize?: number;
+}
+
+export class ProjectNetworkV1_0_0 extends CommonProjectNetworkV1_0_0<void> {
   @IsOptional()
-  dictionary?: string;
+  @IsNetworkEndpoint(EthereumEndpointConfig)
+  endpoint: string | string[] | Record<string, CommonEndpointConfig> = {};
 }
 
 export class DeploymentV1_0_0 extends BaseDeploymentV1_0_0 {

packages/node/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Suport for network endpoint config providing the ability to set headers and rpc batch sizes (#334)
 
 ## [5.0.1] - 2024-07-29
 ### Fixed

packages/node/package.json

@@ -25,7 +25,7 @@
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
     "@subql/common-ethereum": "workspace:*",
-    "@subql/node-core": "^13.0.1",
+    "@subql/node-core": "^14.0.0",
     "@subql/testing": "^2.2.1",
     "@subql/types-ethereum": "workspace:*",
     "cacheable-lookup": "6",

packages/node/src/ethereum/api.connection.ts

@@ -13,7 +13,11 @@ import {
   IApiConnectionSpecific,
   IBlock,
 } from '@subql/node-core';
-import { EthereumBlock, LightEthereumBlock } from '@subql/types-ethereum';
+import {
+  EthereumBlock,
+  IEthereumEndpointConfig,
+  LightEthereumBlock,
+} from '@subql/types-ethereum';
 import { EthereumApi } from './api.ethereum';
 import SafeEthProvider from './safe-api';
 
@@ -54,12 +58,14 @@ export class EthereumApiConnection
     fetchBlocksBatches: GetFetchFunc,
     eventEmitter: EventEmitter2,
     unfinalizedBlocks: boolean,
+    config?: IEthereumEndpointConfig,
   ): Promise<EthereumApiConnection> {
     const api = new EthereumApi(
       endpoint,
       blockConfirmations,
       eventEmitter,
       unfinalizedBlocks,
+      config,
     );
 
     await api.init();

packages/node/src/ethereum/api.ethereum.ts

@@ -23,6 +23,7 @@ import {
   SubqlRuntimeDatasource,
   LightEthereumBlock,
   LightEthereumLog,
+  IEthereumEndpointConfig,
 } from '@subql/types-ethereum';
 import CacheableLookup from 'cacheable-lookup';
 import { hexDataSlice, hexValue } from 'ethers/lib/utils';
@@ -119,6 +120,7 @@ export class EthereumApi implements ApiWrapper {
     private blockConfirmations: number,
     private eventEmitter: EventEmitter2,
     private unfinalizedBlocks = false,
+    private config?: IEthereumEndpointConfig,
   ) {
     const { hostname, protocol, searchParams } = new URL(endpoint);
 
@@ -130,6 +132,7 @@ export class EthereumApi implements ApiWrapper {
         url: this.endpoint,
         headers: {
           'User-Agent': `Subquery-Node ${packageVersion}`,
+          ...config?.headers,
         },
         allowGzip: true,
         throttleLimit: 5,
@@ -141,6 +144,7 @@ export class EthereumApi implements ApiWrapper {
       });
       this.client = new (OPFormatterMixin(JsonRpcBatchProvider))(connection);
       this.nonBatchClient = new (OPFormatterMixin(JsonRpcProvider))(connection);
+      this.applyBatchSize(config?.batchSize);
     } else if (protocolStr === 'ws' || protocolStr === 'wss') {
       this.client = new (OPFormatterMixin(WebSocketProvider))(this.endpoint);
     } else {
@@ -155,6 +159,17 @@ export class EthereumApi implements ApiWrapper {
     return this._genesisBlock;
   }
 
+  private applyBatchSize(batchSize?: number): void {
+    if (batchSize === null || batchSize === undefined) return;
+
+    if (batchSize <= 0 && !!this.nonBatchClient) {
+      logger.info('Endpoint config batch size is 0, not using batch requests');
+      this.client = this.nonBatchClient;
+      return;
+    }
+    (this.client as JsonRpcBatchProvider).setBatchSize(batchSize);
+  }
+
   async init(): Promise<void> {
     this.injectClient();
 
@@ -167,6 +182,7 @@ export class EthereumApi implements ApiWrapper {
       } else {
         this.client = new CeloJsonRpcBatchProvider(this.client.connection);
         this.nonBatchClient = new CeloJsonRpcProvider(this.client.connection);
+        this.applyBatchSize(this.config?.batchSize);
       }
     }
 

packages/node/src/ethereum/api.service.ethereum.ts

@@ -13,9 +13,11 @@ import {
   IBlock,
   exitWithError,
 } from '@subql/node-core';
+import { IEndpointConfig } from '@subql/types-core';
 import {
   EthereumBlock,
   EthereumNetworkConfig,
+  IEthereumEndpointConfig,
   LightEthereumBlock,
 } from '@subql/types-ethereum';
 import { EthereumNodeConfig } from '../configure/NodeConfig';
@@ -35,7 +37,9 @@ const logger = getLogger('api');
 export class EthereumApiService extends ApiService<
   EthereumApi,
   SafeEthProvider,
-  IBlock<EthereumBlock>[] | IBlock<LightEthereumBlock>[]
+  IBlock<EthereumBlock>[] | IBlock<LightEthereumBlock>[],
+  EthereumApiConnection,
+  IEthereumEndpointConfig
 > {
   private fetchBlocksFunction?: FetchFunc;
   private fetchBlocksBatches: GetFetchFunc = () => {
@@ -64,21 +68,19 @@ export class EthereumApiService extends ApiService<
       exitWithError(new Error(`Failed to init api`, { cause: e }), logger);
     }
 
-    const endpoints = Array.isArray(network.endpoint)
-      ? network.endpoint
-      : [network.endpoint];
-
     if (this.nodeConfig.primaryNetworkEndpoint) {
-      endpoints.push(this.nodeConfig.primaryNetworkEndpoint);
+      const [endpoint, config] = this.nodeConfig.primaryNetworkEndpoint;
+      (network.endpoint as Record<string, IEndpointConfig>)[endpoint] = config;
     }
 
-    await this.createConnections(network, (endpoint) =>
+    await this.createConnections(network, (endpoint, config) =>
       EthereumApiConnection.create(
         endpoint,
         this.nodeConfig.blockConfirmations,
         this.fetchBlocksBatches,
         this.eventEmitter,
         this.nodeConfig.unfinalizedBlocks,
+        config,
       ),
     );
 

packages/node/src/indexer/dictionary/v2/ethDictionaryV2.spec.ts

@@ -107,7 +107,7 @@ const nodeConfig = new NodeConfig({
   subquery: 'eth-starter',
   subqueryName: 'eth-starter',
   dictionaryTimeout: 10,
-  networkEndpoint: [HTTP_ENDPOINT],
+  networkEndpoint: { [HTTP_ENDPOINT]: {} },
   networkDictionary: [DEFAULT_DICTIONARY],
 });
 

packages/types/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Suport for network endpoint config (#334)
 
 ## [3.12.0] - 2024-07-26
 ### Changed

packages/types/package.json

@@ -19,6 +19,6 @@
   "dependencies": {
     "@ethersproject/abstract-provider": "^5.6.1",
     "@ethersproject/providers": "^5.7.2",
-    "@subql/types-core": "^0.10.0"
+    "@subql/types-core": "^1.0.0"
   }
 }

packages/types/src/project.ts

@@ -13,6 +13,7 @@ import {
   SecondLayerHandlerProcessor_1_0_0,
   DsProcessor,
   BaseCustomDataSource,
+  IEndpointConfig,
 } from '@subql/types-core';
 import {
   EthereumBlock,
@@ -308,11 +309,18 @@ export type SubqlDatasourceProcessor<
   >
 > = DsProcessor<DS, P, ApiWrapper>;
 
+export interface IEthereumEndpointConfig extends IEndpointConfig {
+  /**
+   *  The JSON RPC batch size, if this is set to 0 it will not use batch requests
+   * */
+  batchSize?: number;
+}
+
 /**
  * Represents a Ethereum subquery network configuration, which is based on the CommonSubqueryNetworkConfig template.
  * @type {IProjectNetworkConfig}
  */
-export type EthereumNetworkConfig = IProjectNetworkConfig;
+export type EthereumNetworkConfig = IProjectNetworkConfig<IEthereumEndpointConfig>;
 
 /**
  * Represents a Ethereum project configuration based on the CommonSubqueryProject template.