Skip to content

Commit

Permalink
rename
Browse files Browse the repository at this point in the history 
update

before fix types

fix types

tidy up
  • Loading branch information
@jiqiang90
jiqiang90 committed Jan 24, 2024
1 parent 47e64e5 commit 72168b9
Show file tree
Hide file tree
Showing 21 changed files with 287 additions and 175 deletions.
10 changes: 6 additions & 4 deletions packages/node-core/src/configure/NodeConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export interface IConfig {
readonly preferRange: boolean;
readonly networkEndpoint?: string[];
readonly primaryNetworkEndpoint?: string;
readonly networkDictionary?: string;
readonly networkDictionary?: string[];
readonly dictionaryResolver?: string | false;
readonly dictionaryRegistry: string;
readonly outputFmt?: 'json';
Expand Down Expand Up @@ -141,8 +141,10 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
return this._config.primaryNetworkEndpoint;
}

get networkDictionary(): string | undefined {
return this._config.networkDictionary;
get networkDictionaries(): string[] | undefined {
return typeof this._config.networkDictionary === 'string'
? [this._config.networkDictionary]
: this._config.networkDictionary;
}

get storeCacheThreshold(): number {
Expand Down Expand Up @@ -177,7 +179,7 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
return this._config.dictionaryRegistry;
}

return 'https://github.com/subquery/templates/raw/main/dictionary.json';
return 'https://github.com/subquery/templates/raw/main/dist/dictionary.json';
}

get timeout(): number {
Expand Down
4 changes: 2 additions & 2 deletions packages/node-core/src/configure/configure.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ export async function registerApp<P extends ISubqueryProject>(
omitBy(
{
endpoint: config.networkEndpoints,
dictionary: config.networkDictionary,
dictionary: config.networkDictionaries,
},
isNil
)
Expand All @@ -160,7 +160,7 @@ export async function registerApp<P extends ISubqueryProject>(
// Apply the network endpoint and dictionary from the source project to the parent projects if they are not defined in the config
{
endpoint: config.networkEndpoints ?? project.network.endpoint,
dictionary: config.networkDictionary ?? project.network.dictionary,
dictionary: config.networkDictionaries ?? project.network.dictionary,
},
isNil
)
Expand Down
39 changes: 32 additions & 7 deletions ...exer/dictionary/coreDictionary.service.ts → .../src/indexer/dictionary/coreDictionary.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,19 @@

import assert from 'assert';
import {EventEmitter2} from '@nestjs/event-emitter';
import {BlockHeightMap} from '@subql/node-core/utils/blockHeightMap';
import {DictionaryQueryEntry as DictionaryV1QueryEntry} from '@subql/types-core/dist/project/types';
import {MetaData as DictionaryV1Metadata} from '@subql/utils';
import {DictionaryServiceInterface, DictionaryV2Metadata, DictionaryV2QueryEntry} from '../';
import {IDictionary, DictionaryV2Metadata, DictionaryV2QueryEntry, DictionaryResponse, DictionaryVersion} from '../';
import {NodeConfig} from '../../configure';
import {BlockHeightMap} from '../../utils/blockHeightMap';

export abstract class CoreDictionaryService<DS> implements DictionaryServiceInterface {
export abstract class CoreDictionary<DS, FB> implements IDictionary<DS, FB> {
queriesMap?: BlockHeightMap<DictionaryV1QueryEntry[] | DictionaryV2QueryEntry>;
protected _startHeight?: number;
protected _genesisHash?: string;
protected _metadata: DictionaryV1Metadata | DictionaryV2Metadata | undefined;
metadataValid: boolean | undefined;
protected _dictionaryVersion: DictionaryVersion | undefined;

constructor(
readonly dictionaryEndpoint: string | undefined,
Expand All @@ -23,6 +24,11 @@ export abstract class CoreDictionaryService<DS> implements DictionaryServiceInte
protected readonly eventEmitter: EventEmitter2
) {}

abstract getData(
startBlock: number,
queryEndBlock: number,
limit: number
): Promise<DictionaryResponse<FB> | undefined>;
abstract initMetadata(): Promise<void>;
abstract get metadata(): DictionaryV1Metadata | DictionaryV2Metadata;
abstract dictionaryValidation(
Expand All @@ -33,6 +39,13 @@ export abstract class CoreDictionaryService<DS> implements DictionaryServiceInte
abstract queryMapValidByHeight(height: number): boolean;
abstract getQueryEndBlock(startHeight: number, apiFinalizedHeight: number): number;

get version(): DictionaryVersion {
if (!this._dictionaryVersion) {
throw new Error(`Dictionary version not been inspected`);
}
return this._dictionaryVersion;
}

get startHeight(): number {
if (!this._startHeight) {
throw new Error('Dictionary start height is not set');
Expand All @@ -44,6 +57,10 @@ export abstract class CoreDictionaryService<DS> implements DictionaryServiceInte
return (!!this.dictionaryEndpoint || !!this.nodeConfig.dictionaryResolver) && !!this.metadataValid;
}

setApiGenesisHash(genesisHash: string): void {
this._genesisHash = genesisHash;
}

get apiGenesisHash(): string {
assert(this._genesisHash, new Error('Endpoint genesis hash is not set in dictionary'));
return this._genesisHash;
Expand All @@ -57,13 +74,21 @@ export abstract class CoreDictionaryService<DS> implements DictionaryServiceInte
this._startHeight = start ?? 1;
}

// register genesisHash, also validate with metadata
metadataValidation(genesisHash: string): boolean {
this._genesisHash = genesisHash;
return this.dictionaryValidation(this.metadata);
// filter dictionary with start height
heightValidation(height: number): boolean {
return this.dictionaryValidation(this._metadata, height);
}

updateQueriesMap(dataSources: BlockHeightMap<DS[]>): void {
this.queriesMap = dataSources.map(this.buildDictionaryQueryEntries);
}

// Base validation is required, and specific validation for each network should be implemented accordingly
protected validateChainMeta(metaData: DictionaryV1Metadata | DictionaryV2Metadata): boolean {
return (
metaData.chain === this.chainId ||
metaData.genesisHash === this.chainId ||
this.apiGenesisHash === metaData.genesisHash
);
}
}
134 changes: 80 additions & 54 deletions packages/node-core/src/indexer/dictionary/dictionary.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,85 +1,110 @@
// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
// SPDX-License-Identifier: GPL-3.0

import {Injectable} from '@nestjs/common';
import {EventEmitter2} from '@nestjs/event-emitter';
import {NETWORK_FAMILY} from '@subql/common';
import fetch from 'cross-fetch';
import {NodeConfig} from '../../configure';
import {getLogger} from '../../logger';
import {BlockHeightMap} from '../../utils/blockHeightMap';
import {Dictionary} from './types';
import {DictionaryResponse, DictionaryVersion, IDictionary, IDictionaryCtrl} from './types';
import {subqlFilterBlocksCapabilities} from './utils';
import {DictionaryServiceV1} from './v1';
import {DictionaryServiceV2} from './v2';
import {DictionaryV2Metadata, DictionaryVersion} from './';
import {DictionaryV1} from './v1';
import {DictionaryV2} from './v2';
import {DictionaryV2Metadata} from './';

const logger = getLogger('DictionaryService');
async function inspectDictionaryVersion(endpoint: string): Promise<DictionaryVersion> {
if (endpoint.includes('/rpc')) {
let resp: DictionaryV2Metadata;
try {
resp = await subqlFilterBlocksCapabilities(endpoint);
if (resp.supported.includes('complete')) {
return DictionaryVersion.v2Complete;
} else {
return DictionaryVersion.v2Basic;
}
} catch (e) {
logger.warn(`${e}. Try to use dictionary V1`);
return DictionaryVersion.v1;
}
}
return DictionaryVersion.v1;
}

export abstract class DictionaryService<RFB, FB, DS, D extends DictionaryServiceV1<DS>> {
protected _dictionary: DictionaryServiceV2<RFB, FB, DS> | D | undefined;
private _dictionaryVersion: DictionaryVersion | undefined;
const logger = getLogger('DictionaryService');
export abstract class DictionaryService<RFB, FB, DS, D extends DictionaryV1<DS>>
implements IDictionaryCtrl<DS, FB, IDictionary<DS, FB>>
{
protected _dictionaries: (DictionaryV2<RFB, FB, DS> | D)[] = [];
protected _currentDictionaryIndex: number | undefined;
// Keep in memory in order one of dictionary failed then we can switch
protected _dictionaryV1Endpoints: string[] = [];
protected _dictionaryV2Endpoints: string[] = [];

constructor(
readonly dictionaryEndpoint: string | undefined,
protected chainId: string,
protected readonly nodeConfig: NodeConfig,
protected readonly eventEmitter: EventEmitter2,
protected readonly metadataKeys = ['lastProcessedHeight', 'genesisHash'] // Cosmos uses chain instead of genesisHash
) {}

get version(): DictionaryVersion {
if (!this._dictionaryVersion) {
throw new Error(`Dictionary version not been inspected`);
}
return this._dictionaryVersion;
}

// TODO, we might need to check v1 valid
async inspectDictionaryVersion(): Promise<void> {
if (!(this.nodeConfig.networkDictionary || this.nodeConfig.dictionaryResolver)) {
throw new Error();
}
abstract initDictionariesV1(): Promise<D[]>;
abstract initDictionariesV2(): Promise<DictionaryV2<RFB, FB, DS>[]> | DictionaryV2<RFB, FB, DS>[];

if (this.nodeConfig.networkDictionary && this.nodeConfig.networkDictionary.includes('/rpc')) {
let resp: DictionaryV2Metadata;
try {
resp = await subqlFilterBlocksCapabilities(this.nodeConfig.networkDictionary);
if (resp.supported.includes('complete')) {
this._dictionaryVersion = DictionaryVersion.v2Complete;
async initDictionaries(apiGenesisHash: string): Promise<void> {
// For now, treat dictionary resolver as V1
if (this.nodeConfig.networkDictionaries) {
for (const endpoint of this.nodeConfig.networkDictionaries) {
const version = await inspectDictionaryVersion(endpoint);
if (version === DictionaryVersion.v1) {
this._dictionaryV1Endpoints.push(endpoint);
} else {
this._dictionaryVersion = DictionaryVersion.v2Basic;
this._dictionaryV2Endpoints.push(endpoint);
}
} catch (e) {
logger.warn(`${e}. Try to use dictionary V1`);
this._dictionaryVersion = DictionaryVersion.v1;
}
} else if (this.nodeConfig.dictionaryResolver) {
this._dictionaryVersion = DictionaryVersion.v1;
}
this._dictionaries.push(...(await this.initDictionariesV1()));
this._dictionaries.push(...(await this.initDictionariesV2()));
for (const dictionary of this._dictionaries) {
dictionary.setApiGenesisHash(apiGenesisHash);
}
}

abstract initDictionary(): Promise<void>;

get dictionary(): DictionaryServiceV2<RFB, FB, DS> | D {
if (!this._dictionary) {
throw new Error(`Dictionary not been init`);
get dictionary(): DictionaryV2<RFB, FB, DS> | D {
if (this._dictionaries.length === 0) {
throw new Error(`No dictionaries available to use`);
}
if (this._currentDictionaryIndex === undefined || this._currentDictionaryIndex <= 0) {
throw new Error(`Dictionary index is not set`);
}
return this._dictionary;
return this._dictionaries[this._currentDictionaryIndex];
}

get useDictionary(): boolean {
return (!!this.dictionaryEndpoint || !!this.nodeConfig.dictionaryResolver) && !!this.dictionary.metadataValid;
if (
(!!this.nodeConfig.networkDictionaries || !!this.nodeConfig.dictionaryResolver) &&
this._currentDictionaryIndex !== undefined
) {
return !!this.dictionary.metadataValid;
}
return false;
}

/**
*
* @param genesisHash
* @protected
*/
async initValidation(genesisHash: string): Promise<boolean> {
await this.dictionary.initMetadata();
return this.dictionary.metadataValidation(genesisHash);
async findDictionary(height: number): Promise<void> {
if (this._dictionaries.length === 0) {
return;
}
// update dictionary metadata
for (const dictionary of this._dictionaries) {
await dictionary.initMetadata();
dictionary.heightValidation(height);
}
const v2Index = this._dictionaries?.findIndex((d) => d.version === DictionaryVersion.v2Complete && d.metadataValid);
const v1Index = (this._currentDictionaryIndex = this._dictionaries?.findIndex(
(d) => d.version === DictionaryVersion.v1 && d.metadataValid
));
// Prioritise v2
this._currentDictionaryIndex = v2Index >= 0 ? v2Index : v1Index >= 0 ? v1Index : undefined;
}

/**
Expand All @@ -88,17 +113,18 @@ export abstract class DictionaryService<RFB, FB, DS, D extends DictionaryService
*/

buildDictionaryEntryMap(dataSources: BlockHeightMap<DS[]>): void {
// this.dictionary.buildDictionaryQueryEntries(dataSources);

this.dictionary.updateQueriesMap(dataSources);
if (this.useDictionary) {
this.dictionary.updateQueriesMap(dataSources);
}
return;
}

async scopedDictionaryEntries(
startBlockHeight: number,
queryEndBlock: number,
scaledBatchSize: number
): Promise<(Dictionary<number | FB> & {queryEndBlock: number}) | undefined> {
const dict = await this.dictionary.getDictionary(startBlockHeight, queryEndBlock, scaledBatchSize);
): Promise<(DictionaryResponse<number | FB> & {queryEndBlock: number}) | undefined> {
const dict = await this.dictionary.getData(startBlockHeight, queryEndBlock, scaledBatchSize);
// Check undefined
if (!dict) return undefined;

Expand Down
38 changes: 34 additions & 4 deletions packages/node-core/src/indexer/dictionary/types.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
// SPDX-License-Identifier: GPL-3.0

import {DictionaryQueryEntry as DictionaryV1QueryEntry} from '@subql/types-core/dist/project/types';
import {MetaData as DictionaryV1Metadata} from '@subql/utils';
import {DictionaryV2Metadata} from './';
import {BlockHeightMap} from '../../utils/blockHeightMap';
import {DictionaryV2Metadata, DictionaryV2QueryEntry} from './';

export type Dictionary<B> = {
export type DictionaryResponse<B> = {
batchBlocks: number[] | B[];
lastBufferedHeight: number;
};
Expand All @@ -15,11 +17,39 @@ export enum DictionaryVersion {
v2Complete = 'v2Complete',
}

export interface DictionaryServiceInterface {
export interface IDictionary<DS, FB> {
getData(
startBlock: number,
queryEndBlock: number,
limit: number
): Promise<DictionaryResponse<FB | number> | undefined>;
initMetadata(): Promise<void>;
metadata: DictionaryV1Metadata | DictionaryV2Metadata;
dictionaryValidation(metaData?: DictionaryV1Metadata | DictionaryV2Metadata, startBlockHeight?: number): boolean;
// buildDictionaryQueryEntries(dataSources: DS[]): DictionaryV1QueryEntry[] | DictionaryV2QueryEntry;
buildDictionaryQueryEntries(dataSources: DS[]): DictionaryV1QueryEntry[] | DictionaryV2QueryEntry;
queryMapValidByHeight(height: number): boolean;
getQueryEndBlock(startHeight: number, apiFinalizedHeight: number): number;

setApiGenesisHash(genesisHash: string): void;
setDictionaryStartHeight(start: number | undefined): void;
version: DictionaryVersion;
startHeight: number;
useDictionary: boolean;
heightValidation(height: number): boolean;
updateQueriesMap(dataSources: BlockHeightMap<DS[]>): void;
}

export interface IDictionaryCtrl<DS, FB, D extends IDictionary<DS, FB>> {
initDictionariesV1(): Promise<D[]>;
initDictionariesV2(): Promise<D[]> | D[];
initDictionaries(apiGenesisHash: string): Promise<void>;
dictionary: D;
useDictionary: boolean;
findDictionary(height: number): Promise<void>;
buildDictionaryEntryMap(dataSources: BlockHeightMap<DS[]>): void;
scopedDictionaryEntries(
startBlockHeight: number,
queryEndBlock: number,
scaledBatchSize: number
): Promise<(DictionaryResponse<number | FB> & {queryEndBlock: number}) | undefined>;
}
1 change: 1 addition & 0 deletions packages/node-core/src/indexer/dictionary/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export async function subqlFilterBlocksCapabilities(
},
});
const metadata: DictionaryV2Metadata = {
chain: '', //TODO, need chain for v2 meta
start: response.data.result.availableBlocks[0].startHeight,
end: response.data.result.availableBlocks[0].endHeight,
genesisHash: response.data.result.genesisHash,
Expand Down
Loading

0 comments on commit 72168b9

Please sign in to comment.