From f57e768246aa28c2a97065b081e6d3f8c6fa4871 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 5 Aug 2024 12:20:43 +0500 Subject: [PATCH 1/7] Processors base class --- src/processors/processor.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/processors/processor.ts diff --git a/src/processors/processor.ts b/src/processors/processor.ts new file mode 100644 index 0000000..70b35cf --- /dev/null +++ b/src/processors/processor.ts @@ -0,0 +1,17 @@ +import { assertNotNull } from '@subsquid/substrate-processor' +import { BlockHeader, Event } from '@subsquid/substrate-processor' + +export abstract class BaseProcessor< + State extends + | Map + | Set + | { [key: string]: Map | Set }, +> { + constructor(protected _state: State) {} + + get state(): State { + return assertNotNull(this.state) + } + + abstract process(event: Event, block: BlockHeader): Promise +} From 7d63317d1cc32fc83d56c9de5290d84d66f1daf2 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 5 Aug 2024 12:23:58 +0500 Subject: [PATCH 2/7] Inherit processors from the base class --- src/main.ts | 10 +++---- src/processors/cereBalancesProcessor.ts | 15 +++++----- src/processors/ddcBalancesProcessor.ts | 15 +++++----- src/processors/ddcBucketsProcessor.ts | 15 +++++----- src/processors/ddcClustersProcessor.ts | 15 +++++----- src/processors/ddcNodesProcessor.ts | 38 ++++++++++++------------- 6 files changed, 56 insertions(+), 52 deletions(-) diff --git a/src/main.ts b/src/main.ts index 275cdf0..5096bc4 100644 --- a/src/main.ts +++ b/src/main.ts @@ -35,11 +35,11 @@ processor.run(new TypeormDatabase({ supportHotBlocks: true }), async (ctx) => { } // retrieving state from processors - const accountToCereBalance = cereBalancesProcessor.getState() - const accountToDdcBalance = ddcBalancesProcessor.getState() - const ddcClusters = ddcClustersProcessor.getState() - const ddcNodes = ddcNodesProcessor.getState() - const ddcBuckets = ddcBucketsProcessor.getState() + const accountToCereBalance = cereBalancesProcessor.state + const accountToDdcBalance = ddcBalancesProcessor.state + const ddcClusters = ddcClustersProcessor.state + const ddcNodes = ddcNodesProcessor.state + const ddcBuckets = ddcBucketsProcessor.state // create missing accounts const accounts = new Map() diff --git a/src/processors/cereBalancesProcessor.ts b/src/processors/cereBalancesProcessor.ts index 171b0fd..61d0e1a 100644 --- a/src/processors/cereBalancesProcessor.ts +++ b/src/processors/cereBalancesProcessor.ts @@ -6,9 +6,14 @@ import { throwUnsupportedStorageSpec, toCereAddress, } from '../utils' +import { BaseProcessor } from './processor' -export class CereBalancesProcessor { - private state = new Map() +type State = Map + +export class CereBalancesProcessor extends BaseProcessor { + constructor() { + super(new Map()) + } private async processBalancesEvent(accountId: string, block: BlockHeader) { try { @@ -32,7 +37,7 @@ export class CereBalancesProcessor { throwUnsupportedStorageSpec(block) } if (accountInStorage) { - this.state.set( + this._state.set( toCereAddress(accountId), accountInStorage.data.free, ) @@ -48,10 +53,6 @@ export class CereBalancesProcessor { } } - getState(): Map { - return this.state - } - async process(event: Event, block: BlockHeader) { switch (event.name) { case events.balances.endowed.name: { diff --git a/src/processors/ddcBalancesProcessor.ts b/src/processors/ddcBalancesProcessor.ts index 5cb2907..56c86df 100644 --- a/src/processors/ddcBalancesProcessor.ts +++ b/src/processors/ddcBalancesProcessor.ts @@ -6,9 +6,14 @@ import { throwUnsupportedStorageSpec, toCereAddress, } from '../utils' +import { BaseProcessor } from './processor' -export class DdcBalancesProcessor { - private state = new Map() +type State = Map + +export class DdcBalancesProcessor extends BaseProcessor { + constructor() { + super(new Map()) + } private async processDdcCustomersBalancesEvents( accountId: string, @@ -24,16 +29,12 @@ export class DdcBalancesProcessor { throwUnsupportedStorageSpec(block) } if (accountInStorage) { - this.state.set(toCereAddress(accountId), accountInStorage.active) + this._state.set(toCereAddress(accountId), accountInStorage.active) } else { logStorageError('DDC Customer ledger', accountId, block) } } - getState(): Map { - return this.state - } - async process(event: Event, block: BlockHeader) { switch (event.name) { case events.ddcCustomers.deposited.name: { diff --git a/src/processors/ddcBucketsProcessor.ts b/src/processors/ddcBucketsProcessor.ts index 76926a7..3fdced5 100644 --- a/src/processors/ddcBucketsProcessor.ts +++ b/src/processors/ddcBucketsProcessor.ts @@ -6,6 +6,7 @@ import { throwUnsupportedStorageSpec, toCereAddress, } from '../utils' +import { BaseProcessor } from './processor' export interface DdcBucketInfo { ownerId: string @@ -19,8 +20,12 @@ export interface DdcBucketInfo { numberOfGets: bigint } -export class DdcBucketsProcessor { - private state = new Map() +type State = Map + +export class DdcBucketsProcessor extends BaseProcessor { + constructor() { + super(new Map()) + } private async processDdcBucketsEvents( bucketId: bigint, @@ -107,16 +112,12 @@ export class DdcBucketsProcessor { } if (bucketInfo) { bucketInfo.ownerId = toCereAddress(bucketInfo.ownerId) - this.state.set(bucketId, bucketInfo) + this._state.set(bucketId, bucketInfo) } else { logStorageError('bucket', bucketId, block) } } - getState(): Map { - return this.state - } - async process(event: Event, block: BlockHeader) { switch (event.name) { case events.ddcCustomers.bucketCreated.name: { diff --git a/src/processors/ddcClustersProcessor.ts b/src/processors/ddcClustersProcessor.ts index c94694a..201df99 100644 --- a/src/processors/ddcClustersProcessor.ts +++ b/src/processors/ddcClustersProcessor.ts @@ -7,6 +7,7 @@ import { toCereAddress, } from '../utils' import { DdcClusterStatus } from '../model' +import { BaseProcessor } from './processor' export interface DdcClusterInfo { id: string @@ -30,8 +31,12 @@ export interface DdcClusterInfo { status: DdcClusterStatus } -export class DdcClustersProcessor { - private state = new Map() +type State = Map + +export class DdcClustersProcessor extends BaseProcessor { + constructor() { + super(new Map()) + } private newClusterInfo( clusterId: string, @@ -173,16 +178,12 @@ export class DdcClustersProcessor { clusterInfo.unitPerGetRequest = clusterGovParams.unitPerGetRequest } - this.state.set(clusterId, clusterInfo) + this._state.set(clusterId, clusterInfo) } else { logStorageError('DDC cluster', clusterId, block) } } - getState(): Map { - return this.state - } - async process(event: Event, block: BlockHeader) { switch (event.name) { case events.ddcClusters.clusterCreated.name: { diff --git a/src/processors/ddcNodesProcessor.ts b/src/processors/ddcNodesProcessor.ts index 01f8ebc..4ab5160 100644 --- a/src/processors/ddcNodesProcessor.ts +++ b/src/processors/ddcNodesProcessor.ts @@ -7,8 +7,9 @@ import { toCereAddress, } from '../utils' import { DdcNodeMode } from '../model' +import { BaseProcessor } from './processor' -export interface DdcNodeInfo { +interface DdcNodeInfo { id: string providerId: string @@ -23,19 +24,21 @@ export interface DdcNodeInfo { mode: DdcNodeMode } -export interface State { +type State = { addedToCluster: Map> removedFromCluster: Map> updatedNodes: Map removedNodes: Set } -export class DdcNodesProcessor { - private state: State = { - addedToCluster: new Map>(), - removedFromCluster: new Map>(), - updatedNodes: new Map(), - removedNodes: new Set(), +export class DdcNodesProcessor extends BaseProcessor { + constructor() { + super({ + addedToCluster: new Map>(), + removedFromCluster: new Map>(), + updatedNodes: new Map(), + removedNodes: new Set(), + }) } private async processDdcNodesEvents(nodeId: string, block: BlockHeader) { @@ -140,16 +143,12 @@ export class DdcNodesProcessor { } if (nodeInfo) { nodeInfo.providerId = toCereAddress(nodeInfo.providerId) - this.state.updatedNodes.set(nodeId, nodeInfo) + this._state.updatedNodes.set(nodeId, nodeInfo) } else { logStorageError('DDC node', nodeId, block) } } - getState(): State { - return this.state - } - async process(event: Event, block: BlockHeader) { switch (event.name) { case events.ddcClusters.clusterNodeAdded.name: { @@ -167,10 +166,11 @@ export class DdcNodesProcessor { } if (decodedEvent) { const nodesInCluster = - this.state.addedToCluster.get(decodedEvent.clusterId) ?? - new Set() + this._state.addedToCluster.get( + decodedEvent.clusterId, + ) ?? new Set() nodesInCluster.add(decodedEvent.nodePubKey.value) - this.state.addedToCluster.set( + this._state.addedToCluster.set( decodedEvent.clusterId, nodesInCluster, ) @@ -196,11 +196,11 @@ export class DdcNodesProcessor { } if (decodedEvent) { const nodesRemovedFromCluster = - this.state.removedFromCluster.get( + this._state.removedFromCluster.get( decodedEvent.clusterId, ) ?? new Set() nodesRemovedFromCluster.add(decodedEvent.nodePubKey.value) - this.state.removedFromCluster.set( + this._state.removedFromCluster.set( decodedEvent.clusterId, nodesRemovedFromCluster, ) @@ -253,7 +253,7 @@ export class DdcNodesProcessor { throwUnsupportedSpec(event, block) } if (removedNode) { - this.state.removedNodes.add(removedNode) + this._state.removedNodes.add(removedNode) } break } From d21ee5f567b4bc401c8164c90130a4650ebe5f1a Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 5 Aug 2024 12:39:51 +0500 Subject: [PATCH 3/7] Replace sequential awaits with `Promise.all()` --- src/main.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main.ts b/src/main.ts index 5096bc4..cda55bf 100644 --- a/src/main.ts +++ b/src/main.ts @@ -26,11 +26,14 @@ processor.run(new TypeormDatabase({ supportHotBlocks: true }), async (ctx) => { logger.debug( `Received event ${event.name} at block ${block.height} (${block.hash})`, ) - await cereBalancesProcessor.process(event, block) - await ddcBalancesProcessor.process(event, block) - await ddcClustersProcessor.process(event, block) - await ddcNodesProcessor.process(event, block) - await ddcBucketsProcessor.process(event, block) + + await Promise.all([ + cereBalancesProcessor.process(event, block), + ddcBalancesProcessor.process(event, block), + ddcClustersProcessor.process(event, block), + ddcNodesProcessor.process(event, block), + ddcBucketsProcessor.process(event, block), + ]) } } From 17a661e1843a48bfa2825755122bfb7c8ce24aa8 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 9 Aug 2024 11:17:21 +0500 Subject: [PATCH 4/7] Remove bucket ID duplication --- db/migrations/1723182802615-Data.js | 15 +++++++++++++++ schema.graphql | 1 - src/main.ts | 1 - src/model/generated/ddcBucket.model.ts | 6 +----- 4 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 db/migrations/1723182802615-Data.js diff --git a/db/migrations/1723182802615-Data.js b/db/migrations/1723182802615-Data.js new file mode 100644 index 0000000..b82fe9e --- /dev/null +++ b/db/migrations/1723182802615-Data.js @@ -0,0 +1,15 @@ +module.exports = class Data1723182802615 { + name = 'Data1723182802615' + + async up(db) { + await db.query(`DROP INDEX "public"."IDX_16f64a1893b3288a5481259a36"`) + await db.query(`ALTER TABLE "ddc_bucket" DROP COLUMN "bucket_id"`) + } + + async down(db) { + await db.query(`CREATE UNIQUE INDEX "IDX_16f64a1893b3288a5481259a36" ON "ddc_bucket" ("bucket_id") `) + await db.query(`ALTER TABLE "ddc_bucket" ADD "bucket_id" numeric`) + await db.query(`UPDATE "ddc_bucket" SET "bucket_id" = CAST("id" as numeric)`) + await db.query(`ALTER TABLE "ddc_bucket" ALTER COLUMN "bucket_id" SET NOT NULL`) + } +} diff --git a/schema.graphql b/schema.graphql index 6dcdd27..ebac1a6 100644 --- a/schema.graphql +++ b/schema.graphql @@ -60,7 +60,6 @@ enum DdcNodeMode { type DdcBucket @entity { id: ID! - bucketId: BigInt! @index @unique ownerId: Account! @index clusterId: DdcCluster! @index diff --git a/src/main.ts b/src/main.ts index cda55bf..fe27a9b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -225,7 +225,6 @@ processor.run(new TypeormDatabase({ supportHotBlocks: true }), async (ctx) => { ddcBucketEntities.push( new DdcBucket({ id: bucket.bucketId.toString(), - bucketId: bucket.bucketId, ownerId: accounts.get(bucket.ownerId), clusterId: cluster, isPublic: bucket.isPublic, diff --git a/src/model/generated/ddcBucket.model.ts b/src/model/generated/ddcBucket.model.ts index 7d11c0c..c711714 100644 --- a/src/model/generated/ddcBucket.model.ts +++ b/src/model/generated/ddcBucket.model.ts @@ -1,4 +1,4 @@ -import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, BigIntColumn as BigIntColumn_, Index as Index_, ManyToOne as ManyToOne_, BooleanColumn as BooleanColumn_} from "@subsquid/typeorm-store" +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, ManyToOne as ManyToOne_, Index as Index_, BooleanColumn as BooleanColumn_, BigIntColumn as BigIntColumn_} from "@subsquid/typeorm-store" import {Account} from "./account.model" import {DdcCluster} from "./ddcCluster.model" @@ -11,10 +11,6 @@ export class DdcBucket { @PrimaryColumn_() id!: string - @Index_({unique: true}) - @BigIntColumn_({nullable: false}) - bucketId!: bigint - @Index_() @ManyToOne_(() => Account, {nullable: true}) ownerId!: Account From a0552bb574b9a7bc1bbce59f2cf5881339f8d79d Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 9 Aug 2024 11:27:26 +0500 Subject: [PATCH 5/7] Remove `migration:clean` step from a generate cmd --- commands.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.json b/commands.json index ac21a21..fad790f 100644 --- a/commands.json +++ b/commands.json @@ -24,7 +24,7 @@ }, "migration:generate": { "description": "Generate a DB migration matching the TypeORM entities", - "deps": ["build", "migration:clean"], + "deps": ["build"], "cmd": ["squid-typeorm-migration", "generate"] }, "migration:clean": { From fec25fea98136a2fe1208469df7a29e4044a1a3c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 9 Aug 2024 11:40:21 +0500 Subject: [PATCH 6/7] Update README.md --- README.md | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 15a9283..833a24a 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,6 @@ # Cere Squid Indexer -This is an indexer for [Cerebellum Network](https://www.cere.network/) build using [Subsquid](https://subsquid.io/) [squid-sdk](https://github.com/subsquid/squid-sdk). - -## Indexed data - -1. Balance transfers, -2. DDC Customers bucket owners relation to bucket IDs. +This is an indexer for [Cerebellum Network](https://www.cere.network/) blockchain built using [Subsquid](https://subsquid.io/) [squid-sdk](https://github.com/subsquid/squid-sdk). ## Usage @@ -16,14 +11,8 @@ Please [install](https://docs.subsquid.io/squid-cli/installation/) it before pro # 1. Install dependencies npm ci -# 2. Start target Postgres database from `docker-compose.yml` and detach +# 2. Start target Postgres database, processor, and GraphQL server from `docker-compose.yml` and detach sqd up - -# 3. Apply database migrations, load .env, and start the squid processor -sqd process - -# 4. Start the GraphQL server -sqd serve ``` A GraphiQL playground will be available at [127.0.0.1:4350/graphql](http://localhost:4350/graphql). @@ -51,25 +40,19 @@ All database changes are applied through migration files located at `db/migratio It is all [TypeORM](https://typeorm.io/#/migrations) under the hood. ```bash +# Update database connection credentials in .env and load them. +source .env + # Connect to database, analyze its state and generate migration to match the target schema. # The target schema is derived from entity classes generated earlier. # Don't forget to compile your entity classes beforehand! -npx squid-typeorm-migration generate - -# Create template file for custom database changes -npx squid-typeorm-migration create - -# Apply database migrations from `db/migrations` -npx squid-typeorm-migration apply - -# Revert the last performed migration -npx squid-typeorm-migration revert +sqd generate ``` ### 4. Extend processor -Now extend the processor adding your events listening and RPC requests to `getEventsInfo` in `src/main.ts` to compose -im-memory data for the indexer and persist it in the `processor.run` callback. +Create or extend an existing processor with new events listening and RPC requests in `src/processors`. +Now you can add processor state persistance to the `processor.run` callback. ## Project conventions @@ -91,5 +74,9 @@ specification file with the runtime metadata. Create blockchain specification file for `typegen.json` using [substrate-metadata-explorer](https://github.com/subsquid/squid-sdk/tree/master/substrate/substrate-metadata-explorer). ```bash -npx @subsquid/substrate-metadata-explorer --rpc $RPC_WS --out specs/out.jsonl +# Export new runtime metadata +npx @subsquid/substrate-metadata-explorer --rpc $RPC_WS --fromBlock $FROM_BLOCK_NUMBER --out specs/out.jsonl + +# Add new runtime metadata to existing file used in `typegen.json` and regenerate types. +npx squid-substrate-typegen ./typegen.json ``` From 9452a101718f1e49749e6b3f77f667a632a9286c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 9 Aug 2024 11:44:59 +0500 Subject: [PATCH 7/7] npm run format --- src/processors/cereBalancesProcessor.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/processors/cereBalancesProcessor.ts b/src/processors/cereBalancesProcessor.ts index 61d0e1a..e92def0 100644 --- a/src/processors/cereBalancesProcessor.ts +++ b/src/processors/cereBalancesProcessor.ts @@ -45,7 +45,10 @@ export class CereBalancesProcessor extends BaseProcessor { logStorageError('account', accountId, block) } } catch (error) { - if (error?.toString() === 'Error: Unexpected EOF' || error?.toString() === 'Error: Unprocessed data left') { + if ( + error?.toString() === 'Error: Unexpected EOF' || + error?.toString() === 'Error: Unprocessed data left' + ) { // some accounts in old blocks can not be parsed, just ignore them } else { throw error