From cec50c1eb0b0edb2fd875b0af9f231c463771d84 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 8 Sep 2023 13:19:39 +0800 Subject: [PATCH 1/7] feat: load blocks from db --- packages/chopsticks/src/cli.ts | 3 +++ packages/chopsticks/src/context.ts | 11 ++++++++++ packages/chopsticks/src/schema/index.ts | 1 + packages/e2e/src/blocks-save.test.ts | 29 ++++++++++++++++++------- packages/testing/src/index.ts | 3 +++ 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/packages/chopsticks/src/cli.ts b/packages/chopsticks/src/cli.ts index 50d18ab1..ce97ffa1 100644 --- a/packages/chopsticks/src/cli.ts +++ b/packages/chopsticks/src/cli.ts @@ -44,6 +44,9 @@ const commands = yargs(hideBin(process.argv)) desc: 'Max memory block count', number: true, }, + resume: { + desc: 'Resume from the lastest block saved in the db, note this will override the block option', + }, }), async (argv) => { await setupWithServer(argv as Config) diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index 6bedd962..dcf8f2f4 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -1,4 +1,5 @@ import './utils/tunnel' +import { BlockEntity } from '@acala-network/chopsticks-core/db/entities' import { Config } from './schema' import { HexString } from '@polkadot/util/types' import { overrideStorage, overrideWasm } from './utils/override' @@ -34,5 +35,15 @@ export const setupContext = async (argv: Config, overrideParent = false) => { await overrideStorage(chain, argv['import-storage'], at) await overrideWasm(chain, argv['wasm-override'], at) + // load blocks from db + if (chain.db && argv.resume) { + const blocks = await chain.db.getRepository(BlockEntity).find({ where: {}, order: { number: 'asc' } }) + let head + for (const block of blocks) { + head = await chain.loadBlockFromDB(block.number) + } + await chain.setHead(head) + } + return { chain } } diff --git a/packages/chopsticks/src/schema/index.ts b/packages/chopsticks/src/schema/index.ts index f2536df1..70b856c8 100644 --- a/packages/chopsticks/src/schema/index.ts +++ b/packages/chopsticks/src/schema/index.ts @@ -22,6 +22,7 @@ export const configSchema = z 'registered-types': z.any().optional(), 'runtime-log-level': z.number().min(0).max(5).optional(), 'offchain-worker': z.boolean().optional(), + resume: z.boolean().optional(), }) .strict() diff --git a/packages/e2e/src/blocks-save.test.ts b/packages/e2e/src/blocks-save.test.ts index 929d8137..0726287c 100644 --- a/packages/e2e/src/blocks-save.test.ts +++ b/packages/e2e/src/blocks-save.test.ts @@ -1,19 +1,18 @@ -import { afterAll, assert, describe, expect, it } from 'vitest' +import { assert, describe, expect, it } from 'vitest' import { resolve } from 'node:path' import { tmpdir } from 'node:os' import networks from './networks' describe('block-save', async () => { - const acala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite') }) - const { chain, dev } = acala - - afterAll(async () => { - await acala.teardown() - }) + let savedBlockHash: string it('saved blocks data', async () => { - await dev.newBlock({ count: 2 }) + const acala = await networks.acala({ db: resolve(tmpdir(), 'testdb.sqlite') }) + const { chain, dev } = acala + const blockNumber = chain.head.number + await dev.newBlock({ count: 2 }) + expect(chain.head.number).eq(blockNumber + 2) const numberOfBlocks = await chain.db!.getRepository('Block').count() expect(numberOfBlocks).toEqual(2) @@ -25,5 +24,19 @@ describe('block-save', async () => { expect(blockData.parentHash).toEqual((await block.parentBlock)!.hash) expect(JSON.stringify(blockData.extrinsics)).toEqual(JSON.stringify(await block.extrinsics)) expect(JSON.stringify(blockData.storageDiff)).toEqual(JSON.stringify(await block.storageDiff())) + + savedBlockHash = block.hash + + await acala.teardown() + }) + + it('load chain from the saved blocks', async () => { + const acala = await networks.acala({ db: resolve(tmpdir(), 'testdb.sqlite'), resume: true }) + const { chain } = acala + const head = await chain.getBlockAt(chain.head.number) + + expect(head?.hash).toEqual(savedBlockHash) + + await acala.teardown() }) }) diff --git a/packages/testing/src/index.ts b/packages/testing/src/index.ts index 38c81062..34b0a1e3 100644 --- a/packages/testing/src/index.ts +++ b/packages/testing/src/index.ts @@ -25,6 +25,7 @@ export type SetupOption = { timeout?: number port?: number maxMemoryBlockCount?: number + resume?: boolean } export type SetupConfig = Config & { @@ -40,6 +41,7 @@ export const createConfig = ({ timeout, port, maxMemoryBlockCount, + resume, }: SetupOption): SetupConfig => { // random port if not specified port = port ?? Math.floor(Math.random() * 10000) + 10000 @@ -53,6 +55,7 @@ export const createConfig = ({ db, 'wasm-override': wasmOverride, timeout, + resume: resume ?? false, } return config } From a995fb5d3f2507cf624ba4ccdcd114568cf268fc Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 8 Sep 2023 19:41:06 +0800 Subject: [PATCH 2/7] fix load blocks --- packages/chopsticks/src/cli.ts | 1 + packages/chopsticks/src/context.ts | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/chopsticks/src/cli.ts b/packages/chopsticks/src/cli.ts index ce97ffa1..b44263c4 100644 --- a/packages/chopsticks/src/cli.ts +++ b/packages/chopsticks/src/cli.ts @@ -46,6 +46,7 @@ const commands = yargs(hideBin(process.argv)) }, resume: { desc: 'Resume from the lastest block saved in the db, note this will override the block option', + boolean: true, }, }), async (argv) => { diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index dcf8f2f4..5c48c4f9 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -36,13 +36,22 @@ export const setupContext = async (argv: Config, overrideParent = false) => { await overrideWasm(chain, argv['wasm-override'], at) // load blocks from db - if (chain.db && argv.resume) { - const blocks = await chain.db.getRepository(BlockEntity).find({ where: {}, order: { number: 'asc' } }) - let head - for (const block of blocks) { - head = await chain.loadBlockFromDB(block.number) + if (chain.db) { + if (argv.resume) { + const blocks = await chain.db.getRepository(BlockEntity).find({ where: {}, order: { number: 'asc' } }) + // validate the first block in db is chain.head+1 and db blocks are consecutive + const canResume = blocks.every((block, index) => block.number === chain.head.number + index + 1) + if (canResume) { + let head + for (const block of blocks) { + head = await chain.loadBlockFromDB(block.number) + } + await chain.setHead(head) + } + } else { + // starting without resume should clear blocks from db + await chain.db.getRepository(BlockEntity).clear() } - await chain.setHead(head) } return { chain } From 42b972735e2d15961f4606fec6f02544a88d50e1 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 11 Sep 2023 10:20:53 +0800 Subject: [PATCH 3/7] fix: make tests independent --- packages/e2e/src/blocks-save.test.ts | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/e2e/src/blocks-save.test.ts b/packages/e2e/src/blocks-save.test.ts index 0726287c..2c284f67 100644 --- a/packages/e2e/src/blocks-save.test.ts +++ b/packages/e2e/src/blocks-save.test.ts @@ -4,20 +4,17 @@ import { tmpdir } from 'node:os' import networks from './networks' describe('block-save', async () => { - let savedBlockHash: string - it('saved blocks data', async () => { - const acala = await networks.acala({ db: resolve(tmpdir(), 'testdb.sqlite') }) + const acala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite') }) const { chain, dev } = acala - const blockNumber = chain.head.number - await dev.newBlock({ count: 2 }) - expect(chain.head.number).eq(blockNumber + 2) + const numberOfBlocks = await chain.db!.getRepository('Block').count() expect(numberOfBlocks).toEqual(2) const block = await chain.getBlockAt(chain.head.number) const blockData = await chain.db!.getRepository('Block').findOne({ where: { number: chain.head.number } }) + assert(block && blockData, 'block and blockData should be defined') expect(blockData.hash).toEqual(block.hash) expect(JSON.stringify(blockData.header)).toEqual(JSON.stringify(block.header)) @@ -25,18 +22,24 @@ describe('block-save', async () => { expect(JSON.stringify(blockData.extrinsics)).toEqual(JSON.stringify(await block.extrinsics)) expect(JSON.stringify(blockData.storageDiff)).toEqual(JSON.stringify(await block.storageDiff())) - savedBlockHash = block.hash - await acala.teardown() }) it('load chain from the saved blocks', async () => { - const acala = await networks.acala({ db: resolve(tmpdir(), 'testdb.sqlite'), resume: true }) - const { chain } = acala + // save blocks + const acala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite') }) + const { chain, dev } = acala + await dev.newBlock({ count: 2 }) const head = await chain.getBlockAt(chain.head.number) + const savedHeadHash = head?.hash + await acala.teardown() - expect(head?.hash).toEqual(savedBlockHash) + // load blocks + const newAcala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite'), resume: true }) + const newHeadNumber = newAcala.chain.head.number + const loadedHead = await newAcala.chain.getBlockAt(newHeadNumber) - await acala.teardown() + expect(loadedHead?.hash).toEqual(savedHeadHash) + await newAcala.teardown() }) }) From 93fdf52184fb524112588dec8f6cc4d35d11fb6f Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 11 Sep 2023 12:26:33 +0800 Subject: [PATCH 4/7] fix: override storage after load blocks --- packages/chopsticks/src/context.ts | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index 5c48c4f9..5ad3e201 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -20,27 +20,12 @@ export const setupContext = async (argv: Config, overrideParent = false) => { maxMemoryBlockCount: argv['max-memory-block-count'], }) - if (argv.timestamp) await timeTravel(chain, argv.timestamp) - - let at: HexString | undefined - if (overrideParent) { - // in case of run block we need to apply wasm-override and import-storage to parent block - const block = await chain.head.parentBlock - if (!block) throw new Error('Cannot find parent block') - at = block.hash - } - - // override wasm before importing storage, in case new pallets have been - // added that have storage imports - await overrideStorage(chain, argv['import-storage'], at) - await overrideWasm(chain, argv['wasm-override'], at) - // load blocks from db if (chain.db) { if (argv.resume) { const blocks = await chain.db.getRepository(BlockEntity).find({ where: {}, order: { number: 'asc' } }) // validate the first block in db is chain.head+1 and db blocks are consecutive - const canResume = blocks.every((block, index) => block.number === chain.head.number + index + 1) + const canResume = blocks.length && blocks.every((block, index) => block.number === chain.head.number + index + 1) if (canResume) { let head for (const block of blocks) { @@ -54,5 +39,20 @@ export const setupContext = async (argv: Config, overrideParent = false) => { } } + if (argv.timestamp) await timeTravel(chain, argv.timestamp) + + let at: HexString | undefined + if (overrideParent) { + // in case of run block we need to apply wasm-override and import-storage to parent block + const block = await chain.head.parentBlock + if (!block) throw new Error('Cannot find parent block') + at = block.hash + } + + // override wasm before importing storage, in case new pallets have been + // added that have storage imports + await overrideStorage(chain, argv['import-storage'], at) + await overrideWasm(chain, argv['wasm-override'], at) + return { chain } } From ce6a7d1cc9ea7ed3ca735126559f0fe87fdabbcd Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 11 Sep 2023 14:30:29 +0800 Subject: [PATCH 5/7] fix: storage --- packages/core/src/blockchain/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/blockchain/index.ts b/packages/core/src/blockchain/index.ts index 31184895..775d10b6 100644 --- a/packages/core/src/blockchain/index.ts +++ b/packages/core/src/blockchain/index.ts @@ -156,6 +156,7 @@ export class Blockchain { const block = new Block(this, number, hash, parentBlock, { header: registry.createType
('Header', header), extrinsics, + storage: parentBlock?.storage, storageDiff, }) this.#registerBlock(block) From 82f1d21ea04b91f8d30f31ce53e68ca9ccfc182b Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 11 Sep 2023 15:17:52 +0800 Subject: [PATCH 6/7] fix: load latest block only --- packages/chopsticks/src/context.ts | 16 +++++----------- packages/core/src/blockchain/index.ts | 8 ++++++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index 5ad3e201..92c6250d 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -20,21 +20,15 @@ export const setupContext = async (argv: Config, overrideParent = false) => { maxMemoryBlockCount: argv['max-memory-block-count'], }) - // load blocks from db + // load block from db if (chain.db) { if (argv.resume) { - const blocks = await chain.db.getRepository(BlockEntity).find({ where: {}, order: { number: 'asc' } }) - // validate the first block in db is chain.head+1 and db blocks are consecutive - const canResume = blocks.length && blocks.every((block, index) => block.number === chain.head.number + index + 1) - if (canResume) { - let head - for (const block of blocks) { - head = await chain.loadBlockFromDB(block.number) - } - await chain.setHead(head) + const blockData = await chain.db.getRepository(BlockEntity).findOne({ where: {}, order: { number: 'desc' } }) + if (blockData) { + const block = await chain.loadBlockFromDB(blockData?.number) + block && (await chain.setHead(block)) } } else { - // starting without resume should clear blocks from db await chain.db.getRepository(BlockEntity).clear() } } diff --git a/packages/core/src/blockchain/index.ts b/packages/core/src/blockchain/index.ts index 775d10b6..58d36a08 100644 --- a/packages/core/src/blockchain/index.ts +++ b/packages/core/src/blockchain/index.ts @@ -149,8 +149,12 @@ export class Blockchain { .getRepository(BlockEntity) .findOne({ where: { [typeof key === 'number' ? 'number' : 'hash']: key } }) if (blockData) { - const { hash, number, header, extrinsics, parentHash } = blockData - const parentBlock = parentHash ? this.#blocksByHash[parentHash] : undefined + const { hash, number, header, extrinsics } = blockData + const parentHash = blockData.parentHash || undefined + let parentBlock = parentHash ? this.#blocksByHash[parentHash] : undefined + if (!parentBlock) { + parentBlock = await this.getBlock(parentHash) + } const storageDiff = blockData.storageDiff ?? undefined const registry = await this.head.registry const block = new Block(this, number, hash, parentBlock, { From 655cf7b539745c3347be0b66e121017221ecb4e9 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 11 Sep 2023 21:01:02 +0800 Subject: [PATCH 7/7] feat: resume using block number hash --- packages/chopsticks/src/cli.ts | 6 ++-- packages/chopsticks/src/context.ts | 15 ++++++-- packages/chopsticks/src/schema/index.ts | 2 +- packages/e2e/src/blocks-save.test.ts | 47 ++++++++++++++++++++----- packages/testing/src/index.ts | 2 +- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/packages/chopsticks/src/cli.ts b/packages/chopsticks/src/cli.ts index b44263c4..5edaabc8 100644 --- a/packages/chopsticks/src/cli.ts +++ b/packages/chopsticks/src/cli.ts @@ -45,8 +45,10 @@ const commands = yargs(hideBin(process.argv)) number: true, }, resume: { - desc: 'Resume from the lastest block saved in the db, note this will override the block option', - boolean: true, + desc: `Resume from the specified block hash or block number in db. + If true, it will resume from the latest block in db. + Note this will override the block option`, + string: true, }, }), async (argv) => { diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index 92c6250d..323c629a 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -23,13 +23,22 @@ export const setupContext = async (argv: Config, overrideParent = false) => { // load block from db if (chain.db) { if (argv.resume) { - const blockData = await chain.db.getRepository(BlockEntity).findOne({ where: {}, order: { number: 'desc' } }) + const where: Record = {} + switch (typeof argv.resume) { + case 'string': + where.hash = argv.resume + break + case 'number': + where.number = argv.resume + break + default: + break + } + const blockData = await chain.db.getRepository(BlockEntity).findOne({ where, order: { number: 'desc' } }) if (blockData) { const block = await chain.loadBlockFromDB(blockData?.number) block && (await chain.setHead(block)) } - } else { - await chain.db.getRepository(BlockEntity).clear() } } diff --git a/packages/chopsticks/src/schema/index.ts b/packages/chopsticks/src/schema/index.ts index 70b856c8..6650373f 100644 --- a/packages/chopsticks/src/schema/index.ts +++ b/packages/chopsticks/src/schema/index.ts @@ -22,7 +22,7 @@ export const configSchema = z 'registered-types': z.any().optional(), 'runtime-log-level': z.number().min(0).max(5).optional(), 'offchain-worker': z.boolean().optional(), - resume: z.boolean().optional(), + resume: z.union([z.string().length(66).startsWith('0x'), z.number(), z.boolean()]).optional(), }) .strict() diff --git a/packages/e2e/src/blocks-save.test.ts b/packages/e2e/src/blocks-save.test.ts index 2c284f67..c19085e0 100644 --- a/packages/e2e/src/blocks-save.test.ts +++ b/packages/e2e/src/blocks-save.test.ts @@ -4,6 +4,18 @@ import { tmpdir } from 'node:os' import networks from './networks' describe('block-save', async () => { + const buildBlocks = async () => { + // save blocks + const acala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite') }) + const { chain, dev } = acala + await dev.newBlock({ count: 2 }) + const head = await chain.getBlockAt(chain.head.number) + const savedHeadHash = head?.hash + await acala.teardown() + + return savedHeadHash + } + it('saved blocks data', async () => { const acala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite') }) const { chain, dev } = acala @@ -25,16 +37,10 @@ describe('block-save', async () => { await acala.teardown() }) - it('load chain from the saved blocks', async () => { - // save blocks - const acala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite') }) - const { chain, dev } = acala - await dev.newBlock({ count: 2 }) - const head = await chain.getBlockAt(chain.head.number) - const savedHeadHash = head?.hash - await acala.teardown() + it('load chain using the latest saved block', async () => { + const savedHeadHash = await buildBlocks() - // load blocks + // load block const newAcala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite'), resume: true }) const newHeadNumber = newAcala.chain.head.number const loadedHead = await newAcala.chain.getBlockAt(newHeadNumber) @@ -42,4 +48,27 @@ describe('block-save', async () => { expect(loadedHead?.hash).toEqual(savedHeadHash) await newAcala.teardown() }) + + it('load chain using a block number', async () => { + await buildBlocks() + + // load blocks + const newAcala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite'), resume: 3000001 }) + const newHeadNumber = newAcala.chain.head.number + + expect(newHeadNumber).toEqual(3000001) + await newAcala.teardown() + }) + + it('load chain using a block hash', async () => { + const savedHeadHash = await buildBlocks() + + // load blocks + const newAcala = await networks.acala({ db: resolve(tmpdir(), 'db.sqlite'), resume: savedHeadHash }) + const newHeadNumber = newAcala.chain.head.number + const loadedHead = await newAcala.chain.getBlockAt(newHeadNumber) + + expect(loadedHead?.hash).toEqual(savedHeadHash) + await newAcala.teardown() + }) }) diff --git a/packages/testing/src/index.ts b/packages/testing/src/index.ts index 34b0a1e3..ebb44ef0 100644 --- a/packages/testing/src/index.ts +++ b/packages/testing/src/index.ts @@ -25,7 +25,7 @@ export type SetupOption = { timeout?: number port?: number maxMemoryBlockCount?: number - resume?: boolean + resume?: boolean | HexString | number } export type SetupConfig = Config & {