From 0bb59d26ac4349f1d4de8553a039fd3e6b27e578 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 7 Nov 2023 16:49:52 +0100 Subject: [PATCH 1/5] fix plugins --- executor/Cargo.lock | 2 +- executor/Cargo.toml | 2 +- executor/package.json | 2 +- packages/chopsticks/package.json | 2 +- packages/chopsticks/src/cli.ts | 6 ++++- packages/chopsticks/src/context.ts | 5 ++++ packages/chopsticks/src/plugins/index.ts | 32 ++++++++++-------------- packages/chopsticks/src/utils/tunnel.ts | 3 ++- packages/core/package.json | 2 +- packages/db/package.json | 2 +- packages/testing/package.json | 2 +- packages/web-test/src/App.tsx | 3 --- 12 files changed, 32 insertions(+), 31 deletions(-) diff --git a/executor/Cargo.lock b/executor/Cargo.lock index 168b5b62..722d0ee5 100644 --- a/executor/Cargo.lock +++ b/executor/Cargo.lock @@ -165,7 +165,7 @@ dependencies = [ [[package]] name = "chopsticks-executor" -version = "0.9.1-4" +version = "0.9.1-5" dependencies = [ "arrayvec 0.7.4", "console_error_panic_hook", diff --git a/executor/Cargo.toml b/executor/Cargo.toml index 3e97ec3f..3f1bdfd6 100644 --- a/executor/Cargo.toml +++ b/executor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chopsticks-executor" -version = "0.9.1-4" +version = "0.9.1-5" description = "Chopsticks executor" repository = "https://github.com/AcalaNetwork/chopsticks" license = "Apache-2.0" diff --git a/executor/package.json b/executor/package.json index be5a1950..3ff4f6ca 100644 --- a/executor/package.json +++ b/executor/package.json @@ -1,7 +1,7 @@ { "name": "@acala-network/chopsticks-executor", "description": "Chopsticks executor", - "version": "0.9.1-4", + "version": "0.9.1-5", "license": "Apache-2.0", "repository": { "type": "git", diff --git a/packages/chopsticks/package.json b/packages/chopsticks/package.json index dadd3c00..638e415e 100644 --- a/packages/chopsticks/package.json +++ b/packages/chopsticks/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks", - "version": "0.9.1-4", + "version": "0.9.1-5", "author": "Acala Developers ", "license": "Apache-2.0", "bin": "./chopsticks.cjs", diff --git a/packages/chopsticks/src/cli.ts b/packages/chopsticks/src/cli.ts index f1a9efb4..7f729c41 100644 --- a/packages/chopsticks/src/cli.ts +++ b/packages/chopsticks/src/cli.ts @@ -107,4 +107,8 @@ const commands = yargs(hideBin(process.argv)) .usage('Usage: $0 [options]') .example('$0', '-c acala') -pluginExtendCli(commands).then(() => commands.parse()) +if (!process.env.NO_PLUGINS) { + pluginExtendCli(commands).then(() => commands.parse()) +} else { + commands.parse() +} diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index 2b8872c7..93646a05 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -3,6 +3,7 @@ import { BlockEntry, GenesisProvider, defaultLogger, isUrl, setup, timeTravel } import { Config } from './schema/index.js' import { HexString } from '@polkadot/util/types' import { SqliteDatabase } from '@acala-network/chopsticks-db' +import { loadRPCPlugins } from './plugins/index.js' import { overrideStorage, overrideWasm } from './utils/override.js' import axios from 'axios' @@ -87,5 +88,9 @@ export const setupContext = async (argv: Config, overrideParent = false) => { await overrideWasm(chain, argv['wasm-override'], at) await overrideStorage(chain, argv['import-storage'], at) + if (!process.env.NO_PLUGINS) { + await loadRPCPlugins() + } + return { chain } } diff --git a/packages/chopsticks/src/plugins/index.ts b/packages/chopsticks/src/plugins/index.ts index deef849c..b64ec046 100644 --- a/packages/chopsticks/src/plugins/index.ts +++ b/packages/chopsticks/src/plugins/index.ts @@ -1,4 +1,5 @@ import { Handlers } from '@acala-network/chopsticks-core' +import { lstatSync, readdirSync } from 'fs' import _ from 'lodash' import type { Argv } from 'yargs' @@ -8,37 +9,30 @@ const logger = defaultLogger.child({ name: 'plugin' }) export const pluginHandlers: Handlers = {} -const plugins = [ - 'decode-key', - 'dry-run', - 'follow-chain', - 'new-block', - 'run-block', - 'set-block-build-mode', - 'set-head', - 'set-runtime-log-level', - 'set-storage', - 'time-travel', - 'try-runtime', -] +// list of plugins directory +const plugins = readdirSync(new URL('.', import.meta.url)).filter((file) => + lstatSync(new URL(file, import.meta.url)).isDirectory(), +) -;(async () => { +export const loadRPCPlugins = async () => { for (const plugin of plugins) { - const { rpc, name } = await import(`./${plugin}`) + const location = new URL(`${plugin}/index.js`, import.meta.url) + const { rpc, name } = await import(location.pathname) if (rpc) { const methodName = name || _.camelCase(plugin) pluginHandlers[`dev_${methodName}`] = rpc - logger.debug(`Registered plugin ${plugin} RPC`) + logger.debug(`Registered plugin RPC: ${`dev_${methodName}`}`) } } -})() +} export const pluginExtendCli = async (y: Argv) => { for (const plugin of plugins) { - const { cli } = await import(`./${plugin}`) + const location = new URL(`${plugin}/index.js`, import.meta.url) + const { cli } = await import(location.pathname) if (cli) { cli(y) - logger.debug(`Registered plugin ${plugin} CLI`) + logger.debug(`Registered plugin CLI: ${plugin}`) } } } diff --git a/packages/chopsticks/src/utils/tunnel.ts b/packages/chopsticks/src/utils/tunnel.ts index 36faf425..a673a9d1 100644 --- a/packages/chopsticks/src/utils/tunnel.ts +++ b/packages/chopsticks/src/utils/tunnel.ts @@ -1,4 +1,5 @@ -import 'global-agent/bootstrap' +import { bootstrap } from 'global-agent' +bootstrap() import npmConf from '@pnpm/npm-conf' const npmConfig = npmConf().config diff --git a/packages/core/package.json b/packages/core/package.json index 734d3927..e9080945 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-core", - "version": "0.9.1-4", + "version": "0.9.1-5", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module", diff --git a/packages/db/package.json b/packages/db/package.json index c36c4101..ad9891a7 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-db", - "version": "0.9.1-4", + "version": "0.9.1-5", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module", diff --git a/packages/testing/package.json b/packages/testing/package.json index 6a746115..04af3567 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -1,6 +1,6 @@ { "name": "@acala-network/chopsticks-testing", - "version": "0.9.1-4", + "version": "0.9.1-5", "author": "Acala Developers ", "license": "Apache-2.0", "type": "module", diff --git a/packages/web-test/src/App.tsx b/packages/web-test/src/App.tsx index 4b2a8098..7e41c474 100644 --- a/packages/web-test/src/App.tsx +++ b/packages/web-test/src/App.tsx @@ -12,7 +12,6 @@ import { Typography, } from '@mui/material' import { ApiPromise } from '@polkadot/api' -import { Buffer } from 'buffer' import { ChopsticksProvider, setStorage, setup } from '@acala-network/chopsticks-core' import { HexString } from '@polkadot/util/types' import { IdbDatabase } from '@acala-network/chopsticks-db/browser.js' @@ -21,8 +20,6 @@ import { styled } from '@mui/system' import { useEffect, useState } from 'react' import type { SetupOptions } from '@acala-network/chopsticks-core' -window.Buffer = Buffer - const { alice, bob } = createTestPairs() const DocsLink = styled('a')` From 652c685598672063e6f35baf9ca358025e2cb816 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 7 Nov 2023 17:17:17 +0100 Subject: [PATCH 2/5] node-test --- .github/workflows/ci.yml | 35 +++++++++++++++++++++- package.json | 6 ++-- packages/node-test/index.cjs | 7 +++++ packages/node-test/index.mjs | 4 +++ packages/node-test/package.json | 6 ++++ packages/web-test/package.json | 2 +- yarn.lock | 52 +++++++++++++++++++-------------- 7 files changed, 85 insertions(+), 27 deletions(-) create mode 100644 packages/node-test/index.cjs create mode 100644 packages/node-test/index.mjs create mode 100644 packages/node-test/package.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c9b02b7..49cef354 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,10 +105,43 @@ jobs: - name: Install Playwright Browsers run: npx playwright install --with-deps - name: Run Playwright tests - run: yarn workspace @acala-network/web-test test + run: yarn workspace web-test run test - uses: actions/upload-artifact@v3 if: always() with: name: playwright-report path: packages/web-test/playwright-report/ retention-days: 30 + + node-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + executor/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Install toolchain + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2022-10-30 + components: rustfmt + target: wasm32-unknown-unknown + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: 'yarn' + - run: yarn --immutable + - run: yarn build + - run: | + cd packages/node-test + node index.cjs + node index.mjs diff --git a/package.json b/package.json index 8632e1cf..9f57faf2 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,9 @@ "fix": "eslint . --ext .js,.ts --fix && prettier -w .", "prepare": "husky install", "clean": "yarn workspaces foreach -pvit run clean", - "build": "yarn workspaces foreach -pvit --exclude @acala-network/web-test run build", - "build-wasm": "yarn workspace @acala-network/chopsticks-executor build", - "build:web-test": "yarn build && yarn workspace @acala-network/web-test run build", + "build": "yarn workspaces foreach -pvit --include '@acala-network/*' run build", + "build-wasm": "yarn workspace @acala-network/chopsticks-executor run build", + "build:web-test": "yarn build && yarn workspace web-test run build", "check": "cd executor && cargo check --locked", "test": "vitest run", "test:watch": "vitest", diff --git a/packages/node-test/index.cjs b/packages/node-test/index.cjs new file mode 100644 index 00000000..2576999d --- /dev/null +++ b/packages/node-test/index.cjs @@ -0,0 +1,7 @@ +const { setupWithServer } = require('@acala-network/chopsticks') + +async function main() { + const server = await setupWithServer({ endpoint: 'wss://rpc.polkadot.io', db: 'db.sqlite' }) + await server.close() +} +main() diff --git a/packages/node-test/index.mjs b/packages/node-test/index.mjs new file mode 100644 index 00000000..c4df4e89 --- /dev/null +++ b/packages/node-test/index.mjs @@ -0,0 +1,4 @@ +import { setupWithServer } from '@acala-network/chopsticks' + +const server = await setupWithServer({ endpoint: 'wss://rpc.polkadot.io', db: 'db.sqlite' }) +await server.close() diff --git a/packages/node-test/package.json b/packages/node-test/package.json new file mode 100644 index 00000000..c04532d2 --- /dev/null +++ b/packages/node-test/package.json @@ -0,0 +1,6 @@ +{ + "type": "module", + "dependencies": { + "@acala-network/chopsticks": "workspace:*" + } +} diff --git a/packages/web-test/package.json b/packages/web-test/package.json index a549caf9..d040fe1b 100644 --- a/packages/web-test/package.json +++ b/packages/web-test/package.json @@ -1,5 +1,5 @@ { - "name": "@acala-network/web-test", + "name": "web-test", "author": "Acala Network ", "license": "Apache-2.0", "private": true, diff --git a/yarn.lock b/yarn.lock index 9cb3020e..3aa57453 100644 --- a/yarn.lock +++ b/yarn.lock @@ -112,28 +112,6 @@ __metadata: languageName: unknown linkType: soft -"@acala-network/web-test@workspace:packages/web-test": - version: 0.0.0-use.local - resolution: "@acala-network/web-test@workspace:packages/web-test" - dependencies: - "@acala-network/chopsticks-core": "workspace:*" - "@acala-network/chopsticks-db": "workspace:*" - "@emotion/react": ^11.11.1 - "@emotion/styled": ^11.11.0 - "@mui/material": ^5.14.11 - "@playwright/test": ^1.37.1 - "@types/react": ^18 - "@types/react-dom": ^18 - "@vitejs/plugin-react": ^4.1.0 - crypto-browserify: ^3.12.0 - path-browserify: ^1.0.1 - react: ^18.2.0 - react-dom: ^18.2.0 - stream-browserify: ^3.0.0 - typescript: ^5.1.6 - languageName: unknown - linkType: soft - "@algolia/autocomplete-core@npm:1.9.3": version: 1.9.3 resolution: "@algolia/autocomplete-core@npm:1.9.3" @@ -6899,6 +6877,14 @@ __metadata: languageName: node linkType: hard +"node-test-8fbed1@workspace:packages/node-test": + version: 0.0.0-use.local + resolution: "node-test-8fbed1@workspace:packages/node-test" + dependencies: + "@acala-network/chopsticks": "workspace:*" + languageName: unknown + linkType: soft + "nopt@npm:^5.0.0": version: 5.0.0 resolution: "nopt@npm:5.0.0" @@ -9229,6 +9215,28 @@ __metadata: languageName: node linkType: hard +"web-test@workspace:packages/web-test": + version: 0.0.0-use.local + resolution: "web-test@workspace:packages/web-test" + dependencies: + "@acala-network/chopsticks-core": "workspace:*" + "@acala-network/chopsticks-db": "workspace:*" + "@emotion/react": ^11.11.1 + "@emotion/styled": ^11.11.0 + "@mui/material": ^5.14.11 + "@playwright/test": ^1.37.1 + "@types/react": ^18 + "@types/react-dom": ^18 + "@vitejs/plugin-react": ^4.1.0 + crypto-browserify: ^3.12.0 + path-browserify: ^1.0.1 + react: ^18.2.0 + react-dom: ^18.2.0 + stream-browserify: ^3.0.0 + typescript: ^5.1.6 + languageName: unknown + linkType: soft + "webidl-conversions@npm:^3.0.0": version: 3.0.1 resolution: "webidl-conversions@npm:3.0.1" From e59ed08b4be7d825b35f79fc74af2a2196fde145 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 7 Nov 2023 17:29:14 +0100 Subject: [PATCH 3/5] update node test --- .eslintignore | 1 + .github/workflows/ci.yml | 3 ++- packages/node-test/index.cjs | 1 + packages/node-test/index.mjs | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.eslintignore b/.eslintignore index a5577cd6..dc53f507 100644 --- a/.eslintignore +++ b/.eslintignore @@ -8,3 +8,4 @@ packages/core/scripts/ chopsticks.js packages/core/src/wasm-executor/browser-wasm-executor.js packages/core/src/wasm-executor/node-wasm-executor.js +packages/web-test/playwright-report diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49cef354..d91dc8da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,8 @@ jobs: cache: 'yarn' - run: yarn --immutable - run: yarn build - - run: | + - name: Run Node tests + run: | cd packages/node-test node index.cjs node index.mjs diff --git a/packages/node-test/index.cjs b/packages/node-test/index.cjs index 2576999d..e3a116a5 100644 --- a/packages/node-test/index.cjs +++ b/packages/node-test/index.cjs @@ -2,6 +2,7 @@ const { setupWithServer } = require('@acala-network/chopsticks') async function main() { const server = await setupWithServer({ endpoint: 'wss://rpc.polkadot.io', db: 'db.sqlite' }) + await server.chain.newBlock() await server.close() } main() diff --git a/packages/node-test/index.mjs b/packages/node-test/index.mjs index c4df4e89..4b94d233 100644 --- a/packages/node-test/index.mjs +++ b/packages/node-test/index.mjs @@ -1,4 +1,5 @@ import { setupWithServer } from '@acala-network/chopsticks' const server = await setupWithServer({ endpoint: 'wss://rpc.polkadot.io', db: 'db.sqlite' }) +await server.chain.newBlock() await server.close() From 72bb7ff6dd4bd791893797c6afc33763451ba0bc Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 7 Nov 2023 19:10:07 +0100 Subject: [PATCH 4/5] fix test --- package.json | 4 ++-- packages/chopsticks/src/cli.ts | 2 +- packages/chopsticks/src/context.ts | 2 +- packages/e2e/src/helper.ts | 5 +++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 9f57faf2..e26271d3 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "check": "cd executor && cargo check --locked", "test": "vitest run", "test:watch": "vitest", - "start": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings --experimental-specifier-resolution=node' node packages/chopsticks/src/cli.ts", - "dev": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings --experimental-specifier-resolution=node' LOG_LEVEL=trace node packages/chopsticks/src/cli.ts -- --config=configs/dev.yml", + "start": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings' node packages/chopsticks/src/cli.ts", + "dev": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings' LOG_LEVEL=trace node packages/chopsticks/src/cli.ts -- --config=configs/dev.yml", "script:start": "yarn start", "docs:prep": "yarn workspaces foreach -pvit run docs:prep && cp docs/* docs-src", "docs:dev": "yarn docs:prep && vitepress dev", diff --git a/packages/chopsticks/src/cli.ts b/packages/chopsticks/src/cli.ts index 7f729c41..3296b446 100644 --- a/packages/chopsticks/src/cli.ts +++ b/packages/chopsticks/src/cli.ts @@ -107,7 +107,7 @@ const commands = yargs(hideBin(process.argv)) .usage('Usage: $0 [options]') .example('$0', '-c acala') -if (!process.env.NO_PLUGINS) { +if (!process.env.DISABLE_PLUGINS) { pluginExtendCli(commands).then(() => commands.parse()) } else { commands.parse() diff --git a/packages/chopsticks/src/context.ts b/packages/chopsticks/src/context.ts index 93646a05..f74dc353 100644 --- a/packages/chopsticks/src/context.ts +++ b/packages/chopsticks/src/context.ts @@ -88,7 +88,7 @@ export const setupContext = async (argv: Config, overrideParent = false) => { await overrideWasm(chain, argv['wasm-override'], at) await overrideStorage(chain, argv['import-storage'], at) - if (!process.env.NO_PLUGINS) { + if (!process.env.DISABLE_PLUGINS) { await loadRPCPlugins() } diff --git a/packages/e2e/src/helper.ts b/packages/e2e/src/helper.ts index 235ace83..e9d02b20 100644 --- a/packages/e2e/src/helper.ts +++ b/packages/e2e/src/helper.ts @@ -19,6 +19,7 @@ import { createServer } from '@acala-network/chopsticks/server.js' import { defer } from '@acala-network/chopsticks-core/utils/index.js' import { genesisFromUrl } from '@acala-network/chopsticks/context.js' import { handler } from '@acala-network/chopsticks/rpc/index.js' +import { loadRPCPlugins } from '@acala-network/chopsticks/plugins/index.js' export { expectJson, expectHex, testingPairs } from '@acala-network/chopsticks-testing' @@ -105,6 +106,10 @@ export const setupAll = async ({ db: !process.env.RUN_TESTS_WITHOUT_DB ? new SqliteDatabase('e2e-tests-db.sqlite') : undefined, }) + if (!process.env.DISABLE_PLUGINS) { + await loadRPCPlugins() + } + const { port, close } = await createServer(handler({ chain })) const ws = new WsProvider(`ws://localhost:${port}`, undefined, undefined, 300_000) From 5d8a08d1d8b830568db31c25d9fb6c35d79c763c Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Tue, 7 Nov 2023 20:13:05 +0100 Subject: [PATCH 5/5] fix flaky test --- .../src/plugins/run-block/index.test.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/chopsticks/src/plugins/run-block/index.test.ts b/packages/chopsticks/src/plugins/run-block/index.test.ts index 1d9426c9..ebaca708 100644 --- a/packages/chopsticks/src/plugins/run-block/index.test.ts +++ b/packages/chopsticks/src/plugins/run-block/index.test.ts @@ -1,5 +1,5 @@ +import { SqliteDatabase } from '@acala-network/chopsticks-db' import { describe, expect, it } from 'vitest' - import { setup } from '@acala-network/chopsticks-core' import { rpc } from './index.js' @@ -9,26 +9,26 @@ describe('run-block', () => { const chain = await setup({ endpoint: 'wss://rpc.polkadot.io', block: 18000000, + db: !process.env.RUN_TESTS_WITHOUT_DB ? new SqliteDatabase('e2e-tests-db.sqlite') : undefined, }) const block = (await chain.getBlockAt(18000000))! const header = await block.header const parent = header.parentHash.toHex() - expect( - await rpc({ chain }, [ - { - includeRaw: true, - includeParsed: true, - includeBlockDetails: true, - parent, - block: { - header: header.toJSON(), - extrinsics: await block.extrinsics, - }, + const result = await rpc({ chain }, [ + { + includeRaw: true, + includeParsed: true, + includeBlockDetails: true, + parent, + block: { + header: header.toJSON(), + extrinsics: await block.extrinsics, }, - ]), - ).toMatchSnapshot() + }, + ]) + expect(result).toMatchSnapshot() await chain.close() }, 90000)