fix plugins (#523)

.eslintignore

@@ -8,3 +8,4 @@ packages/core/scripts/
 chopsticks.js
 packages/core/src/wasm-executor/browser-wasm-executor.js
 packages/core/src/wasm-executor/node-wasm-executor.js
+packages/web-test/playwright-report

.github/workflows/ci.yml

@@ -105,10 +105,44 @@ jobs:
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps
       - name: Run Playwright tests
-        run: yarn workspace @acala-network/web-test test
+        run: yarn workspace web-test run test
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-report
           path: packages/web-test/playwright-report/
           retention-days: 30
+
+  node-test:
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            executor/target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          toolchain: nightly-2022-10-30
+          components: rustfmt
+          target: wasm32-unknown-unknown
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+          cache: 'yarn'
+      - run: yarn --immutable
+      - run: yarn build
+      - name: Run Node tests
+        run: |
+          cd packages/node-test
+          node index.cjs
+          node index.mjs

executor/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "chopsticks-executor"
-version = "0.9.1-4"
+version = "0.9.1-5"
 description = "Chopsticks executor"
 repository = "https://github.com/AcalaNetwork/chopsticks"
 license = "Apache-2.0"

executor/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@acala-network/chopsticks-executor",
 	"description": "Chopsticks executor",
-	"version": "0.9.1-4",
+	"version": "0.9.1-5",
 	"license": "Apache-2.0",
 	"repository": {
 		"type": "git",

package.json

@@ -14,14 +14,14 @@
 		"fix": "eslint . --ext .js,.ts --fix && prettier -w .",
 		"prepare": "husky install",
 		"clean": "yarn workspaces foreach -pvit run clean",
-		"build": "yarn workspaces foreach -pvit --exclude @acala-network/web-test run build",
-		"build-wasm": "yarn workspace @acala-network/chopsticks-executor build",
-		"build:web-test": "yarn build && yarn workspace @acala-network/web-test run build",
+		"build": "yarn workspaces foreach -pvit --include '@acala-network/*' run build",
+		"build-wasm": "yarn workspace @acala-network/chopsticks-executor run build",
+		"build:web-test": "yarn build && yarn workspace web-test run build",
 		"check": "cd executor && cargo check --locked",
 		"test": "vitest run",
 		"test:watch": "vitest",
-		"start": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings --experimental-specifier-resolution=node' node packages/chopsticks/src/cli.ts",
-		"dev": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings --experimental-specifier-resolution=node' LOG_LEVEL=trace node packages/chopsticks/src/cli.ts -- --config=configs/dev.yml",
+		"start": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings' node packages/chopsticks/src/cli.ts",
+		"dev": "NODE_OPTIONS='--loader ts-node/esm --loader ./loader.js --no-warnings' LOG_LEVEL=trace node packages/chopsticks/src/cli.ts -- --config=configs/dev.yml",
 		"script:start": "yarn start",
 		"docs:prep": "yarn workspaces foreach -pvit run docs:prep && cp docs/* docs-src",
 		"docs:dev": "yarn docs:prep && vitepress dev",

packages/chopsticks/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@acala-network/chopsticks",
-	"version": "0.9.1-4",
+	"version": "0.9.1-5",
 	"author": "Acala Developers <[email protected]>",
 	"license": "Apache-2.0",
 	"bin": "./chopsticks.cjs",

packages/chopsticks/src/cli.ts

@@ -107,4 +107,8 @@ const commands = yargs(hideBin(process.argv))
   .usage('Usage: $0 <command> [options]')
   .example('$0', '-c acala')
 
-pluginExtendCli(commands).then(() => commands.parse())
+if (!process.env.DISABLE_PLUGINS) {
+  pluginExtendCli(commands).then(() => commands.parse())
+} else {
+  commands.parse()
+}

packages/chopsticks/src/context.ts

@@ -3,6 +3,7 @@ import { BlockEntry, GenesisProvider, defaultLogger, isUrl, setup, timeTravel }
 import { Config } from './schema/index.js'
 import { HexString } from '@polkadot/util/types'
 import { SqliteDatabase } from '@acala-network/chopsticks-db'
+import { loadRPCPlugins } from './plugins/index.js'
 import { overrideStorage, overrideWasm } from './utils/override.js'
 import axios from 'axios'
 
@@ -87,5 +88,9 @@ export const setupContext = async (argv: Config, overrideParent = false) => {
   await overrideWasm(chain, argv['wasm-override'], at)
   await overrideStorage(chain, argv['import-storage'], at)
 
+  if (!process.env.DISABLE_PLUGINS) {
+    await loadRPCPlugins()
+  }
+
   return { chain }
 }

packages/chopsticks/src/plugins/index.ts

@@ -1,4 +1,5 @@
 import { Handlers } from '@acala-network/chopsticks-core'
+import { lstatSync, readdirSync } from 'fs'
 import _ from 'lodash'
 import type { Argv } from 'yargs'
 
@@ -8,37 +9,30 @@ const logger = defaultLogger.child({ name: 'plugin' })
 
 export const pluginHandlers: Handlers = {}
 
-const plugins = [
-  'decode-key',
-  'dry-run',
-  'follow-chain',
-  'new-block',
-  'run-block',
-  'set-block-build-mode',
-  'set-head',
-  'set-runtime-log-level',
-  'set-storage',
-  'time-travel',
-  'try-runtime',
-]
+// list of plugins directory
+const plugins = readdirSync(new URL('.', import.meta.url)).filter((file) =>
+  lstatSync(new URL(file, import.meta.url)).isDirectory(),
+)
 
-;(async () => {
+export const loadRPCPlugins = async () => {
   for (const plugin of plugins) {
-    const { rpc, name } = await import(`./${plugin}`)
+    const location = new URL(`${plugin}/index.js`, import.meta.url)
+    const { rpc, name } = await import(location.pathname)
     if (rpc) {
       const methodName = name || _.camelCase(plugin)
       pluginHandlers[`dev_${methodName}`] = rpc
-      logger.debug(`Registered plugin ${plugin} RPC`)
+      logger.debug(`Registered plugin RPC: ${`dev_${methodName}`}`)
     }
   }
-})()
+}
 
 export const pluginExtendCli = async (y: Argv) => {
   for (const plugin of plugins) {
-    const { cli } = await import(`./${plugin}`)
+    const location = new URL(`${plugin}/index.js`, import.meta.url)
+    const { cli } = await import(location.pathname)
     if (cli) {
       cli(y)
-      logger.debug(`Registered plugin ${plugin} CLI`)
+      logger.debug(`Registered plugin CLI: ${plugin}`)
     }
   }
 }

packages/chopsticks/src/plugins/run-block/index.test.ts

@@ -1,5 +1,5 @@
+import { SqliteDatabase } from '@acala-network/chopsticks-db'
 import { describe, expect, it } from 'vitest'
-
 import { setup } from '@acala-network/chopsticks-core'
 
 import { rpc } from './index.js'
@@ -9,26 +9,26 @@ describe('run-block', () => {
     const chain = await setup({
       endpoint: 'wss://rpc.polkadot.io',
       block: 18000000,
+      db: !process.env.RUN_TESTS_WITHOUT_DB ? new SqliteDatabase('e2e-tests-db.sqlite') : undefined,
     })
 
     const block = (await chain.getBlockAt(18000000))!
     const header = await block.header
     const parent = header.parentHash.toHex()
 
-    expect(
-      await rpc({ chain }, [
-        {
-          includeRaw: true,
-          includeParsed: true,
-          includeBlockDetails: true,
-          parent,
-          block: {
-            header: header.toJSON(),
-            extrinsics: await block.extrinsics,
-          },
+    const result = await rpc({ chain }, [
+      {
+        includeRaw: true,
+        includeParsed: true,
+        includeBlockDetails: true,
+        parent,
+        block: {
+          header: header.toJSON(),
+          extrinsics: await block.extrinsics,
         },
-      ]),
-    ).toMatchSnapshot()
+      },
+    ])
+    expect(result).toMatchSnapshot()
 
     await chain.close()
   }, 90000)

packages/chopsticks/src/utils/tunnel.ts

@@ -1,4 +1,5 @@
-import 'global-agent/bootstrap'
+import { bootstrap } from 'global-agent'
+bootstrap()
 import npmConf from '@pnpm/npm-conf'
 
 const npmConfig = npmConf().config

packages/core/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@acala-network/chopsticks-core",
-	"version": "0.9.1-4",
+	"version": "0.9.1-5",
 	"author": "Acala Developers <[email protected]>",
 	"license": "Apache-2.0",
 	"type": "module",

packages/db/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@acala-network/chopsticks-db",
-	"version": "0.9.1-4",
+	"version": "0.9.1-5",
 	"author": "Acala Developers <[email protected]>",
 	"license": "Apache-2.0",
 	"type": "module",

packages/e2e/src/helper.ts

@@ -19,6 +19,7 @@ import { createServer } from '@acala-network/chopsticks/server.js'
 import { defer } from '@acala-network/chopsticks-core/utils/index.js'
 import { genesisFromUrl } from '@acala-network/chopsticks/context.js'
 import { handler } from '@acala-network/chopsticks/rpc/index.js'
+import { loadRPCPlugins } from '@acala-network/chopsticks/plugins/index.js'
 
 export { expectJson, expectHex, testingPairs } from '@acala-network/chopsticks-testing'
 
@@ -105,6 +106,10 @@ export const setupAll = async ({
         db: !process.env.RUN_TESTS_WITHOUT_DB ? new SqliteDatabase('e2e-tests-db.sqlite') : undefined,
       })
 
+      if (!process.env.DISABLE_PLUGINS) {
+        await loadRPCPlugins()
+      }
+
       const { port, close } = await createServer(handler({ chain }))
 
       const ws = new WsProvider(`ws://localhost:${port}`, undefined, undefined, 300_000)

packages/node-test/index.cjs

@@ -0,0 +1,8 @@
+const { setupWithServer } = require('@acala-network/chopsticks')
+
+async function main() {
+	const server = await setupWithServer({ endpoint: 'wss://rpc.polkadot.io', db: 'db.sqlite' })
+	await server.chain.newBlock()
+	await server.close()
+}
+main()

packages/node-test/index.mjs

@@ -0,0 +1,5 @@
+import { setupWithServer } from '@acala-network/chopsticks'
+
+const server = await setupWithServer({ endpoint: 'wss://rpc.polkadot.io', db: 'db.sqlite' })
+await server.chain.newBlock()
+await server.close()

packages/node-test/package.json

@@ -0,0 +1,6 @@
+{
+	"type": "module",
+	"dependencies": {
+		"@acala-network/chopsticks": "workspace:*"
+	}
+}

packages/testing/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@acala-network/chopsticks-testing",
-	"version": "0.9.1-4",
+	"version": "0.9.1-5",
 	"author": "Acala Developers <[email protected]>",
 	"license": "Apache-2.0",
 	"type": "module",

packages/web-test/package.json

@@ -1,5 +1,5 @@
 {
-	"name": "@acala-network/web-test",
+	"name": "web-test",
 	"author": "Acala Network <[email protected]>",
 	"license": "Apache-2.0",
 	"private": true,

packages/web-test/src/App.tsx

@@ -12,7 +12,6 @@ import {
 	Typography,
 } from '@mui/material'
 import { ApiPromise } from '@polkadot/api'
-import { Buffer } from 'buffer'
 import { ChopsticksProvider, setStorage, setup } from '@acala-network/chopsticks-core'
 import { HexString } from '@polkadot/util/types'
 import { IdbDatabase } from '@acala-network/chopsticks-db/browser.js'
@@ -21,8 +20,6 @@ import { styled } from '@mui/system'
 import { useEffect, useState } from 'react'
 import type { SetupOptions } from '@acala-network/chopsticks-core'
 
-window.Buffer = Buffer
-
 const { alice, bob } = createTestPairs()
 
 const DocsLink = styled('a')`