feat: add revert tests (external node) to zk_toolbox

.github/workflows/ci-zk-toolbox-reusable.yml

@@ -113,6 +113,10 @@ jobs:
         run: |
           ci_run zk_supervisor test revert --ignore-prerequisites --verbose
 
+      - name: Run revert tests (external node)
+        run: |
+          ci_run zk_supervisor test revert --external-node --ignore-prerequisites --verbose
+
       - name: Show server.log logs
         if: always()
         run: ci_run cat server.log || true

core/tests/revert-test/tests/revert-and-restart-en.test.ts

@@ -5,27 +5,76 @@
 // main_contract.getTotalBatchesExecuted actually checks the number of batches executed.
 import * as utils from 'utils';
 import { Tester } from './tester';
+import { exec, runServerInBackground, runExternalNodeInBackground } from './utils';
 import * as zksync from 'zksync-ethers';
 import * as ethers from 'ethers';
 import { expect, assert } from 'chai';
 import fs from 'fs';
 import * as child_process from 'child_process';
 import * as dotenv from 'dotenv';
+import { getAllConfigsPath, loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs';
+import path from 'path';
+
+const pathToHome = path.join(__dirname, '../../../..');
+const fileConfig = shouldLoadConfigFromFile();
 
 let mainEnv: string;
 let extEnv: string;
-if (process.env.DEPLOYMENT_MODE == 'Validium') {
+
+let deploymentMode: string;
+
+if (fileConfig.loadFromFile) {
+    const genesisConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'genesis.yaml' });
+    deploymentMode = genesisConfig.deploymentMode;
+} else {
+    if (!process.env.DEPLOYMENT_MODE) {
+        throw new Error('DEPLOYMENT_MODE is not set');
+    }
+    if (!['Validium', 'Rollup'].includes(process.env.DEPLOYMENT_MODE)) {
+        throw new Error(`Unknown deployment mode: ${process.env.DEPLOYMENT_MODE}`);
+    }
+    deploymentMode = process.env.DEPLOYMENT_MODE;
+}
+
+if (deploymentMode == 'Validium') {
     mainEnv = process.env.IN_DOCKER ? 'dev_validium_docker' : 'dev_validium';
     extEnv = process.env.IN_DOCKER ? 'ext-node-validium-docker' : 'ext-node-validium';
-} else if (process.env.DEPLOYMENT_MODE == 'Rollup') {
+} else {
+    // Rollup deployment mode
     mainEnv = process.env.IN_DOCKER ? 'docker' : 'dev';
     extEnv = process.env.IN_DOCKER ? 'ext-node-docker' : 'ext-node';
-} else {
-    throw new Error(`Unknown deployment mode: ${process.env.DEPLOYMENT_MODE}`);
 }
 const mainLogsPath: string = 'revert_main.log';
 const extLogsPath: string = 'revert_ext.log';
 
+let ethClientWeb3Url: string;
+let apiWeb3JsonRpcHttpUrl: string;
+let baseTokenAddress: string;
+let enEthClientUrl: string;
+let operatorAddress: string;
+
+if (fileConfig.loadFromFile) {
+    const secretsConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'secrets.yaml' });
+    const generalConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'general.yaml' });
+    const contractsConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'contracts.yaml' });
+    const externalNodeConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'external_node.yaml' });
+    const walletsConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'wallets.yaml' });
+
+    ethClientWeb3Url = secretsConfig.l1.l1_rpc_url;
+    apiWeb3JsonRpcHttpUrl = generalConfig.api.web3_json_rpc.http_url;
+    baseTokenAddress = contractsConfig.l1.base_token_addr;
+    enEthClientUrl = externalNodeConfig.main_node_url;
+    operatorAddress = walletsConfig.operator.address;
+} else {
+    let env = fetchEnv(mainEnv);
+    ethClientWeb3Url = env.ETH_CLIENT_WEB3_URL;
+    apiWeb3JsonRpcHttpUrl = env.API_WEB3_JSON_RPC_HTTP_URL;
+    baseTokenAddress = env.CONTRACTS_BASE_TOKEN_ADDR;
+    enEthClientUrl = `http://127.0.0.1:${env.EN_HTTP_PORT}`;
+    // TODO use env variable for this?
+    operatorAddress = '0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7';
+}
+
 interface SuggestedValues {
     lastExecutedL1BatchNumber: bigint;
     nonce: number;
@@ -46,10 +95,6 @@ function parseSuggestedValues(jsonString: string): SuggestedValues {
     };
 }
 
-function spawn(cmd: string, args: string[], options: child_process.SpawnOptions): child_process.ChildProcess {
-    return child_process.spawn(cmd, args, options);
-}
-
 function run(cmd: string, args: string[], options: child_process.SpawnOptions): child_process.SpawnSyncReturns<Buffer> {
     let res = child_process.spawnSync(cmd, args, options);
     expect(res.error).to.be.undefined;
@@ -79,18 +124,33 @@ function fetchEnv(zksyncEnv: string): any {
     return { ...process.env, ...dotenv.parse(res.stdout) };
 }
 
-function runBlockReverter(args: string[]): string {
+async function runBlockReverter(args: string[]): Promise<string> {
     let env = fetchEnv(mainEnv);
-    env.RUST_LOG = 'off';
-    let res = run('./target/release/block_reverter', args, {
+
+    let fileConfigFlags = '';
+    if (fileConfig.loadFromFile) {
+        const configPaths = getAllConfigsPath({ pathToHome, chain: fileConfig.chain });
+        fileConfigFlags = `
+                --config-path=${configPaths['general.yaml']}
+                --contracts-config-path=${configPaths['contracts.yaml']}
+                --secrets-path=${configPaths['secrets.yaml']}
+                --wallets-path=${configPaths['wallets.yaml']}
+                --genesis-path=${configPaths['genesis.yaml']}
+            `;
+    }
+
+    const cmd = `cd ${pathToHome} && RUST_LOG=off cargo run --bin block_reverter --release -- ${args.join(
+        ' '
+    )} ${fileConfigFlags}`;
+    const executedProcess = await exec(cmd, {
         cwd: env.ZKSYNC_HOME,
         env: {
             ...env,
             PATH: process.env.PATH
         }
     });
-    console.log(res.stderr.toString());
-    return res.stdout.toString();
+
+    return executedProcess.stdout;
 }
 
 async function killServerAndWaitForShutdown(tester: Tester, server: string) {
@@ -112,7 +172,7 @@ async function killServerAndWaitForShutdown(tester: Tester, server: string) {
 }
 
 class MainNode {
-    constructor(public tester: Tester, private proc: child_process.ChildProcess) {}
+    constructor(public tester: Tester) {}
 
     // Terminates all main node processes running.
     public static async terminateAll() {
@@ -135,45 +195,38 @@ class MainNode {
         env.ETH_SENDER_SENDER_AGGREGATED_BLOCK_EXECUTE_DEADLINE = enableExecute ? '1' : '10000';
         // Set full mode for the Merkle tree as it is required to get blocks committed.
         env.DATABASE_MERKLE_TREE_MODE = 'full';
-        console.log(`DATABASE_URL = ${env.DATABASE_URL}`);
 
         let components = 'api,tree,eth,state_keeper,commitment_generator,da_dispatcher';
         if (enableConsensus) {
             components += ',consensus';
         }
 
-        let proc = spawn('./target/release/zksync_server', ['--components', components], {
-            cwd: env.ZKSYNC_HOME,
+        runServerInBackground({
+            components: [components],
             stdio: [null, logs, logs],
-            env: {
-                ...env,
-                PATH: process.env.PATH
-            }
+            cwd: pathToHome,
+            env: env,
+            useZkInception: fileConfig.loadFromFile
         });
+
         // Wait until the main node starts responding.
-        let tester: Tester = await Tester.init(
-            env.ETH_CLIENT_WEB3_URL,
-            env.API_WEB3_JSON_RPC_HTTP_URL,
-            env.CONTRACTS_BASE_TOKEN_ADDR
-        );
+        let tester: Tester = await Tester.init(ethClientWeb3Url, apiWeb3JsonRpcHttpUrl, baseTokenAddress);
         while (true) {
             try {
                 await tester.syncWallet.provider.getBlockNumber();
                 break;
             } catch (err) {
-                if (proc.exitCode != null) {
-                    assert.fail(`server failed to start, exitCode = ${proc.exitCode}`);
-                }
+                // NOTE: not an infinite loop, as the test will eventually timeout.
                 console.log('waiting for api endpoint');
                 await utils.sleep(1);
             }
         }
-        return new MainNode(tester, proc);
+        return new MainNode(tester);
     }
 }
 
 class ExtNode {
-    constructor(public tester: Tester, private proc: child_process.ChildProcess) {}
+    constructor(public tester: Tester) {}
 
     // Terminates all main node processes running.
     public static async terminateAll() {
@@ -188,51 +241,37 @@ class ExtNode {
     // If enableConsensus is set, the node will use consensus P2P network to fetch blocks.
     public static async spawn(logs: fs.WriteStream, enableConsensus: boolean): Promise<ExtNode> {
         let env = fetchEnv(extEnv);
-        console.log(`DATABASE_URL = ${env.DATABASE_URL}`);
         let args = [];
         if (enableConsensus) {
             args.push('--enable-consensus');
         }
-        let proc = spawn('./target/release/zksync_external_node', args, {
-            cwd: env.ZKSYNC_HOME,
+
+        // Run server in background.
+        runExternalNodeInBackground({
             stdio: [null, logs, logs],
-            env: {
-                ...env,
-                PATH: process.env.PATH
-            }
+            cwd: pathToHome,
+            env: env,
+            useZkInception: fileConfig.loadFromFile
         });
+
         // Wait until the node starts responding.
-        let tester: Tester = await Tester.init(
-            env.EN_ETH_CLIENT_URL,
-            `http://127.0.0.1:${env.EN_HTTP_PORT}`,
-            env.CONTRACTS_BASE_TOKEN_ADDR
-        );
+        let tester: Tester = await Tester.init(ethClientWeb3Url, enEthClientUrl, baseTokenAddress);
         while (true) {
             try {
                 await tester.syncWallet.provider.getBlockNumber();
                 break;
             } catch (err) {
-                if (proc.exitCode != null) {
-                    assert.fail(`node failed to start, exitCode = ${proc.exitCode}`);
-                }
+                // NOTE: not an infinite loop, as the test will eventually timeout.
                 console.log('waiting for api endpoint');
                 await utils.sleep(1);
             }
         }
-        return new ExtNode(tester, proc);
-    }
-
-    // Waits for the node process to exit.
-    public async waitForExit(): Promise<number> {
-        while (this.proc.exitCode === null) {
-            await utils.sleep(1);
-        }
-        return this.proc.exitCode;
+        return new ExtNode(tester);
     }
 }
 
 describe('Block reverting test', function () {
-    if (process.env.SKIP_COMPILATION !== 'true') {
+    if (process.env.SKIP_COMPILATION !== 'true' && !fileConfig.loadFromFile) {
         compileBinaries();
     }
     console.log(`PWD = ${process.env.PWD}`);
@@ -319,18 +358,18 @@ describe('Block reverting test', function () {
         await killServerAndWaitForShutdown(mainNode.tester, 'zksync_server');
 
         console.log('Ask block_reverter to suggest to which L1 batch we should revert');
-        const values_json = runBlockReverter([
+        const values_json = await runBlockReverter([
             'print-suggested-values',
             '--json',
             '--operator-address',
-            '0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7'
+            operatorAddress
         ]);
         console.log(`values = ${values_json}`);
         const values = parseSuggestedValues(values_json);
         assert(lastExecuted === values.lastExecutedL1BatchNumber);
 
         console.log('Send reverting transaction to L1');
-        runBlockReverter([
+        await runBlockReverter([
             'send-eth-transaction',
             '--l1-batch-number',
             values.lastExecutedL1BatchNumber.toString(),
@@ -346,7 +385,7 @@ describe('Block reverting test', function () {
         assert(lastCommitted2 === lastExecuted);
 
         console.log('Rollback db');
-        runBlockReverter([
+        await runBlockReverter([
             'rollback-db',
             '--l1-batch-number',
             values.lastExecutedL1BatchNumber.toString(),
@@ -359,10 +398,11 @@ describe('Block reverting test', function () {
         mainNode = await MainNode.spawn(mainLogs, enableConsensus, true);
 
         console.log('Wait for the external node to detect reorg and terminate');
-        await extNode.waitForExit();
+        await utils.sleep(3);
 
         console.log('Restart external node and wait for it to revert.');
         extNode = await ExtNode.spawn(extLogs, enableConsensus);
+        await utils.sleep(30);
 
         console.log('Execute an L1 transaction');
         const depositHandle = await extNode.tester.syncWallet.deposit({

core/tests/revert-test/tests/utils.ts

@@ -0,0 +1,81 @@
+import { exec as _exec, spawn as _spawn, type ProcessEnvOptions } from 'child_process';
+import { promisify } from 'util';
+
+// executes a command in background and returns a child process handle
+// by default pipes data to parent's stdio but this can be overridden
+export function background({
+    command,
+    stdio = 'inherit',
+    cwd,
+    env
+}: {
+    command: string;
+    stdio: any;
+    cwd?: ProcessEnvOptions['cwd'];
+    env?: ProcessEnvOptions['env'];
+}) {
+    command = command.replace(/\n/g, ' ');
+    return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env });
+}
+
+export function runInBackground({
+    command,
+    components,
+    stdio,
+    cwd,
+    env
+}: {
+    command: string;
+    components?: string[];
+    stdio: any;
+    cwd?: Parameters<typeof background>[0]['cwd'];
+    env?: Parameters<typeof background>[0]['env'];
+}) {
+    if (components && components.length > 0) {
+        command += ` --components=${components.join(',')}`;
+    }
+    background({ command, stdio, cwd, env });
+}
+
+export function runServerInBackground({
+    components,
+    stdio,
+    cwd,
+    env,
+    useZkInception
+}: {
+    components?: string[];
+    stdio: any;
+    cwd?: Parameters<typeof background>[0]['cwd'];
+    env?: Parameters<typeof background>[0]['env'];
+    useZkInception?: boolean;
+}) {
+    let command = useZkInception ? 'zk_inception server' : 'zk server';
+    runInBackground({ command, components, stdio, cwd, env });
+}
+
+export function runExternalNodeInBackground({
+    components,
+    stdio,
+    cwd,
+    env,
+    useZkInception
+}: {
+    components?: string[];
+    stdio: any;
+    cwd?: Parameters<typeof background>[0]['cwd'];
+    env?: Parameters<typeof background>[0]['env'];
+    useZkInception?: boolean;
+}) {
+    let command = useZkInception ? 'zk_inception external-node run' : 'zk external-node';
+    runInBackground({ command, components, stdio, cwd, env });
+}
+
+// async executor of shell commands
+// spawns a new shell and can execute arbitrary commands, like "ls -la | grep .env"
+// returns { stdout, stderr }
+const promisified = promisify(_exec);
+export function exec(command: string, options: ProcessEnvOptions) {
+    command = command.replace(/\n/g, ' ');
+    return promisified(command, options);
+}