Skip to content

Commit

Permalink
feat: Add recovery tests to zk_supervisor (#2444)
Browse files Browse the repository at this point in the history 
## What ❔

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

Add recovery tests to zk_supervisor

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [x] Tests for the changes have been added / updated.
- [x] Documentation comments have been added / updated.
- [x] Code has been formatted via `zk fmt` and `zk lint`.

---------

Signed-off-by: Danil <[email protected]>
Co-authored-by: Manuel <[email protected]>
Co-authored-by: Danil <[email protected]>
  • Loading branch information
@manuelmauro @Deniallugo
3 people authored Jul 26, 2024
1 parent 2fa2249 commit 0c0d10a
Show file tree
Hide file tree
Showing 15 changed files with 395 additions and 68 deletions.
13 changes: 12 additions & 1 deletion .github/workflows/ci-zk-toolbox-reusable.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,22 @@ jobs:
run: |
ci_run zk_supervisor test integration --ignore-prerequisites --verbose
- name: Run external node server
- name: Init external node server
run: |
ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \
--db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545
ci_run zk_inception external-node init --ignore-prerequisites
- name: Run recovery tests (from snapshot)
run: |
ci_run zk_supervisor test recovery --snapshot --ignore-prerequisites --verbose
- name: Run recovery tests (from genesis)
run: |
ci_run zk_supervisor test recovery --ignore-prerequisites --verbose
- name: Run external node server
run: |
ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log &
ci_run sleep 5
Expand Down
42 changes: 19 additions & 23 deletions core/tests/recovery-test/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import * as zksync from 'zksync-ethers';
import * as ethers from 'ethers';
import path from 'node:path';
import { expect } from 'chai';
import { runExternalNodeInBackground } from './utils';

export interface Health<T> {
readonly status: string;
Expand Down Expand Up @@ -65,11 +66,9 @@ export async function sleep(millis: number) {
await new Promise((resolve) => setTimeout(resolve, millis));
}

export async function getExternalNodeHealth() {
const EXTERNAL_NODE_HEALTH_URL = 'http://127.0.0.1:3081/health';

export async function getExternalNodeHealth(url: string) {
try {
const response: HealthCheckResponse = await fetch(EXTERNAL_NODE_HEALTH_URL).then((response) => response.json());
const response: HealthCheckResponse = await fetch(url).then((response) => response.json());
return response;
} catch (e) {
let displayedError = e;
Expand All @@ -84,12 +83,13 @@ export async function getExternalNodeHealth() {
}
}

export async function dropNodeDatabase(env: { [key: string]: string }) {
await executeNodeCommand(env, 'zk db reset');
}

export async function dropNodeStorage(env: { [key: string]: string }) {
await executeNodeCommand(env, 'zk clean --database');
export async function dropNodeData(useZkSupervisor: boolean, env: { [key: string]: string }) {
if (useZkSupervisor) {
await executeNodeCommand(env, 'zk_inception external-node init');
} else {
await executeNodeCommand(env, 'zk db reset');
await executeNodeCommand(env, 'zk clean --database');
}
}

async function executeNodeCommand(env: { [key: string]: string }, command: string) {
Expand Down Expand Up @@ -127,15 +127,6 @@ export enum NodeComponents {
WITH_TREE_FETCHER_AND_NO_TREE = 'core,api,tree_fetcher'
}

function externalNodeArgs(components: NodeComponents = NodeComponents.STANDARD) {
const enableConsensus = process.env.ENABLE_CONSENSUS === 'true';
const args = ['external-node', '--', `--components=${components}`];
if (enableConsensus) {
args.push('--enable-consensus');
}
return args;
}

export class NodeProcess {
static async stopAll(signal: 'INT' | 'KILL' = 'INT') {
interface ChildProcessError extends Error {
Expand All @@ -157,15 +148,20 @@ export class NodeProcess {
static async spawn(
env: { [key: string]: string },
logsFile: FileHandle | string,
pathToHome: string,
useZkInception: boolean,
components: NodeComponents = NodeComponents.STANDARD
) {
const logs = typeof logsFile === 'string' ? await fs.open(logsFile, 'w') : logsFile;
const childProcess = spawn('zk', externalNodeArgs(components), {
cwd: process.env.ZKSYNC_HOME!!,

let childProcess = runExternalNodeInBackground({
components: [components],
stdio: [null, logs.fd, logs.fd],
shell: true,
env
cwd: pathToHome,
env,
useZkInception
});

return new NodeProcess(childProcess, logs);
}

Expand Down
71 changes: 71 additions & 0 deletions core/tests/recovery-test/src/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import { spawn as _spawn, ChildProcessWithoutNullStreams, type ProcessEnvOptions } from 'child_process';

// executes a command in background and returns a child process handle
// by default pipes data to parent's stdio but this can be overridden
export function background({
command,
stdio = 'inherit',
cwd,
env
}: {
command: string;
stdio: any;
cwd?: ProcessEnvOptions['cwd'];
env?: ProcessEnvOptions['env'];
}): ChildProcessWithoutNullStreams {
command = command.replace(/\n/g, ' ');
console.log(`Running command in background: ${command}`);
return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env });
}

export function runInBackground({
command,
components,
stdio,
cwd,
env
}: {
command: string;
components?: string[];
stdio: any;
cwd?: Parameters<typeof background>[0]['cwd'];
env?: Parameters<typeof background>[0]['env'];
}): ChildProcessWithoutNullStreams {
if (components && components.length > 0) {
command += ` --components=${components.join(',')}`;
}

return background({
command,
stdio,
cwd,
env
});
}

export function runExternalNodeInBackground({
components,
stdio,
cwd,
env,
useZkInception
}: {
components?: string[];
stdio: any;
cwd?: Parameters<typeof background>[0]['cwd'];
env?: Parameters<typeof background>[0]['env'];
useZkInception?: boolean;
}): ChildProcessWithoutNullStreams {
let command = '';
if (useZkInception) {
command = 'zk_inception external-node run';
} else {
command = 'zk external-node --';

const enableConsensus = process.env.ENABLE_CONSENSUS === 'true';
if (enableConsensus) {
command += ' --enable-consensus';
}
}
return runInBackground({ command, components, stdio, cwd, env });
}
59 changes: 38 additions & 21 deletions core/tests/recovery-test/tests/genesis-recovery.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,12 @@ import { expect } from 'chai';
import * as zksync from 'zksync-ethers';
import { ethers } from 'ethers';

import {
NodeProcess,
dropNodeDatabase,
dropNodeStorage,
getExternalNodeHealth,
NodeComponents,
sleep,
FundedWallet
} from '../src';
import { NodeProcess, dropNodeData, getExternalNodeHealth, NodeComponents, sleep, FundedWallet } from '../src';
import { loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs';
import path from 'path';

const pathToHome = path.join(__dirname, '../../../..');
const fileConfig = shouldLoadConfigFromFile();

/**
* Tests recovery of an external node from scratch.
Expand Down Expand Up @@ -43,18 +40,38 @@ describe('genesis recovery', () => {
let externalNodeProcess: NodeProcess;
let externalNodeBatchNumber: number;

let apiWeb3JsonRpcHttpUrl: string;
let ethRpcUrl: string;
let externalNodeUrl: string;
let extNodeHealthUrl: string;

before('prepare environment', async () => {
expect(process.env.ZKSYNC_ENV, '`ZKSYNC_ENV` should not be set to allow running both server and EN components')
.to.be.undefined;
mainNode = new zksync.Provider('http://127.0.0.1:3050');
externalNode = new zksync.Provider('http://127.0.0.1:3060');

if (fileConfig.loadFromFile) {
const secretsConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'secrets.yaml' });
const generalConfig = loadConfig({ pathToHome, chain: fileConfig.chain, config: 'general.yaml' });

ethRpcUrl = secretsConfig.l1.l1_rpc_url;
apiWeb3JsonRpcHttpUrl = generalConfig.api.web3_json_rpc.http_url;
externalNodeUrl = 'http://127.0.0.1:3150';
extNodeHealthUrl = 'http://127.0.0.1:3171/health';
} else {
ethRpcUrl = process.env.ETH_CLIENT_WEB3_URL ?? 'http://127.0.0.1:8545';
apiWeb3JsonRpcHttpUrl = 'http://127.0.0.1:3050';
externalNodeUrl = 'http://127.0.0.1:3060';
extNodeHealthUrl = 'http://127.0.0.1:3081/health';
}

mainNode = new zksync.Provider(apiWeb3JsonRpcHttpUrl);
externalNode = new zksync.Provider(externalNodeUrl);
await NodeProcess.stopAll('KILL');
});

let fundedWallet: FundedWallet;

before('create test wallet', async () => {
const ethRpcUrl = process.env.ETH_CLIENT_WEB3_URL ?? 'http://127.0.0.1:8545';
console.log(`Using L1 RPC at ${ethRpcUrl}`);
const eth = new ethers.JsonRpcProvider(ethRpcUrl);
fundedWallet = await FundedWallet.create(mainNode, eth);
Expand All @@ -78,18 +95,16 @@ describe('genesis recovery', () => {
}
});

step('drop external node database', async () => {
await dropNodeDatabase(externalNodeEnv);
});

step('drop external node storage', async () => {
await dropNodeStorage(externalNodeEnv);
step('drop external node data', async () => {
await dropNodeData(fileConfig.loadFromFile, externalNodeEnv);
});

step('initialize external node w/o a tree', async () => {
externalNodeProcess = await NodeProcess.spawn(
externalNodeEnv,
'genesis-recovery.log',
pathToHome,
fileConfig.loadFromFile,
NodeComponents.WITH_TREE_FETCHER_AND_NO_TREE
);

Expand All @@ -103,7 +118,7 @@ describe('genesis recovery', () => {

while (!treeFetcherSucceeded || !reorgDetectorSucceeded || !consistencyCheckerSucceeded) {
await sleep(1000);
const health = await getExternalNodeHealth();
const health = await getExternalNodeHealth(extNodeHealthUrl);
if (health === null) {
continue;
}
Expand Down Expand Up @@ -170,13 +185,15 @@ describe('genesis recovery', () => {
externalNodeProcess = await NodeProcess.spawn(
externalNodeEnv,
externalNodeProcess.logs,
pathToHome,
fileConfig.loadFromFile,
NodeComponents.WITH_TREE_FETCHER
);

let isNodeReady = false;
while (!isNodeReady) {
await sleep(1000);
const health = await getExternalNodeHealth();
const health = await getExternalNodeHealth(extNodeHealthUrl);
if (health === null) {
continue;
}
Expand All @@ -197,7 +214,7 @@ describe('genesis recovery', () => {

while (!treeSucceeded || !reorgDetectorSucceeded || !consistencyCheckerSucceeded) {
await sleep(1000);
const health = await getExternalNodeHealth();
const health = await getExternalNodeHealth(extNodeHealthUrl);
if (health === null) {
continue;
}
Expand Down
Loading

0 comments on commit 0c0d10a

Please sign in to comment.