From df50cce18374b6af51c685c30cb002d84e719308 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Sat, 25 Nov 2023 16:45:54 +0100 Subject: [PATCH 01/17] feat: support remote signer in voluntary exit command --- .../cli/src/cmds/validator/voluntaryExit.ts | 52 ++++-- .../e2e/voluntaryExitRemoteSigner.test.ts | 171 ++++++++++++++++++ packages/validator/src/index.ts | 1 + 3 files changed, 209 insertions(+), 15 deletions(-) create mode 100644 packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 2b56751edf41..7df524fb63e5 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -1,16 +1,24 @@ import inquirer from "inquirer"; +import bls from "@chainsafe/bls"; import { - computeSigningRoot, computeEpochAtSlot, + computeSigningRoot, computeStartSlotAtEpoch, getCurrentSlot, } from "@lodestar/state-transition"; import {createBeaconConfig} from "@lodestar/config"; -import {ssz, phase0} from "@lodestar/types"; +import {phase0, ssz} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; -import {Signer, SignerLocal, SignerType} from "@lodestar/validator"; +import { + externalSignerPostSignature, + SignableMessageType, + Signer, + SignerLocal, + SignerRemote, + SignerType, +} from "@lodestar/validator"; import {Api, ApiError, getClient} from "@lodestar/api"; -import {ensure0xPrefix, CliCommand, YargsError} from "../../util/index.js"; +import {CliCommand, ensure0xPrefix, YargsError} from "../../util/index.js"; import {GlobalArgs} from "../../options/index.js"; import {getBeaconConfigFromArgs} from "../../config/index.js"; import {IValidatorCliArgs} from "./options.js"; @@ -46,7 +54,7 @@ If no `pubkeys` are provided, it will exit all validators that have been importe }, pubkeys: { - description: "Public keys to exit, must be available as local signers", + description: "Public keys to exit", type: "array", string: true, // Ensures the pubkey string is not automatically converted to numbers coerce: (pubkeys: string[]): string[] => @@ -106,25 +114,41 @@ ${validatorsToExit.map((v) => `${v.pubkey} ${v.index} ${v.status}`).join("\n")}` } for (const [i, {index, signer, pubkey}] of validatorsToExit.entries()) { - const domain = config.getDomainForVoluntaryExit(computeStartSlotAtEpoch(exitEpoch)); + const slot = computeStartSlotAtEpoch(exitEpoch); + const domain = config.getDomainForVoluntaryExit(slot); const voluntaryExit: phase0.VoluntaryExit = {epoch: exitEpoch, validatorIndex: index}; const signingRoot = computeSigningRoot(ssz.phase0.VoluntaryExit, voluntaryExit, domain); + let signature; + switch (signer.type) { + case SignerType.Local: + signature = signer.secretKey.sign(signingRoot); + break; + case SignerType.Remote: { + const signatureHex = await externalSignerPostSignature(config, signer.url, pubkey, signingRoot, slot, { + data: voluntaryExit, + type: SignableMessageType.VOLUNTARY_EXIT, + }); + signature = bls.Signature.fromHex(signatureHex); + break; + } + default: + throw new YargsError(`Unexpected signer type for ${pubkey}`); + } ApiError.assert( await client.beacon.submitPoolVoluntaryExit({ message: voluntaryExit, - signature: signer.secretKey.sign(signingRoot).toBytes(), + signature: signature.toBytes(), }) ); - console.log(`Submitted voluntary exit for ${pubkey} ${i + 1}/${signersToExit.length}`); } }, }; -type SignerLocalPubkey = {signer: SignerLocal; pubkey: string}; +type SignerPubkey = {signer: SignerLocal | SignerRemote; pubkey: string}; -function selectSignersToExit(args: VoluntaryExitArgs, signers: Signer[]): SignerLocalPubkey[] { +function selectSignersToExit(args: VoluntaryExitArgs, signers: Signer[]): SignerPubkey[] { const signersWithPubkey = signers.map((signer) => ({ signer, pubkey: getSignerPubkeyHex(signer), @@ -132,14 +156,12 @@ function selectSignersToExit(args: VoluntaryExitArgs, signers: Signer[]): Signer if (args.pubkeys) { const signersByPubkey = new Map(signersWithPubkey.map(({pubkey, signer}) => [pubkey, signer])); - const selectedSigners: SignerLocalPubkey[] = []; + const selectedSigners: SignerPubkey[] = []; for (const pubkey of args.pubkeys) { const signer = signersByPubkey.get(pubkey); if (!signer) { throw new YargsError(`Unknown pubkey ${pubkey}`); - } else if (signer.type !== SignerType.Local) { - throw new YargsError(`pubkey ${pubkey} is not a local signer`); } else { selectedSigners.push({pubkey, signer}); } @@ -147,12 +169,12 @@ function selectSignersToExit(args: VoluntaryExitArgs, signers: Signer[]): Signer return selectedSigners; } else { - return signersWithPubkey.filter((signer): signer is SignerLocalPubkey => signer.signer.type === SignerType.Local); + return signersWithPubkey; } } // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -async function resolveValidatorIndexes(client: Api, signersToExit: SignerLocalPubkey[]) { +async function resolveValidatorIndexes(client: Api, signersToExit: SignerPubkey[]) { const pubkeys = signersToExit.map(({pubkey}) => pubkey); const res = await client.beacon.getStateValidators("head", {id: pubkeys}); diff --git a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts new file mode 100644 index 000000000000..9f88003387de --- /dev/null +++ b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts @@ -0,0 +1,171 @@ +import path from "node:path"; +import fs from "node:fs"; +import {GenericContainer, Wait, StartedTestContainer} from "testcontainers"; +import tmp from "tmp"; +import {retry} from "@lodestar/utils"; +import {ApiError, getClient} from "@lodestar/api"; +import {config} from "@lodestar/config/default"; +import {interopSecretKey} from "@lodestar/state-transition"; +import {spawnCliCommand, execCliCommand} from "@lodestar/test-utils"; +import {getMochaContext} from "@lodestar/test-utils/mocha"; +import {testFilesDir} from "../utils.js"; + +let web3signerUrl: string; +// using the latest image to be alerted in case there is a breaking change +const web3signerVersion = "latest"; + +describe("voluntaryExit using remote signer", function () { + const testContext = getMochaContext(this); + this.timeout("60s"); + + let startedContainer: StartedTestContainer; + + after("stop web3signer container", async function () { + await startedContainer.stop(); + }); + + before("start web3signer container", async function () { + this.timeout("300s"); + // path to store configuration + const tmpDir = tmp.dirSync({ + unsafeCleanup: true, + // In Github runner NodeJS process probably runs as root, so web3signer doesn't have permissions to read config dir + mode: 755, + }); + // Apply permissions again to hopefully make Github runner happy >.< + fs.chmodSync(tmpDir.name, 0o755); + + const configDirPathHost = tmpDir.name; + const configDirPathContainer = "/var/web3signer/config"; + + // keystore content and file paths + const passwordFilename = "password.txt"; + const password = "password"; + + const keystoreStrings = getKeystoresToExit(); + for (const [idx, keystoreString] of keystoreStrings.entries()) { + fs.writeFileSync(path.join(configDirPathHost, `keystore-${idx}.json`), keystoreString); + } + fs.writeFileSync(path.join(configDirPathHost, passwordFilename), password); + const port = 9000; + + startedContainer = await new GenericContainer(`consensys/web3signer:${web3signerVersion}`) + .withHealthCheck({ + test: ["CMD-SHELL", `curl -f http://localhost:${port}/healthcheck || exit 1`], + interval: 1000, + timeout: 3000, + retries: 5, + startPeriod: 1000, + }) + .withWaitStrategy(Wait.forHealthCheck()) + .withExposedPorts(port) + .withBindMounts([{source: configDirPathHost, target: configDirPathContainer, mode: "ro"}]) + .withCommand([ + "eth2", + `--keystores-path=${configDirPathContainer}`, + // Don't use path.join here, the container is running on unix filesystem + `--keystores-password-file=${configDirPathContainer}/${passwordFilename}`, + "--slashing-protection-enabled=false", + ]) + .start(); + + web3signerUrl = `http://localhost:${startedContainer.getMappedPort(port)}`; + + const stream = await startedContainer.logs(); + stream + .on("data", (line) => process.stdout.write(line)) + .on("err", (line) => process.stderr.write(line)) + // eslint-disable-next-line no-console + .on("end", () => console.log("Stream closed")); + }); + + it("Perform a voluntary exit", async () => { + const restPort = 9596; + + const devBnProc = await spawnCliCommand( + "packages/cli/bin/lodestar.js", + [ + // ⏎ + "dev", + `--dataDir=${path.join(testFilesDir, "dev-voluntary-exit")}`, + "--genesisValidators=8", + "--startValidators=0..7", + "--rest", + `--rest.port=${restPort}`, + // Speed up test to make genesis happen faster + "--params.SECONDS_PER_SLOT=2", + // Allow voluntary exists to be valid immediately + "--params.SHARD_COMMITTEE_PERIOD=0", + ], + {pipeStdioToParent: false, logPrefix: "dev", testContext} + ); + + // Exit early if process exits + devBnProc.on("exit", (code) => { + if (code !== null && code > 0) { + throw new Error(`devBnProc process exited with code ${code}`); + } + }); + + const baseUrl = `http://127.0.0.1:${restPort}`; + // To cleanup the event stream connection + const httpClientController = new AbortController(); + const client = getClient({baseUrl, getAbortSignal: () => httpClientController.signal}, {config}); + + // Wait for beacon node API to be available + genesis + await retry( + async () => { + const head = await client.beacon.getBlockHeader("head"); + ApiError.assert(head); + if (head.response.data.header.message.slot < 1) throw Error("pre-genesis"); + }, + {retryDelay: 1000, retries: 20} + ); + + const indexesToExit = [0, 1]; + const pubkeysToExit = indexesToExit.map((i) => interopSecretKey(i).toPublicKey().toHex()); + + await execCliCommand( + "packages/cli/bin/lodestar.js", + [ + "validator", + "voluntary-exit", + "--network=dev", + "--yes", + `--externalSigner.url=${web3signerUrl}`, + "--externalSigner.fetch=true", + `--server=${baseUrl}`, + `--pubkeys=${pubkeysToExit.join(",")}`, + ], + {pipeStdioToParent: false, logPrefix: "voluntary-exit"} + ); + + for (const pubkey of pubkeysToExit) { + await retry( + async () => { + const res = await client.beacon.getStateValidator("head", pubkey); + ApiError.assert(res); + if (res.response.data.status !== "active_exiting") { + throw Error("Validator not exiting"); + } else { + // eslint-disable-next-line no-console + console.log(`Confirmed validator ${pubkey} = ${res.response.data.status}`); + } + }, + {retryDelay: 1000, retries: 20} + ); + } + + // Disconnect the event stream for the client + httpClientController.abort(); + }); +}); + +function getKeystoresToExit(): string[] { + return [ + // eslint-disable-next-line quotes + `{"crypto": {"kdf": {"function": "scrypt", "params": {"dklen": 32, "n": 262144, "r": 8, "p": 1, "salt": "f75a7a266418913f3094dc6f1286b1cbacd9f1ff7d05413d5f9b289f79b8b53c"}, "message": ""}, "checksum": {"function": "sha256", "params": {}, "message": "5783b95fa51507968d6093a5ededa85e6c9837f85f4a8529f89f6aaeb107b9b8"}, "cipher": {"function": "aes-128-ctr", "params": {"iv": "74d0c19c6e325410cddba27241378ffc"}, "message": "0308b6e1b8b633539c905e739c605e4faa536481f405f8e65647433ae5cffd19"}}, "description": "", "pubkey": "a99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c", "path": "m/12381/3600/0/0/0", "uuid": "7903eb75-80a8-461a-87ab-19e0ca3dfeb0", "version": 4}`, + // eslint-disable-next-line quotes + `{"crypto": {"kdf": {"function": "scrypt", "params": {"dklen": 32, "n": 262144, "r": 8, "p": 1, "salt": "463e4375b2f09982ad19b53ae83b837d069f85f1c486a4c23b4b604cf4419c54"}, "message": ""}, "checksum": {"function": "sha256", "params": {}, "message": "024caab63a417aa7d8b5863dc5170e73a2a8df77cf9c16ec8b1557f366232617"}, "cipher": {"function": "aes-128-ctr", "params": {"iv": "528e9b03cbec63e8adb238209bef7bcf"}, "message": "fd5f912a4241a02e613e730e527a03840f106c5a37ac994233c5485c61b7cdf4"}}, "description": "", "pubkey": "b89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b", "path": "m/12381/3600/0/0/0", "uuid": "f7ce9bcc-aaa1-4023-aec6-1bc03c8af20c", "version": 4}`, + ]; +} diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 6582b660d011..39a331af6657 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -15,6 +15,7 @@ export { externalSignerGetKeys, externalSignerPostSignature, externalSignerUpCheck, + SignableMessageType, } from "./util/externalSignerClient.js"; // Types From acb5dcdff27a4667614d8a8976206759278bc4e0 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:20:42 +0100 Subject: [PATCH 02/17] Pin web3signer version --- packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts | 2 +- packages/validator/test/e2e/web3signer.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts index 9f88003387de..64453d2ecac5 100644 --- a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts +++ b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts @@ -12,7 +12,7 @@ import {testFilesDir} from "../utils.js"; let web3signerUrl: string; // using the latest image to be alerted in case there is a breaking change -const web3signerVersion = "latest"; +const web3signerVersion = "23.11.0"; describe("voluntaryExit using remote signer", function () { const testContext = getMochaContext(this); diff --git a/packages/validator/test/e2e/web3signer.test.ts b/packages/validator/test/e2e/web3signer.test.ts index 3cd6db833750..70c940f7154a 100644 --- a/packages/validator/test/e2e/web3signer.test.ts +++ b/packages/validator/test/e2e/web3signer.test.ts @@ -17,7 +17,7 @@ import {Interchange, ISlashingProtection, Signer, SignerType, ValidatorStore} fr import {IndicesService} from "../../src/services/indices.js"; import {testLogger} from "../utils/logger.js"; -const web3signerVersion = "22.8.1"; +const web3signerVersion = "23.11.0"; /** Till what version is the image updated for signature verification */ const validTillSignatureForkSeq = ForkSeq.bellatrix; From 292d12637cd4639ca4a8e6152b27ce2d26ec07ca Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:22:05 +0100 Subject: [PATCH 03/17] Reuse existing Signer type --- packages/cli/src/cmds/validator/voluntaryExit.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 7df524fb63e5..98f4d079b224 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -13,8 +13,6 @@ import { externalSignerPostSignature, SignableMessageType, Signer, - SignerLocal, - SignerRemote, SignerType, } from "@lodestar/validator"; import {Api, ApiError, getClient} from "@lodestar/api"; @@ -146,7 +144,7 @@ ${validatorsToExit.map((v) => `${v.pubkey} ${v.index} ${v.status}`).join("\n")}` }, }; -type SignerPubkey = {signer: SignerLocal | SignerRemote; pubkey: string}; +type SignerPubkey = {signer: Signer; pubkey: string}; function selectSignersToExit(args: VoluntaryExitArgs, signers: Signer[]): SignerPubkey[] { const signersWithPubkey = signers.map((signer) => ({ From 3523db4cc22d39f122285d850b77eab707288f37 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:23:01 +0100 Subject: [PATCH 04/17] Update command description, examples --- packages/cli/src/cmds/validator/options.ts | 2 +- .../cli/src/cmds/validator/voluntaryExit.ts | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index d3af927deca6..bca48491a819 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -297,7 +297,7 @@ export const validatorOptions: CliCommandOptions = { "externalSigner.fetch": { conflicts: ["externalSigner.pubkeys"], - description: "Fetch then list of public keys to validate from an external signer", + description: "Fetch the list of public keys to validate from an external signer", type: "boolean", group: "externalSignerUrl", }, diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 98f4d079b224..5b4091737f6e 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -28,6 +28,9 @@ type VoluntaryExitArgs = { exitEpoch?: number; pubkeys?: string[]; yes?: boolean; + "externalSigner.url"?: string; + "externalSigner.pubkeys"?: string[]; + "externalSigner.fetch"?: boolean; }; export const voluntaryExit: CliCommand = { @@ -35,13 +38,20 @@ export const voluntaryExit: CliCommand + // Parse ["0x11,0x22"] to ["0x11", "0x22"] + pubkeys + .map((item) => item.split(",")) + .flat(1) + .map(ensure0xPrefix), + group: "externalSignerUrl", + }, + + "externalSigner.fetch": { + conflicts: ["externalSigner.pubkeys"], + description: "Fetch the list of public keys to validate from an external signer", + type: "boolean", + group: "externalSignerUrl", + }, }, handler: async (args) => { From ec5880dbe333ed152c280419a81b3d5b21c670b0 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:27:33 +0100 Subject: [PATCH 05/17] Update error message when signers.length 0 --- packages/cli/src/cmds/validator/voluntaryExit.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 5b4091737f6e..baf346ed6981 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -127,9 +127,12 @@ Exiting validators on remote signers is also supported.", // Select signers to exit const signers = await getSignersFromArgs(args, network, {logger: console, signal: new AbortController().signal}); if (signers.length === 0) { - throw new YargsError(`No local keystores found with current args. + throw new YargsError(`No validators to exit found with current args. Ensure --dataDir and --network match values used when importing keys via validator import - or alternatively, import keys by providing --importKeystores arg to voluntary-exit command.`); + or alternatively, import keys by providing --importKeystores arg to voluntary-exit command. + If attempting to exit validators on a remote signer, make sure values are provided for + the necessary --externalSigner options. + `); } const signersToExit = selectSignersToExit(args, signers); const validatorsToExit = await resolveValidatorIndexes(client, signersToExit); From d2bf6231e8acf3038d756e615d9182b8d687c8fe Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:27:54 +0100 Subject: [PATCH 06/17] Use getKeystoresStr in test --- .../test/e2e/voluntaryExitRemoteSigner.test.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts index 64453d2ecac5..0b9ab8be580e 100644 --- a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts +++ b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts @@ -5,10 +5,11 @@ import tmp from "tmp"; import {retry} from "@lodestar/utils"; import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; -import {interopSecretKey} from "@lodestar/state-transition"; +import {interopSecretKey, interopSecretKeys} from "@lodestar/state-transition"; import {spawnCliCommand, execCliCommand} from "@lodestar/test-utils"; import {getMochaContext} from "@lodestar/test-utils/mocha"; import {testFilesDir} from "../utils.js"; +import {getKeystoresStr} from "../utils/keystores.js"; let web3signerUrl: string; // using the latest image to be alerted in case there is a breaking change @@ -42,7 +43,11 @@ describe("voluntaryExit using remote signer", function () { const passwordFilename = "password.txt"; const password = "password"; - const keystoreStrings = getKeystoresToExit(); + const keystoreStrings = await getKeystoresStr( + password, + interopSecretKeys(2).map((key) => key.toHex()) + ); + for (const [idx, keystoreString] of keystoreStrings.entries()) { fs.writeFileSync(path.join(configDirPathHost, `keystore-${idx}.json`), keystoreString); } @@ -160,12 +165,3 @@ describe("voluntaryExit using remote signer", function () { httpClientController.abort(); }); }); - -function getKeystoresToExit(): string[] { - return [ - // eslint-disable-next-line quotes - `{"crypto": {"kdf": {"function": "scrypt", "params": {"dklen": 32, "n": 262144, "r": 8, "p": 1, "salt": "f75a7a266418913f3094dc6f1286b1cbacd9f1ff7d05413d5f9b289f79b8b53c"}, "message": ""}, "checksum": {"function": "sha256", "params": {}, "message": "5783b95fa51507968d6093a5ededa85e6c9837f85f4a8529f89f6aaeb107b9b8"}, "cipher": {"function": "aes-128-ctr", "params": {"iv": "74d0c19c6e325410cddba27241378ffc"}, "message": "0308b6e1b8b633539c905e739c605e4faa536481f405f8e65647433ae5cffd19"}}, "description": "", "pubkey": "a99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c", "path": "m/12381/3600/0/0/0", "uuid": "7903eb75-80a8-461a-87ab-19e0ca3dfeb0", "version": 4}`, - // eslint-disable-next-line quotes - `{"crypto": {"kdf": {"function": "scrypt", "params": {"dklen": 32, "n": 262144, "r": 8, "p": 1, "salt": "463e4375b2f09982ad19b53ae83b837d069f85f1c486a4c23b4b604cf4419c54"}, "message": ""}, "checksum": {"function": "sha256", "params": {}, "message": "024caab63a417aa7d8b5863dc5170e73a2a8df77cf9c16ec8b1557f366232617"}, "cipher": {"function": "aes-128-ctr", "params": {"iv": "528e9b03cbec63e8adb238209bef7bcf"}, "message": "fd5f912a4241a02e613e730e527a03840f106c5a37ac994233c5485c61b7cdf4"}}, "description": "", "pubkey": "b89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b", "path": "m/12381/3600/0/0/0", "uuid": "f7ce9bcc-aaa1-4023-aec6-1bc03c8af20c", "version": 4}`, - ]; -} From 3b262d0be9e497b00d1460ec0e3d0cfc895a1bf2 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 20:08:42 +0100 Subject: [PATCH 07/17] Undo adding explicit externalSigner.* options --- .../cli/src/cmds/validator/voluntaryExit.ts | 39 +------------------ 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index baf346ed6981..586e3fd83b75 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -9,12 +9,7 @@ import { import {createBeaconConfig} from "@lodestar/config"; import {phase0, ssz} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; -import { - externalSignerPostSignature, - SignableMessageType, - Signer, - SignerType, -} from "@lodestar/validator"; +import {externalSignerPostSignature, SignableMessageType, Signer, SignerType} from "@lodestar/validator"; import {Api, ApiError, getClient} from "@lodestar/api"; import {CliCommand, ensure0xPrefix, YargsError} from "../../util/index.js"; import {GlobalArgs} from "../../options/index.js"; @@ -28,9 +23,6 @@ type VoluntaryExitArgs = { exitEpoch?: number; pubkeys?: string[]; yes?: boolean; - "externalSigner.url"?: string; - "externalSigner.pubkeys"?: string[]; - "externalSigner.fetch"?: boolean; }; export const voluntaryExit: CliCommand = { @@ -77,35 +69,6 @@ Exiting validators on remote signers is also supported.", description: "Skip confirmation prompt", type: "boolean", }, - - // Remote signer - - "externalSigner.url": { - description: "URL to connect to an external signing server", - type: "string", - group: "externalSignerUrl", - }, - - "externalSigner.pubkeys": { - description: - "List of validator public keys used by an external signer. May also provide a single string a comma separated public keys", - type: "array", - string: true, // Ensures the pubkey string is not automatically converted to numbers - coerce: (pubkeys: string[]): string[] => - // Parse ["0x11,0x22"] to ["0x11", "0x22"] - pubkeys - .map((item) => item.split(",")) - .flat(1) - .map(ensure0xPrefix), - group: "externalSignerUrl", - }, - - "externalSigner.fetch": { - conflicts: ["externalSigner.pubkeys"], - description: "Fetch the list of public keys to validate from an external signer", - type: "boolean", - group: "externalSignerUrl", - }, }, handler: async (args) => { From b49f9a2a5be1af7e3a74a75cc38ec5e4dceb5738 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Mon, 27 Nov 2023 20:09:17 +0100 Subject: [PATCH 08/17] Remove extra line from description, example is sufficient --- packages/cli/src/cmds/validator/voluntaryExit.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 586e3fd83b75..0e8560ca5e61 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -30,8 +30,7 @@ export const voluntaryExit: CliCommand Date: Mon, 27 Nov 2023 20:14:40 +0100 Subject: [PATCH 09/17] Deduplicate, implement startExternalSigner --- .../cli/test/e2e/importFromFsDirect.test.ts | 2 +- .../cli/test/e2e/importFromFsPreStep.test.ts | 2 +- .../test/e2e/importKeystoresFromApi.test.ts | 2 +- .../e2e/propserConfigfromKeymanager.test.ts | 2 +- .../e2e/voluntaryExitRemoteSigner.test.ts | 91 +++-------- .../decryptKeystoreDefinitions.test.ts | 2 +- packages/test-utils/package.json | 6 +- packages/test-utils/src/externalSigner.ts | 74 +++++++++ packages/test-utils/src/index.ts | 2 + .../utils => test-utils/src}/keystores.ts | 0 .../validator/test/e2e/web3signer.test.ts | 142 ++++-------------- .../test/utils/createExternalSignerServer.ts | 50 ------ yarn.lock | 93 +++++++++++- 13 files changed, 223 insertions(+), 245 deletions(-) create mode 100644 packages/test-utils/src/externalSigner.ts rename packages/{cli/test/utils => test-utils/src}/keystores.ts (100%) delete mode 100644 packages/validator/test/utils/createExternalSignerServer.ts diff --git a/packages/cli/test/e2e/importFromFsDirect.test.ts b/packages/cli/test/e2e/importFromFsDirect.test.ts index f55587ced2e3..9d64421c97af 100644 --- a/packages/cli/test/e2e/importFromFsDirect.test.ts +++ b/packages/cli/test/e2e/importFromFsDirect.test.ts @@ -2,10 +2,10 @@ import fs from "node:fs"; import path from "node:path"; import {rimraf} from "rimraf"; import {getMochaContext} from "@lodestar/test-utils/mocha"; +import {getKeystoresStr} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js"; -import {getKeystoresStr} from "../utils/keystores.js"; describe("import from fs same cmd as validate", function () { const testContext = getMochaContext(this); diff --git a/packages/cli/test/e2e/importFromFsPreStep.test.ts b/packages/cli/test/e2e/importFromFsPreStep.test.ts index 9dd48acaa1a6..efbe7a6b35e4 100644 --- a/packages/cli/test/e2e/importFromFsPreStep.test.ts +++ b/packages/cli/test/e2e/importFromFsPreStep.test.ts @@ -4,10 +4,10 @@ import {rimraf} from "rimraf"; import {expect} from "chai"; import {getMochaContext} from "@lodestar/test-utils/mocha"; import {execCliCommand} from "@lodestar/test-utils"; +import {getKeystoresStr} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js"; -import {getKeystoresStr} from "../utils/keystores.js"; describe("import from fs then validate", function () { const testContext = getMochaContext(this); diff --git a/packages/cli/test/e2e/importKeystoresFromApi.test.ts b/packages/cli/test/e2e/importKeystoresFromApi.test.ts index d7bd90033c90..dcd0f38b2182 100644 --- a/packages/cli/test/e2e/importKeystoresFromApi.test.ts +++ b/packages/cli/test/e2e/importKeystoresFromApi.test.ts @@ -7,11 +7,11 @@ import {Interchange} from "@lodestar/validator"; import {ApiError, HttpStatusCode} from "@lodestar/api"; import {bufferStderr, spawnCliCommand} from "@lodestar/test-utils"; import {getMochaContext} from "@lodestar/test-utils/mocha"; +import {getKeystoresStr} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; import {expectDeepEquals} from "../utils/runUtils.js"; import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js"; -import {getKeystoresStr} from "../utils/keystores.js"; describe("import keystores from api", function () { const testContext = getMochaContext(this); diff --git a/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts b/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts index 01a2ba81c984..9d6eeafedfd3 100644 --- a/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts +++ b/packages/cli/test/e2e/propserConfigfromKeymanager.test.ts @@ -3,11 +3,11 @@ import {rimraf} from "rimraf"; import {Interchange} from "@lodestar/validator"; import {ApiError} from "@lodestar/api"; import {getMochaContext} from "@lodestar/test-utils/mocha"; +import {getKeystoresStr} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js"; import {expectDeepEquals} from "../utils/runUtils.js"; import {startValidatorWithKeyManager} from "../utils/validator.js"; -import {getKeystoresStr} from "../utils/keystores.js"; describe("import keystores from api, test DefaultProposerConfig", function () { this.timeout("30s"); diff --git a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts index 0b9ab8be580e..11e1d19ef055 100644 --- a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts +++ b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts @@ -1,92 +1,37 @@ import path from "node:path"; -import fs from "node:fs"; -import {GenericContainer, Wait, StartedTestContainer} from "testcontainers"; -import tmp from "tmp"; import {retry} from "@lodestar/utils"; import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {interopSecretKey, interopSecretKeys} from "@lodestar/state-transition"; -import {spawnCliCommand, execCliCommand} from "@lodestar/test-utils"; +import {spawnCliCommand, execCliCommand, startExternalSigner, ExternalSignerTests} from "@lodestar/test-utils"; import {getMochaContext} from "@lodestar/test-utils/mocha"; +import {getKeystoresStr} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; -import {getKeystoresStr} from "../utils/keystores.js"; - -let web3signerUrl: string; -// using the latest image to be alerted in case there is a breaking change -const web3signerVersion = "23.11.0"; describe("voluntaryExit using remote signer", function () { - const testContext = getMochaContext(this); - this.timeout("60s"); - - let startedContainer: StartedTestContainer; - - after("stop web3signer container", async function () { - await startedContainer.stop(); - }); + this.timeout("30s"); - before("start web3signer container", async function () { - this.timeout("300s"); - // path to store configuration - const tmpDir = tmp.dirSync({ - unsafeCleanup: true, - // In Github runner NodeJS process probably runs as root, so web3signer doesn't have permissions to read config dir - mode: 755, - }); - // Apply permissions again to hopefully make Github runner happy >.< - fs.chmodSync(tmpDir.name, 0o755); - - const configDirPathHost = tmpDir.name; - const configDirPathContainer = "/var/web3signer/config"; + let externalSigner: ExternalSignerTests; - // keystore content and file paths - const passwordFilename = "password.txt"; + before("set up validator stores", async () => { const password = "password"; + externalSigner = await startExternalSigner({ + keystoreStrings: await getKeystoresStr( + password, + interopSecretKeys(2).map((k) => k.toHex()) + ), + password: password, + }); + }); - const keystoreStrings = await getKeystoresStr( - password, - interopSecretKeys(2).map((key) => key.toHex()) - ); - - for (const [idx, keystoreString] of keystoreStrings.entries()) { - fs.writeFileSync(path.join(configDirPathHost, `keystore-${idx}.json`), keystoreString); - } - fs.writeFileSync(path.join(configDirPathHost, passwordFilename), password); - const port = 9000; - - startedContainer = await new GenericContainer(`consensys/web3signer:${web3signerVersion}`) - .withHealthCheck({ - test: ["CMD-SHELL", `curl -f http://localhost:${port}/healthcheck || exit 1`], - interval: 1000, - timeout: 3000, - retries: 5, - startPeriod: 1000, - }) - .withWaitStrategy(Wait.forHealthCheck()) - .withExposedPorts(port) - .withBindMounts([{source: configDirPathHost, target: configDirPathContainer, mode: "ro"}]) - .withCommand([ - "eth2", - `--keystores-path=${configDirPathContainer}`, - // Don't use path.join here, the container is running on unix filesystem - `--keystores-password-file=${configDirPathContainer}/${passwordFilename}`, - "--slashing-protection-enabled=false", - ]) - .start(); - - web3signerUrl = `http://localhost:${startedContainer.getMappedPort(port)}`; - - const stream = await startedContainer.logs(); - stream - .on("data", (line) => process.stdout.write(line)) - .on("err", (line) => process.stderr.write(line)) - // eslint-disable-next-line no-console - .on("end", () => console.log("Stream closed")); + after("stop external signer container", async () => { + await externalSigner.container.stop(); }); it("Perform a voluntary exit", async () => { - const restPort = 9596; + const testContext = getMochaContext(this); + const restPort = 9596; const devBnProc = await spawnCliCommand( "packages/cli/bin/lodestar.js", [ @@ -137,7 +82,7 @@ describe("voluntaryExit using remote signer", function () { "voluntary-exit", "--network=dev", "--yes", - `--externalSigner.url=${web3signerUrl}`, + `--externalSigner.url=${externalSigner.url}`, "--externalSigner.fetch=true", `--server=${baseUrl}`, `--pubkeys=${pubkeysToExit.join(",")}`, diff --git a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts index 296105012b69..dc3ce5dff5ad 100644 --- a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts +++ b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts @@ -2,8 +2,8 @@ import fs from "node:fs"; import path from "node:path"; import {rimraf} from "rimraf"; import {expect} from "chai"; +import {getKeystoresStr} from "@lodestar/test-utils"; import {cachedSeckeysHex} from "../../utils/cachedKeys.js"; -import {getKeystoresStr} from "../../utils/keystores.js"; import {testFilesDir} from "../../utils.js"; import {decryptKeystoreDefinitions} from "../../../src/cmds/validator/keymanager/decryptKeystoreDefinitions.js"; import {LocalKeystoreDefinition} from "../../../src/cmds/validator/keymanager/interface.js"; diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index b03a5f7c68d7..f073fe003fdf 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -61,11 +61,15 @@ "blockchain" ], "dependencies": { + "@chainsafe/bls": "^7.1.2", + "@chainsafe/bls-keystore": "^3.0.0", "@lodestar/utils": "^1.12.0", "axios": "^1.3.4", "chai": "^4.3.7", "mocha": "^10.2.0", - "sinon": "^15.0.3" + "sinon": "^15.0.3", + "testcontainers": "^10.3.2", + "tmp": "^0.2.1" }, "devDependencies": { "@types/mocha": "^10.0.1", diff --git a/packages/test-utils/src/externalSigner.ts b/packages/test-utils/src/externalSigner.ts new file mode 100644 index 000000000000..57b41c6a40c9 --- /dev/null +++ b/packages/test-utils/src/externalSigner.ts @@ -0,0 +1,74 @@ +import fs from "node:fs"; +import path from "node:path"; +import tmp from "tmp"; +import {GenericContainer, Wait, StartedTestContainer} from "testcontainers"; + +const web3signerVersion = "23.11.0"; + +export type ExternalSignerTests = { + container: StartedTestContainer; + url: string; +}; + +export async function startExternalSigner({ + keystoreStrings, + password, +}: { + keystoreStrings: string[]; + password: string; +}): Promise { + // path to store configuration + const tmpDir = tmp.dirSync({ + unsafeCleanup: true, + // In Github runner NodeJS process probably runs as root, so web3signer doesn't have permissions to read config dir + mode: 755, + }); + // Apply permissions again to hopefully make Github runner happy >.< + fs.chmodSync(tmpDir.name, 0o755); + + const configDirPathHost = tmpDir.name; + const configDirPathContainer = "/var/web3signer/config"; + + // keystore content and file paths + const passwordFilename = "password.txt"; + + for (const [idx, keystoreString] of keystoreStrings.entries()) { + fs.writeFileSync(path.join(configDirPathHost, `keystore-${idx}.json`), keystoreString); + } + fs.writeFileSync(path.join(configDirPathHost, passwordFilename), password); + const port = 9000; + + const startedContainer = await new GenericContainer(`consensys/web3signer:${web3signerVersion}`) + .withHealthCheck({ + test: ["CMD-SHELL", `curl -f http://localhost:${port}/healthcheck || exit 1`], + interval: 1000, + timeout: 3000, + retries: 5, + startPeriod: 1000, + }) + .withWaitStrategy(Wait.forHealthCheck()) + .withExposedPorts(port) + .withBindMounts([{source: configDirPathHost, target: configDirPathContainer, mode: "ro"}]) + .withCommand([ + "eth2", + `--keystores-path=${configDirPathContainer}`, + // Don't use path.join here, the container is running on unix filesystem + `--keystores-password-file=${configDirPathContainer}/${passwordFilename}`, + "--slashing-protection-enabled=false", + ]) + .start(); + + const url = `http://localhost:${startedContainer.getMappedPort(port)}`; + + const stream = await startedContainer.logs(); + stream + .on("data", (line) => process.stdout.write(line)) + .on("err", (line) => process.stderr.write(line)) + // eslint-disable-next-line no-console + .on("end", () => console.log("Stream closed")); + + return { + container: startedContainer, + url: url, + }; +} diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index 84f0efe0f587..1b90c5419f60 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -1,5 +1,7 @@ export * from "./cli.js"; export * from "./childProcess.js"; +export * from "./externalSigner.js"; +export * from "./keystores.js"; export * from "./path.js"; export * from "./timeout.js"; export * from "./http.js"; diff --git a/packages/cli/test/utils/keystores.ts b/packages/test-utils/src/keystores.ts similarity index 100% rename from packages/cli/test/utils/keystores.ts rename to packages/test-utils/src/keystores.ts diff --git a/packages/validator/test/e2e/web3signer.test.ts b/packages/validator/test/e2e/web3signer.test.ts index 70c940f7154a..e8fe205ad6ab 100644 --- a/packages/validator/test/e2e/web3signer.test.ts +++ b/packages/validator/test/e2e/web3signer.test.ts @@ -1,44 +1,40 @@ -import fs from "node:fs"; -import path from "node:path"; -import tmp from "tmp"; import {expect} from "chai"; -import {GenericContainer, Wait, StartedTestContainer} from "testcontainers"; -import {Keystore} from "@chainsafe/bls-keystore"; -import bls from "@chainsafe/bls"; import {fromHex, toHex} from "@lodestar/utils"; import {config} from "@lodestar/config/default"; -import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {computeStartSlotAtEpoch, interopSecretKey, interopSecretKeys} from "@lodestar/state-transition"; import {createBeaconConfig} from "@lodestar/config"; import {genesisData} from "@lodestar/config/networks"; -import {getClient, routes} from "@lodestar/api"; +import {getClient} from "@lodestar/api"; import {ssz} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; +import {ExternalSignerTests, getKeystoresStr, startExternalSigner} from "@lodestar/test-utils"; import {Interchange, ISlashingProtection, Signer, SignerType, ValidatorStore} from "../../src/index.js"; import {IndicesService} from "../../src/services/indices.js"; import {testLogger} from "../utils/logger.js"; -const web3signerVersion = "23.11.0"; /** Till what version is the image updated for signature verification */ -const validTillSignatureForkSeq = ForkSeq.bellatrix; +const validTillSignatureForkSeq = ForkSeq.capella; /* eslint-disable no-console */ describe("web3signer signature test", function () { this.timeout("60s"); - let validatorStoreRemote: ValidatorStore; - let validatorStoreLocal: ValidatorStore; - let startedContainer: StartedTestContainer; - - const pubkey = "0x8837af2a7452aff5a8b6906c3e5adefce5690e1bba6d73d870b9e679fece096b97a255bae0978e3a344aa832f68c6b47"; - const pubkeyBytes = fromHex(pubkey); const altairSlot = 2375711; const epoch = 0; // Sample validator const validatorIndex = 4; const subcommitteeIndex = 0; - const duty: routes.validator.AttesterDuty = { + const secretKey = interopSecretKey(0); + const pubkeyBytes = secretKey.toPublicKey().toBytes(); + + let validatorStoreRemote: ValidatorStore; + let validatorStoreLocal: ValidatorStore; + + let externalSigner: ExternalSignerTests; + + const duty = { slot: altairSlot, committeeIndex: 0, committeeLength: 120, @@ -48,71 +44,26 @@ describe("web3signer signature test", function () { pubkey: pubkeyBytes, }; - after("stop container", async function () { - await startedContainer.stop(); - }); - - before("start container", async function () { - this.timeout("300s"); - // path to store configuration - const tmpDir = tmp.dirSync({ - unsafeCleanup: true, - // In Github runner NodeJS process probably runs as root, so web3signer doesn't have permissions to read config dir - mode: 755, - }); - // Apply permissions again to hopefully make Github runner happy >.< - fs.chmodSync(tmpDir.name, 0o755); - - const configDirPathHost = tmpDir.name; - const configDirPathContainer = "/var/web3signer/config"; - - // keystore content and file paths - // const keystoreStr = getKeystore(); - // const password = "password"; - const passwordFilename = "password.txt"; + before("set up validator stores", async () => { + validatorStoreLocal = await getValidatorStore({type: SignerType.Local, secretKey: secretKey}); - const keystoreStr = getKeystore(); const password = "password"; + externalSigner = await startExternalSigner({ + keystoreStrings: await getKeystoresStr( + password, + interopSecretKeys(2).map((k) => k.toHex()) + ), + password: password, + }); + validatorStoreRemote = await getValidatorStore({ + type: SignerType.Remote, + url: externalSigner.url, + pubkey: secretKey.toPublicKey().toHex(), + }); + }); - fs.writeFileSync(path.join(configDirPathHost, "keystore.json"), keystoreStr); - fs.writeFileSync(path.join(configDirPathHost, passwordFilename), password); - - const secretKey = bls.SecretKey.fromBytes(await Keystore.parse(keystoreStr).decrypt(password)); - - const port = 9000; - - // using the latest image to be alerted in case there is a breaking change - startedContainer = await new GenericContainer(`consensys/web3signer:${web3signerVersion}`) - .withHealthCheck({ - test: ["CMD-SHELL", `curl -f http://localhost:${port}/healthcheck || exit 1`], - interval: 1000, - timeout: 3000, - retries: 5, - startPeriod: 1000, - }) - .withWaitStrategy(Wait.forHealthCheck()) - .withExposedPorts(port) - .withBindMounts([{source: configDirPathHost, target: configDirPathContainer, mode: "ro"}]) - .withCommand([ - "eth2", - `--keystores-path=${configDirPathContainer}`, - // Don't use path.join here, the container is running on unix filesystem - `--keystores-password-file=${configDirPathContainer}/${passwordFilename}`, - "--slashing-protection-enabled=false", - ]) - .start(); - - const web3signerUrl = `http://localhost:${startedContainer.getMappedPort(port)}`; - - // http://localhost:9000/api/v1/eth2/sign/0x8837af2a7452aff5a8b6906c3e5adefce5690e1bba6d73d870b9e679fece096b97a255bae0978e3a344aa832f68c6b47 - validatorStoreRemote = await getValidatorStore({type: SignerType.Remote, url: web3signerUrl, pubkey}); - validatorStoreLocal = await getValidatorStore({type: SignerType.Local, secretKey}); - - const stream = await startedContainer.logs(); - stream - .on("data", (line) => process.stdout.write(line)) - .on("err", (line) => process.stderr.write(line)) - .on("end", () => console.log("Stream closed")); + after("stop external signer container", async () => { + await externalSigner.container.stop(); }); for (const fork of config.forksAscendingEpochOrder) { @@ -261,36 +212,3 @@ class SlashingProtectionDisabled implements ISlashingProtection { throw Error("not implemented"); } } - -function getKeystore(): string { - return `{ - "version": 4, - "uuid": "f31f3377-694d-4943-8686-5b20356b2597", - "path": "m/12381/3600/0/0/0", - "pubkey": "8837af2a7452aff5a8b6906c3e5adefce5690e1bba6d73d870b9e679fece096b97a255bae0978e3a344aa832f68c6b47", - "crypto": { - "kdf": { - "function": "pbkdf2", - "params": { - "dklen": 32, - "c": 262144, - "prf": "hmac-sha256", - "salt": "ab2c11fe1a288a8344972e5e03a746f42867f5a9e749bf286f8e26cf16702c93" - }, - "message": "" - }, - "checksum": { - "function": "sha256", - "params": {}, - "message": "1f0eda362360b51b85591e99fee6c5d030cc48f36af28eb055b19a2bf55b38a6" - }, - "cipher": { - "function": "aes-128-ctr", - "params": { - "iv": "acf3173c5d0b074e1646bb6058dc0f2a" - }, - "message": "402d1cecaa378e4f079c96437bd1d4771e09a85df2073d014b43980b623b9978" - } - } - }`; -} diff --git a/packages/validator/test/utils/createExternalSignerServer.ts b/packages/validator/test/utils/createExternalSignerServer.ts deleted file mode 100644 index 9c09f7e33beb..000000000000 --- a/packages/validator/test/utils/createExternalSignerServer.ts +++ /dev/null @@ -1,50 +0,0 @@ -import fastify from "fastify"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; -import type {SecretKey} from "@chainsafe/bls/types"; -import {PubkeyHex} from "../../src/types.js"; - -/** - * Creates a fastify server with registered remote signer routes. - * Must call .listen() on the server to start - */ -export function createExternalSignerServer(secretKeys: SecretKey[]): ReturnType { - const secretKeyMap = new Map(); - for (const secretKey of secretKeys) { - const pubkeyHex = toHexString(secretKey.toPublicKey().toBytes()); - secretKeyMap.set(pubkeyHex, secretKey); - } - - const server = fastify(); - - server.get("/upcheck", async () => { - return {status: "OK"}; - }); - - server.get("/api/v1/eth2/publicKeys", async () => { - return Array.from(secretKeyMap.keys()); - }); - - /* eslint-disable @typescript-eslint/naming-convention */ - server.post<{ - Params: { - /** BLS public key as a hex string. */ - identifier: string; - }; - Body: { - /** Data to sign as a hex string */ - signingRoot: string; - }; - }>("/api/v1/eth2/sign/:identifier", async (req) => { - const pubkeyHex: string = req.params.identifier; - const signingRootHex: string = req.body.signingRoot; - - const secretKey = secretKeyMap.get(pubkeyHex); - if (!secretKey) { - throw Error(`pubkey not known ${pubkeyHex}`); - } - - return {signature: secretKey.sign(fromHexString(signingRootHex)).toHex()}; - }); - - return server; -} diff --git a/yarn.lock b/yarn.lock index 8c78c64bd89c..a7d4c5921d81 100644 --- a/yarn.lock +++ b/yarn.lock @@ -497,6 +497,14 @@ ethereum-cryptography "^0.1.3" uuid "^3.3.3" +"@chainsafe/bls-keystore@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/bls-keystore/-/bls-keystore-3.0.0.tgz#e28c979f7664417e4917fa0d4d32fa2b9416e9c6" + integrity sha512-vlRIIXnn555wq2emhqnSR7btno17M0sCcfdQ+Dhgr7IH6n0CMoTGw9qcrpnNYwM+9OPm3matSYeZc9mNlXf7fQ== + dependencies: + ethereum-cryptography "^1.0.0" + uuid "8.3.2" + "@chainsafe/bls@7.1.1": version "7.1.1" resolved "https://registry.npmjs.org/@chainsafe/bls/-/bls-7.1.1.tgz" @@ -506,6 +514,15 @@ bls-eth-wasm "^0.4.8" randombytes "^2.1.0" +"@chainsafe/bls@^7.1.2": + version "7.1.2" + resolved "https://registry.yarnpkg.com/@chainsafe/bls/-/bls-7.1.2.tgz#9150b9ae7ce31bbd3dfca18f46108f81caf2786a" + integrity sha512-eVFxfUpKJmuuu04vEqmCtUuV9jHvVssc2Kz0pUGCSUGsKdr0UAu3jOpkviqLXgfpip+N0ZHxoLbCGqJhXB5TXQ== + dependencies: + "@chainsafe/bls-keygen" "^0.4.0" + bls-eth-wasm "^0.4.8" + randombytes "^2.1.0" + "@chainsafe/blst@^0.2.9": version "0.2.9" resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-0.2.9.tgz#b356b47759c3ce127677227fc24faa3ac6c72032" @@ -2931,6 +2948,14 @@ "@types/docker-modem" "*" "@types/node" "*" +"@types/dockerode@^3.3.21": + version "3.3.23" + resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.23.tgz#07b2084013d01e14d5d97856446f4d9c9f27c223" + integrity sha512-Lz5J+NFgZS4cEVhquwjIGH4oQwlVn2h7LXD3boitujBnzOE5o7s9H8hchEjoDK2SlRsJTogdKnQeiJgPPKLIEw== + dependencies: + "@types/docker-modem" "*" + "@types/node" "*" + "@types/eslint-scope@^3.7.3": version "3.7.4" resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.4.tgz#37fc1223f0786c39627068a12e94d6e6fc61de16" @@ -3922,6 +3947,19 @@ archiver@^5.3.1: tar-stream "^2.2.0" zip-stream "^4.1.0" +archiver@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.2.tgz#99991d5957e53bd0303a392979276ac4ddccf3b0" + integrity sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw== + dependencies: + archiver-utils "^2.1.0" + async "^3.2.4" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.1.2" + tar-stream "^2.2.0" + zip-stream "^4.1.0" + archy@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz" @@ -4112,6 +4150,11 @@ async@^3.2.3, async@~3.2.2: resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== +async@^3.2.4: + version "3.2.5" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" + integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -5826,9 +5869,9 @@ dir-glob@^3.0.1: path-type "^4.0.0" dns-over-http-resolver@^2.1.0, dns-over-http-resolver@^2.1.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/dns-over-http-resolver/-/dns-over-http-resolver-2.1.2.tgz#fb478af244dd4fed5e0f798a3e6426d92730378c" - integrity sha512-Bjbf6aZjr3HMnwGslZnoW3MJVqgbTsh39EZWpikx2yLl9xEjw4eZhlOHCFhkOu89zoWaS4rqe2Go53TXW4Byiw== + version "2.1.3" + resolved "https://registry.yarnpkg.com/dns-over-http-resolver/-/dns-over-http-resolver-2.1.3.tgz#bb7f2e10cc18d960339a6e30e21b8c1d99be7b38" + integrity sha512-zjRYFhq+CsxPAouQWzOsxNMvEN+SHisjzhX8EMxd2Y0EG3thvn6wXQgMJLnTDImkhe4jhLbOQpXtL10nALBOSA== dependencies: debug "^4.3.1" native-fetch "^4.0.2" @@ -6596,7 +6639,7 @@ ethereum-cryptography@^0.1.3: secp256k1 "^4.0.1" setimmediate "^1.0.5" -ethereum-cryptography@^1.2.0: +ethereum-cryptography@^1.0.0, ethereum-cryptography@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/ethereum-cryptography/-/ethereum-cryptography-1.2.0.tgz#5ccfa183e85fdaf9f9b299a79430c044268c9b3a" integrity sha512-6yFQC9b5ug6/17CQpCyE3k9eKBMdhyVjzUy1WkiuY/E4vj/SXDBbCw8QEIaXqf0Mf2SnY6RmpDcwlUmBSS0EJw== @@ -10341,6 +10384,13 @@ node-fetch@^2.6.12: dependencies: whatwg-url "^5.0.0" +node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.1.0: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -11542,6 +11592,13 @@ properties-reader@^2.2.0: dependencies: mkdirp "^1.0.4" +properties-reader@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/properties-reader/-/properties-reader-2.3.0.tgz#f3ab84224c9535a7a36e011ae489a79a13b472b2" + integrity sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw== + dependencies: + mkdirp "^1.0.4" + protobufjs@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.0.0.tgz#8c678e1351fd926178fce5a4213913e8d990974f" @@ -11864,6 +11921,13 @@ readdir-glob@^1.0.0: dependencies: minimatch "^5.1.0" +readdir-glob@^1.1.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.3.tgz#c3d831f51f5e7bfa62fa2ffbe4b508c640f09584" + integrity sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA== + dependencies: + minimatch "^5.1.0" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz" @@ -13208,6 +13272,27 @@ testcontainers@^10.2.1: tar-fs "^3.0.4" tmp "^0.2.1" +testcontainers@^10.3.2: + version "10.3.2" + resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-10.3.2.tgz#1477771585b328953ef6641f83b43368adac1242" + integrity sha512-IsV6NgS5reHcVF1nJLCDJv1hM9gAWUhLwh9b3ybgzvM3X7T2dcmuLFKt1RAR8qN8k+44tW2Drj7idxW6oeGvvg== + dependencies: + "@balena/dockerignore" "^1.0.2" + "@types/dockerode" "^3.3.21" + archiver "^5.3.2" + async-lock "^1.4.0" + byline "^5.0.0" + debug "^4.3.4" + docker-compose "^0.24.2" + dockerode "^3.3.5" + get-port "^5.1.1" + node-fetch "^2.7.0" + proper-lockfile "^4.1.2" + properties-reader "^2.3.0" + ssh-remote-port-forward "^1.0.4" + tar-fs "^3.0.4" + tmp "^0.2.1" + text-extensions@^1.0.0: version "1.9.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" From ea3f556b7179f685021c67e788b7e7f33c18f3f9 Mon Sep 17 00:00:00 2001 From: eth2353 <70237279+eth2353@users.noreply.github.com> Date: Tue, 28 Nov 2023 11:26:20 +0100 Subject: [PATCH 10/17] Fix "before" test naming --- packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts index 11e1d19ef055..09aed8f52672 100644 --- a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts +++ b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts @@ -13,7 +13,7 @@ describe("voluntaryExit using remote signer", function () { let externalSigner: ExternalSignerTests; - before("set up validator stores", async () => { + before("start external signer container", async () => { const password = "password"; externalSigner = await startExternalSigner({ keystoreStrings: await getKeystoresStr( From 3f3c57a4d61f93b964c0092aed18a209b09f9933 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 30 Nov 2023 12:18:47 +0100 Subject: [PATCH 11/17] Install same version of bls packages in test-utils --- packages/test-utils/package.json | 6 +-- yarn.lock | 93 ++------------------------------ 2 files changed, 7 insertions(+), 92 deletions(-) diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index f073fe003fdf..46ac502f73ad 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -61,14 +61,14 @@ "blockchain" ], "dependencies": { - "@chainsafe/bls": "^7.1.2", - "@chainsafe/bls-keystore": "^3.0.0", + "@chainsafe/bls": "7.1.1", + "@chainsafe/bls-keystore": "^2.0.0", "@lodestar/utils": "^1.12.0", "axios": "^1.3.4", "chai": "^4.3.7", "mocha": "^10.2.0", "sinon": "^15.0.3", - "testcontainers": "^10.3.2", + "testcontainers": "^10.2.1", "tmp": "^0.2.1" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index a7d4c5921d81..8c78c64bd89c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -497,14 +497,6 @@ ethereum-cryptography "^0.1.3" uuid "^3.3.3" -"@chainsafe/bls-keystore@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@chainsafe/bls-keystore/-/bls-keystore-3.0.0.tgz#e28c979f7664417e4917fa0d4d32fa2b9416e9c6" - integrity sha512-vlRIIXnn555wq2emhqnSR7btno17M0sCcfdQ+Dhgr7IH6n0CMoTGw9qcrpnNYwM+9OPm3matSYeZc9mNlXf7fQ== - dependencies: - ethereum-cryptography "^1.0.0" - uuid "8.3.2" - "@chainsafe/bls@7.1.1": version "7.1.1" resolved "https://registry.npmjs.org/@chainsafe/bls/-/bls-7.1.1.tgz" @@ -514,15 +506,6 @@ bls-eth-wasm "^0.4.8" randombytes "^2.1.0" -"@chainsafe/bls@^7.1.2": - version "7.1.2" - resolved "https://registry.yarnpkg.com/@chainsafe/bls/-/bls-7.1.2.tgz#9150b9ae7ce31bbd3dfca18f46108f81caf2786a" - integrity sha512-eVFxfUpKJmuuu04vEqmCtUuV9jHvVssc2Kz0pUGCSUGsKdr0UAu3jOpkviqLXgfpip+N0ZHxoLbCGqJhXB5TXQ== - dependencies: - "@chainsafe/bls-keygen" "^0.4.0" - bls-eth-wasm "^0.4.8" - randombytes "^2.1.0" - "@chainsafe/blst@^0.2.9": version "0.2.9" resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-0.2.9.tgz#b356b47759c3ce127677227fc24faa3ac6c72032" @@ -2948,14 +2931,6 @@ "@types/docker-modem" "*" "@types/node" "*" -"@types/dockerode@^3.3.21": - version "3.3.23" - resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.23.tgz#07b2084013d01e14d5d97856446f4d9c9f27c223" - integrity sha512-Lz5J+NFgZS4cEVhquwjIGH4oQwlVn2h7LXD3boitujBnzOE5o7s9H8hchEjoDK2SlRsJTogdKnQeiJgPPKLIEw== - dependencies: - "@types/docker-modem" "*" - "@types/node" "*" - "@types/eslint-scope@^3.7.3": version "3.7.4" resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.4.tgz#37fc1223f0786c39627068a12e94d6e6fc61de16" @@ -3947,19 +3922,6 @@ archiver@^5.3.1: tar-stream "^2.2.0" zip-stream "^4.1.0" -archiver@^5.3.2: - version "5.3.2" - resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.2.tgz#99991d5957e53bd0303a392979276ac4ddccf3b0" - integrity sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw== - dependencies: - archiver-utils "^2.1.0" - async "^3.2.4" - buffer-crc32 "^0.2.1" - readable-stream "^3.6.0" - readdir-glob "^1.1.2" - tar-stream "^2.2.0" - zip-stream "^4.1.0" - archy@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz" @@ -4150,11 +4112,6 @@ async@^3.2.3, async@~3.2.2: resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== -async@^3.2.4: - version "3.2.5" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" - integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -5869,9 +5826,9 @@ dir-glob@^3.0.1: path-type "^4.0.0" dns-over-http-resolver@^2.1.0, dns-over-http-resolver@^2.1.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/dns-over-http-resolver/-/dns-over-http-resolver-2.1.3.tgz#bb7f2e10cc18d960339a6e30e21b8c1d99be7b38" - integrity sha512-zjRYFhq+CsxPAouQWzOsxNMvEN+SHisjzhX8EMxd2Y0EG3thvn6wXQgMJLnTDImkhe4jhLbOQpXtL10nALBOSA== + version "2.1.2" + resolved "https://registry.yarnpkg.com/dns-over-http-resolver/-/dns-over-http-resolver-2.1.2.tgz#fb478af244dd4fed5e0f798a3e6426d92730378c" + integrity sha512-Bjbf6aZjr3HMnwGslZnoW3MJVqgbTsh39EZWpikx2yLl9xEjw4eZhlOHCFhkOu89zoWaS4rqe2Go53TXW4Byiw== dependencies: debug "^4.3.1" native-fetch "^4.0.2" @@ -6639,7 +6596,7 @@ ethereum-cryptography@^0.1.3: secp256k1 "^4.0.1" setimmediate "^1.0.5" -ethereum-cryptography@^1.0.0, ethereum-cryptography@^1.2.0: +ethereum-cryptography@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/ethereum-cryptography/-/ethereum-cryptography-1.2.0.tgz#5ccfa183e85fdaf9f9b299a79430c044268c9b3a" integrity sha512-6yFQC9b5ug6/17CQpCyE3k9eKBMdhyVjzUy1WkiuY/E4vj/SXDBbCw8QEIaXqf0Mf2SnY6RmpDcwlUmBSS0EJw== @@ -10384,13 +10341,6 @@ node-fetch@^2.6.12: dependencies: whatwg-url "^5.0.0" -node-fetch@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.1.0: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -11592,13 +11542,6 @@ properties-reader@^2.2.0: dependencies: mkdirp "^1.0.4" -properties-reader@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/properties-reader/-/properties-reader-2.3.0.tgz#f3ab84224c9535a7a36e011ae489a79a13b472b2" - integrity sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw== - dependencies: - mkdirp "^1.0.4" - protobufjs@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.0.0.tgz#8c678e1351fd926178fce5a4213913e8d990974f" @@ -11921,13 +11864,6 @@ readdir-glob@^1.0.0: dependencies: minimatch "^5.1.0" -readdir-glob@^1.1.2: - version "1.1.3" - resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.3.tgz#c3d831f51f5e7bfa62fa2ffbe4b508c640f09584" - integrity sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA== - dependencies: - minimatch "^5.1.0" - readdirp@~3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz" @@ -13272,27 +13208,6 @@ testcontainers@^10.2.1: tar-fs "^3.0.4" tmp "^0.2.1" -testcontainers@^10.3.2: - version "10.3.2" - resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-10.3.2.tgz#1477771585b328953ef6641f83b43368adac1242" - integrity sha512-IsV6NgS5reHcVF1nJLCDJv1hM9gAWUhLwh9b3ybgzvM3X7T2dcmuLFKt1RAR8qN8k+44tW2Drj7idxW6oeGvvg== - dependencies: - "@balena/dockerignore" "^1.0.2" - "@types/dockerode" "^3.3.21" - archiver "^5.3.2" - async-lock "^1.4.0" - byline "^5.0.0" - debug "^4.3.4" - docker-compose "^0.24.2" - dockerode "^3.3.5" - get-port "^5.1.1" - node-fetch "^2.7.0" - proper-lockfile "^4.1.2" - properties-reader "^2.3.0" - ssh-remote-port-forward "^1.0.4" - tar-fs "^3.0.4" - tmp "^0.2.1" - text-extensions@^1.0.0: version "1.9.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" From 79aa426b7d3b5e8bc7d76a9eb5e217aaf138ff26 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 30 Nov 2023 12:22:09 +0100 Subject: [PATCH 12/17] Remove abort controller from tests --- .../test/e2e/voluntaryExitRemoteSigner.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts index 09aed8f52672..1b5befee99df 100644 --- a/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts +++ b/packages/cli/test/e2e/voluntaryExitRemoteSigner.test.ts @@ -3,9 +3,14 @@ import {retry} from "@lodestar/utils"; import {ApiError, getClient} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {interopSecretKey, interopSecretKeys} from "@lodestar/state-transition"; -import {spawnCliCommand, execCliCommand, startExternalSigner, ExternalSignerTests} from "@lodestar/test-utils"; +import { + spawnCliCommand, + execCliCommand, + startExternalSigner, + ExternalSignerTests, + getKeystoresStr, +} from "@lodestar/test-utils"; import {getMochaContext} from "@lodestar/test-utils/mocha"; -import {getKeystoresStr} from "@lodestar/test-utils"; import {testFilesDir} from "../utils.js"; describe("voluntaryExit using remote signer", function () { @@ -58,9 +63,7 @@ describe("voluntaryExit using remote signer", function () { }); const baseUrl = `http://127.0.0.1:${restPort}`; - // To cleanup the event stream connection - const httpClientController = new AbortController(); - const client = getClient({baseUrl, getAbortSignal: () => httpClientController.signal}, {config}); + const client = getClient({baseUrl}, {config}); // Wait for beacon node API to be available + genesis await retry( @@ -105,8 +108,5 @@ describe("voluntaryExit using remote signer", function () { {retryDelay: 1000, retries: 20} ); } - - // Disconnect the event stream for the client - httpClientController.abort(); }); }); From c205a47defd85f052d6a05807e746f61ec435dc0 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 30 Nov 2023 12:23:37 +0100 Subject: [PATCH 13/17] Move web3signer supported fork seq const to test-utils --- packages/test-utils/package.json | 1 + packages/test-utils/src/externalSigner.ts | 4 ++++ packages/validator/test/e2e/web3signer.test.ts | 18 +++++++++--------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 46ac502f73ad..47d0570dc95c 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -63,6 +63,7 @@ "dependencies": { "@chainsafe/bls": "7.1.1", "@chainsafe/bls-keystore": "^2.0.0", + "@lodestar/params": "^1.12.0", "@lodestar/utils": "^1.12.0", "axios": "^1.3.4", "chai": "^4.3.7", diff --git a/packages/test-utils/src/externalSigner.ts b/packages/test-utils/src/externalSigner.ts index 57b41c6a40c9..3331ce9df460 100644 --- a/packages/test-utils/src/externalSigner.ts +++ b/packages/test-utils/src/externalSigner.ts @@ -2,9 +2,13 @@ import fs from "node:fs"; import path from "node:path"; import tmp from "tmp"; import {GenericContainer, Wait, StartedTestContainer} from "testcontainers"; +import {ForkSeq} from "@lodestar/params"; const web3signerVersion = "23.11.0"; +/** Till what version is the web3signer image updated for signature verification */ +export const externalSignerSupportedForkSeq = ForkSeq.capella; + export type ExternalSignerTests = { container: StartedTestContainer; url: string; diff --git a/packages/validator/test/e2e/web3signer.test.ts b/packages/validator/test/e2e/web3signer.test.ts index e8fe205ad6ab..f01150926588 100644 --- a/packages/validator/test/e2e/web3signer.test.ts +++ b/packages/validator/test/e2e/web3signer.test.ts @@ -4,19 +4,19 @@ import {config} from "@lodestar/config/default"; import {computeStartSlotAtEpoch, interopSecretKey, interopSecretKeys} from "@lodestar/state-transition"; import {createBeaconConfig} from "@lodestar/config"; import {genesisData} from "@lodestar/config/networks"; -import {getClient} from "@lodestar/api"; +import {getClient, routes} from "@lodestar/api"; import {ssz} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; -import {ExternalSignerTests, getKeystoresStr, startExternalSigner} from "@lodestar/test-utils"; +import { + ExternalSignerTests, + getKeystoresStr, + startExternalSigner, + externalSignerSupportedForkSeq, +} from "@lodestar/test-utils"; import {Interchange, ISlashingProtection, Signer, SignerType, ValidatorStore} from "../../src/index.js"; import {IndicesService} from "../../src/services/indices.js"; import {testLogger} from "../utils/logger.js"; -/** Till what version is the image updated for signature verification */ -const validTillSignatureForkSeq = ForkSeq.capella; - -/* eslint-disable no-console */ - describe("web3signer signature test", function () { this.timeout("60s"); @@ -34,7 +34,7 @@ describe("web3signer signature test", function () { let externalSigner: ExternalSignerTests; - const duty = { + const duty: routes.validator.AttesterDuty = { slot: altairSlot, committeeIndex: 0, committeeLength: 120, @@ -69,7 +69,7 @@ describe("web3signer signature test", function () { for (const fork of config.forksAscendingEpochOrder) { it(`signBlock ${fork.name}`, async function () { // Only test till the fork the signer version supports - if (ForkSeq[fork.name] > validTillSignatureForkSeq) { + if (ForkSeq[fork.name] > externalSignerSupportedForkSeq) { this.skip(); } From 8110b3175d39fa8ada1507012b8a7f7c785f642a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 30 Nov 2023 12:23:58 +0100 Subject: [PATCH 14/17] Remove testcontaines from validator package --- packages/validator/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/validator/package.json b/packages/validator/package.json index 8e659c94bd9e..c69499b89e7e 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -63,7 +63,6 @@ "devDependencies": { "@types/dockerode": "^3.3.19", "bigint-buffer": "^1.1.5", - "rimraf": "^4.4.1", - "testcontainers": "^10.2.1" + "rimraf": "^4.4.1" } } From 21caba1a31d414bfcde5a30b81270c060aef83e3 Mon Sep 17 00:00:00 2001 From: Luca Winter <70237279+eth2353@users.noreply.github.com> Date: Thu, 30 Nov 2023 13:07:09 +0100 Subject: [PATCH 15/17] Update example command Co-authored-by: Nico Flaig --- packages/cli/src/cmds/validator/voluntaryExit.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 0e8560ca5e61..770843c6445b 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -39,7 +39,7 @@ If no `pubkeys` are provided, it will exit all validators that have been importe }, { command: - "validator voluntary-exit --network goerli --externalSigner.url=http://signer:9000 --externalSigner.pubkeys 0xF00", + "validator voluntary-exit --network goerli --externalSigner.url http://signer:9000 --externalSigner.fetch --pubkeys 0xF00", description: "Perform a voluntary exit for the validator who has a public key 0xF00 and its secret key is on a remote signer", }, From 73612a140f02dc753a1e3be14cbd91f476ee2ffb Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 30 Nov 2023 14:04:46 +0100 Subject: [PATCH 16/17] Move dockerode types to test-utils --- packages/test-utils/package.json | 1 + packages/validator/package.json | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 47d0570dc95c..a55937bf5acb 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -73,6 +73,7 @@ "tmp": "^0.2.1" }, "devDependencies": { + "@types/dockerode": "^3.3.19", "@types/mocha": "^10.0.1", "@types/yargs": "^17.0.24", "yargs": "^17.7.1" diff --git a/packages/validator/package.json b/packages/validator/package.json index c69499b89e7e..09742f88a439 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -61,7 +61,6 @@ "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@types/dockerode": "^3.3.19", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From 165eefa1d4bc36e39ce750954c09d06b55867970 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 30 Nov 2023 14:06:23 +0100 Subject: [PATCH 17/17] Add test-utils to validator dev dependencies --- packages/validator/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/validator/package.json b/packages/validator/package.json index 09742f88a439..a853057637cd 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -61,6 +61,7 @@ "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { + "@lodestar/test-utils": "^1.12.0", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" }