Add validator-manager (sigp#3502)

Addresses sigp#2557 Adds the `lighthouse validator-manager` command, which provides: - `lighthouse validator-manager create` - Creates a `validators.json` file and a `deposits.json` (same format as https://github.com/ethereum/staking-deposit-cli) - `lighthouse validator-manager import` - Imports validators from a `validators.json` file to the VC via the HTTP API. - `lighthouse validator-manager move` - Moves validators from one VC to the other, utilizing only the VC API. In 98bcb94 I've reduced some VC `ERRO` and `CRIT` warnings to `WARN` or `DEBG` for the case where a pubkey is missing from the validator store. These were being triggered when we removed a validator but still had it in caches. It seems to me that `UnknownPubkey` will only happen in the case where we've removed a validator, so downgrading the logs is prudent. All the logs are `DEBG` apart from attestations and blocks which are `WARN`. I thought having *some* logging about this condition might help us down the track. In sigp@856cd7e I've made the VC delete the corresponding password file when it's deleting a keystore. This seemed like nice hygiene. Notably, it'll only delete that password file after it scans the validator definitions and finds that no other validator is also using that password file.
Woodpile37 · Jan 6, 2024 · 94839a3 · 94839a3
1 parent a0a3c4b
commit 94839a3
Show file tree

Hide file tree

Showing 67 changed files with 6,026 additions and 743 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -92,6 +92,8 @@ members = [
     "validator_client",
     "validator_client/slashing_protection",
 
+    "validator_manager",
+
     "watch",
 ]
 resolver = "2"

diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml
@@ -24,6 +24,8 @@ safe_arith = {path = "../consensus/safe_arith"}
 slot_clock = { path = "../common/slot_clock" }
 filesystem = { path = "../common/filesystem" }
 sensitive_url = { path = "../common/sensitive_url" }
+serde = { version = "1.0.116", features = ["derive"] }
+serde_json = "1.0.58"
 
 [dev-dependencies]
 tempfile = "3.1.0"
diff --git a/account_manager/src/common.rs b/account_manager/src/common.rs
@@ -1,55 +1,7 @@
-use account_utils::PlainText;
-use account_utils::{read_input_from_user, strip_off_newlines};
-use eth2_wallet::bip39::{Language, Mnemonic};
-use std::fs;
-use std::path::PathBuf;
-use std::str::from_utf8;
-use std::thread::sleep;
-use std::time::Duration;
+use account_utils::read_input_from_user;
 
-pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-pub fn read_mnemonic_from_cli(
-    mnemonic_path: Option<PathBuf>,
-    stdin_inputs: bool,
-) -> Result<Mnemonic, String> {
-    let mnemonic = match mnemonic_path {
-        Some(path) => fs::read(&path)
-            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
-            .and_then(|bytes| {
-                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
-                let phrase = from_utf8(bytes_no_newlines.as_ref())
-                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
-                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
-                    format!(
-                        "Unable to derive mnemonic from string {:?}: {:?}",
-                        phrase, e
-                    )
-                })
-            })?,
-        None => loop {
-            eprintln!();
-            eprintln!("{}", MNEMONIC_PROMPT);
-
-            let mnemonic = read_input_from_user(stdin_inputs)?;
-
-            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
-                Ok(mnemonic_m) => {
-                    eprintln!("Valid mnemonic provided.");
-                    eprintln!();
-                    sleep(Duration::from_secs(1));
-                    break mnemonic_m;
-                }
-                Err(_) => {
-                    eprintln!("Invalid mnemonic");
-                }
-            }
-        },
-    };
-    Ok(mnemonic)
-}
-
 /// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
 /// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
 pub fn read_wallet_name_from_cli(

diff --git a/account_manager/src/validator/import.rs b/account_manager/src/validator/import.rs
@@ -4,8 +4,8 @@ use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
     validator_definitions::{
-        recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
-        CONFIG_FILENAME,
+        recursively_find_voting_keystores, PasswordStorage, ValidatorDefinition,
+        ValidatorDefinitions, CONFIG_FILENAME,
     },
     ZeroizeString,
 };
@@ -277,7 +277,9 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
         let suggested_fee_recipient = None;
         let validator_def = ValidatorDefinition::new_keystore_with_password(
             &dest_keystore,
-            password_opt,
+            password_opt
+                .map(PasswordStorage::ValidatorDefinitions)
+                .unwrap_or(PasswordStorage::None),
             graffiti,
             suggested_fee_recipient,
             None,

diff --git a/account_manager/src/validator/recover.rs b/account_manager/src/validator/recover.rs
@@ -1,10 +1,9 @@
 use super::create::STORE_WITHDRAW_FLAG;
-use crate::common::read_mnemonic_from_cli;
 use crate::validator::create::COUNT_FLAG;
 use crate::wallet::create::STDIN_INPUTS_FLAG;
 use crate::SECRETS_DIR_FLAG;
 use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
-use account_utils::random_password;
+use account_utils::{random_password, read_mnemonic_from_cli};
 use clap::{App, Arg, ArgMatches};
 use directory::ensure_dir_exists;
 use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};

diff --git a/account_manager/src/wallet/recover.rs b/account_manager/src/wallet/recover.rs
@@ -1,6 +1,6 @@
-use crate::common::read_mnemonic_from_cli;
 use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
 use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use account_utils::read_mnemonic_from_cli;
 use clap::{App, Arg, ArgMatches};
 use std::path::PathBuf;
 

diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md
@@ -12,6 +12,9 @@
 * [Run a Node](./run_a_node.md)
 * [Become a Validator](./mainnet-validator.md)
 * [Validator Management](./validator-management.md)
+    * [The `validator-manager` Command](./validator-manager.md)
+        * [Creating validators](./validator-manager-create.md)
+        * [Moving validators](./validator-manager-move.md)
     * [Slashing Protection](./slashing-protection.md)
     * [Voluntary Exits](./voluntary-exit.md)
     * [Partial Withdrawals](./partial-withdrawal.md)
@@ -41,7 +44,7 @@
     * [Remote Signing with Web3Signer](./validator-web3signer.md)
     * [Database Configuration](./advanced_database.md)
     * [Database Migrations](./database-migrations.md)
-    * [Key Management](./key-management.md)
+    * [Key Management (Deprecated)](./key-management.md)
     * [Key Recovery](./key-recovery.md)
     * [Advanced Networking](./advanced_networking.md)
     * [Running a Slasher](./slasher.md)

diff --git a/book/src/key-management.md b/book/src/key-management.md
@@ -1,9 +1,30 @@
-# Key Management
+# Key Management (Deprecated)
 
 [launchpad]: https://launchpad.ethereum.org/
 
->
-> **Note: While Lighthouse is able to generate the validator keys and the deposit data file to submit to the deposit contract, we strongly recommend using the [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) to create validators keys and the deposit data file. This is because the [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) has the option to assign a withdrawal address during the key generation process, while Lighthouse wallet will always generate keys with withdrawal credentials of type 0x00. This means that users who created keys using Lighthouse will have to update their withdrawal credentials in the future to enable withdrawals. In addition, Lighthouse generates the deposit data file in the form of `*.rlp`, which cannot be uploaded to the [Staking launchpad][launchpad] that accepts only `*.json` file. This means that users have to directly interact with the deposit contract to be able to submit the deposit if they were to generate the files using Lighthouse.**
+**⚠️ The information on this page refers to tooling and process that have been deprecated. Please read the "Deprecation Notice". ⚠️**
+
+## Deprecation Notice
+
+This page recommends the use of the `lighthouse account-manager` tool to create
+validators. This tool will always generate keys with the withdrawal credentials
+of type `0x00`. This means the users who created keys using `lighthouse
+account-manager` will have to update their withdrawal credentials in a
+separate step to receive staking rewards.
+
+In addition, Lighthouse generates the deposit data file in the form of `*.rlp`,
+which cannot be uploaded to the [Staking launchpad][launchpad] that accepts only
+`*.json` file. This means that users have to directly interact with the deposit
+contract to be able to submit the deposit if they were to generate the files
+using Lighthouse.
+
+Rather than continuing to read this page, we recommend users visit either:
+
+- The [Staking Launchpad][launchpad] for detailed, beginner-friendly instructions.
+- The [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) for a CLI tool used by the [Staking Launchpad][launchpad].
+- The [validator-manager documentation](./validator-manager.md) for a Lighthouse-specific tool for streamlined validator management tools.
+
+## The `lighthouse account-manager`
 
 Lighthouse uses a _hierarchical_ key management system for producing validator
 keys. It is hierarchical because each validator key can be _derived_ from a

diff --git a/book/src/validator-management.md b/book/src/validator-management.md
@@ -13,6 +13,10 @@ standard directories and do not start their `lighthouse vc` with the
 this document. However, users with more complex needs may find this document
 useful.
 
+The [lighthouse validator-manager](./validator-manager.md) command can be used
+to create and import validators to a Lighthouse VC. It can also be used to move
+validators between two Lighthouse VCs.
+
 ## Introducing the `validator_definitions.yml` file
 
 The `validator_definitions.yml` file is located in the `validator-dir`, which

diff --git a/book/src/validator-manager-create.md b/book/src/validator-manager-create.md
@@ -0,0 +1,206 @@
+# Creating and Importing Validators
+
+[Ethereum Staking Launchpad]: https://launchpad.ethereum.org/en/
+
+The `lighthouse validator-manager create` command derives validators from a
+mnemonic and produces two files:
+
+- `validators.json`: the keystores and passwords for the newly generated
+    validators, in JSON format.
+- `deposits.json`: a JSON file of the same format as
+    [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) which can
+    be used for deposit submission via the [Ethereum Staking
+    Launchpad][].
+
+The `lighthouse validator-manager import` command accepts a `validators.json`
+file (from the `create` command) and submits those validators to a running
+Lighthouse Validator Client via the HTTP API.
+
+These two commands enable a workflow of:
+
+1. Creating the validators via the `create` command.
+1. Importing the validators via the `import` command.
+1. Depositing validators via the [Ethereum Staking
+    Launchpad][].
+
+The separation of the `create` and `import` commands allows for running the
+`create` command on an air-gapped host whilst performing the `import` command on
+an internet-connected host.
+
+The `create` and `import` commands are recommended for advanced users who are
+familiar with command line tools and the practicalities of managing sensitive
+cryptographic material. **We recommend that novice users follow the workflow on
+[Ethereum Staking Launchpad][] rather than using the `create` and `import`
+commands.**
+
+## Simple Example
+
+Create validators from a mnemonic with:
+
+```bash
+lighthouse \
+    validator-manager \
+    create \
+    --network mainnet \
+    --first-index 0 \
+    --count 2 \
+    --eth1-withdrawal-address <ADDRESS> \
+    --suggested-fee-recipient <ADDRESS> \
+    --output-path ./
+```
+> If the flag `--first-index` is not provided, it will default to using index 0.
+> The `--suggested-fee-recipient` flag may be omitted to use whatever default
+> value the VC uses. It does not necessarily need to be identical to
+> `--eth1-withdrawal-address`.
+> The command will create the `deposits.json` and `validators.json` in the present working directory. If you would like these files to be created in a different directory, change the value of `output-path`, for example `--output-path /desired/directory`. The directory will be created if the path does not exist.
+
+Then, import the validators to a running VC with:
+
+```bash
+lighthouse \
+    validator-manager \
+    import \
+    --validators-file validators.json \
+    --vc-token <API-TOKEN-PATH>
+```
+> This is assuming that `validators.json` is in the present working directory. If it is not, insert the directory of the file.
+> Be sure to remove `./validators.json` after the import is successful since it
+> contains unencrypted validator keystores.
+
+## Detailed Guide
+
+This guide will create two validators and import them to a VC. For simplicity,
+the same host will be used to generate the keys and run the VC. In reality,
+users may want to perform the `create` command on an air-gapped machine and then
+move the `validators.json` and `deposits.json` files to an Internet-connected
+host. This would help protect the mnemonic from being exposed to the Internet.
+
+### 1. Create the Validators
+
+Run the `create` command, substituting `<ADDRESS>` for an execution address that
+you control. This is where all the staked ETH and rewards will ultimately
+reside, so it's very important that this address is secure, accessible and
+backed-up. The `create` command:
+
+```bash
+lighthouse \
+    validator-manager \
+    create \
+    --first-index 0 \
+    --count 2 \
+    --eth1-withdrawal-address <ADDRESS> \
+    --output-path ./
+```
+
+If successful, the command output will appear like below:
+
+```bash
+Running validator manager for mainnet network
+
+Enter the mnemonic phrase:
+<REDACTED>
+Valid mnemonic provided.
+
+Starting derivation of 2 keystores. Each keystore may take several seconds.
+Completed 1/2: 0x8885c29b8f88ee9b9a37b480fd4384fed74bda33d85bc8171a904847e65688b6c9bb4362d6597fd30109fb2def6c3ae4
+Completed 2/2: 0xa262dae3dcd2b2e280af534effa16bedb27c06f2959e114d53bd2a248ca324a018dc73179899a066149471a94a1bc92f
+Keystore generation complete
+Writing "./validators.json"
+Writing "./deposits.json"
+```
+
+This command will create validators at indices `0, 1`. The exact indices created
+can be influenced with the `--first-index` and `--count` flags. Use these flags
+with caution to prevent creating the same validator twice, this may result in a
+slashing!
+
+The command will create two files:
+
+- `./deposits.json`: this file does *not* contain sensitive information and may be uploaded to the [Ethereum Staking Launchpad].
+- `./validators.json`: this file contains **sensitive unencrypted validator keys, do not share it with anyone or upload it to any website**.
+
+### 2. Import the validators
+
+The VC which will receive the validators needs to have the following flags at a minimum:
+
+- `--http`
+- `--http-port 5062`
+- `--enable-doppelganger-protection`
+
+Therefore, the VC command might look like:
+
+```bash
+lighthouse \
+    vc \
+    --http \
+    --http-port 5062 \
+    --enable-doppelganger-protection
+```
+
+In order to import the validators, the location of the VC `api-token.txt` file
+must be known. The location of the file varies, but it is located in the
+"validator directory" of your data directory. For example:
+`~/.lighthouse/mainnet/validators/api-token.txt`. We will use `<API-TOKEN-PATH>`
+to subsitute this value. If you are unsure of the `api-token.txt` path, you can run `curl http://localhost:5062/lighthouse/auth` which will show the path.
+
+
+Once the VC is running, use the `import` command to import the validators to the VC:
+
+```bash
+lighthouse \
+    validator-manager \
+    import \
+    --validators-file validators.json \
+    --vc-token <API-TOKEN-PATH>
+```
+
+If successful, the command output will appear like below:
+
+```bash
+Running validator manager for mainnet network
+Validator client is reachable at http://localhost:5062/ and reports 0 validators
+Starting to submit 2 validators to VC, each validator may take several seconds
+Uploaded keystore 1 of 2 to the VC
+Uploaded keystore 2 of 2 to the VC
+```
+
+The user should now *securely* delete the `validators.json` file (e.g., `shred -u validators.json`).
+The `validators.json` contains the unencrypted validator keys and must not be
+shared with anyone.
+At the same time, `lighthouse vc` will log:
+```bash
+INFO Importing keystores via standard HTTP API, count: 1
+WARN No slashing protection data provided with keystores
+INFO Enabled validator                       voting_pubkey: 0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f, signing_method: local_keystore
+INFO Modified key_cache saved successfully
+```
+The WARN message means that the `validators.json` file does not contain the slashing protection data. This is normal if you are starting a new validator. The flag `--enable-doppelganger-protection` will also protect users from potential slashing risk. 
+The validators will now go through 2-3 epochs of [doppelganger
+protection](./validator-doppelganger.md) and will automatically start performing
+their duties when they are deposited and activated. 
+
+If the host VC contains the same public key as the `validators.json` file, an error will be shown and the `import` process will stop:
+
+```bash
+Duplicate validator   0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f already exists on the destination validator client. This may indicate that some validators are running in two places at once, which can lead to slashing. If you are certain that there is no risk, add the --ignore-duplicates flag.
+Err(DuplicateValidator(0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f))
+```
+
+If you are certain that it is safe, you can add the flag `--ignore-duplicates` in the `import` command. The command becomes:
+
+```bash
+lighthouse \
+    validator-manager \
+    import \
+    --validators-file validators.json \
+    --vc-token <API-TOKEN-PATH> \
+    --ignore-duplicates
+```
+and the output will be as follows:
+
+```bash
+Duplicate validators are ignored, ignoring 0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f which exists on the destination validator client
+Re-uploaded keystore 1 of 6 to the VC
+```
+
+The guide is complete.