From 1e9f50f3fd02bd1fbb15688c20c51c5506d127b7 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 23 Aug 2020 18:04:47 +0200 Subject: [PATCH 01/24] Adds language detection and verification of mnemonics --- .../key_handling/key_derivation/mnemonic.py | 68 +++++++++++++++++-- .../test_key_derivation/test_mnemonic.py | 8 +++ 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index 7c8537af..d2ad84a0 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -26,12 +26,27 @@ def _resource_path(relative_path: str) -> str: def _get_word_list(language: str, path: str) -> Sequence[str]: path = _resource_path(path) - return open(os.path.join(path, '%s.txt' % language), encoding='utf-8').readlines() + dirty_list = open(os.path.join(path, '%s.txt' % language), encoding='utf-8').readlines() + return [word.replace('\n', '') for word in dirty_list] -def _get_word(*, word_list: Sequence[str], index: int) -> str: +def _index_to_word(word_list: Sequence[str], index: int) -> str: + """ + Given the index of a word in the word list, return the corresponding word. + """ assert index < 2048 - return word_list[index][:-1] + return word_list[index] + + +def _word_to_index(word_list: Sequence[str], word: str) -> int: + try: + return word_list.index(word) + except ValueError: + raise ValueError('Word %s not in BIP39 word-list' % word) + + +def _uint11_array_to_uint(unit_array: Sequence[int]) -> int: + return sum([x << i * 11 for i, x in enumerate(reversed(unit_array))]) def get_seed(*, mnemonic: str, password: str) -> bytes: @@ -55,6 +70,49 @@ def get_languages(path: str) -> Tuple[str, ...]: return languages +def determine_mnemonic_language(mnemonic: str, words_path:str) -> Sequence[str]: + """ + Given a `mnemonic` determine what language it is written in. + """ + languages = get_languages(words_path) + word_language_map = {word: lang for lang in languages for word in _get_word_list(lang, words_path)} + try: + mnemonic_list = mnemonic.split(' ') + word_languages = [word_language_map[word] for word in mnemonic_list] + return set(word_languages) + except KeyError: + raise ValueError('Word not found in mnemonic word lists for any language.') + + +def _get_checksum(entropy: bytes) -> int: + """ + Determine the index of the checksum word given the entropy + """ + entropy_length = len(entropy) * 8 + assert entropy_length in range(128, 257, 32) + checksum_length = (entropy_length // 32) + return int.from_bytes(SHA256(entropy), 'big') >> 256 - checksum_length + + +def verify_mnemonic(mnemonic: str, words_path: str) -> bool: + languages = determine_mnemonic_language(mnemonic, words_path) + for language in languages: + try: + word_list = _get_word_list(language, words_path) + mnemonic_list = mnemonic.split(' ') + word_indices = [_word_to_index(word_list, word) for word in mnemonic_list] + mnemonic_int = _uint11_array_to_uint(word_indices) + checksum_length = len(mnemonic_list)//3 + checksum = mnemonic_int & 2**checksum_length - 1 + entropy = (mnemonic_int - checksum) >> checksum_length + entropy_bits = entropy.to_bytes(checksum_length * 4, 'big') + return _get_checksum(entropy_bits) == checksum + except ValueError: + pass + return False + + + def get_mnemonic(*, language: str, words_path: str, entropy: Optional[bytes]=None) -> str: """ Return a mnemonic string in a given `language` based on `entropy`. @@ -64,7 +122,7 @@ def get_mnemonic(*, language: str, words_path: str, entropy: Optional[bytes]=Non entropy_length = len(entropy) * 8 assert entropy_length in range(128, 257, 32) checksum_length = (entropy_length // 32) - checksum = int.from_bytes(SHA256(entropy), 'big') >> 256 - checksum_length + checksum = _get_checksum(entropy) entropy_bits = int.from_bytes(entropy, 'big') << checksum_length entropy_bits += checksum entropy_length += checksum_length @@ -72,6 +130,6 @@ def get_mnemonic(*, language: str, words_path: str, entropy: Optional[bytes]=Non word_list = _get_word_list(language, words_path) for i in range(entropy_length // 11 - 1, -1, -1): index = (entropy_bits >> i * 11) & 2**11 - 1 - word = _get_word(word_list=word_list, index=index) + word = _index_to_word(word_list, index) mnemonic.append(word) return ' '.join(mnemonic) diff --git a/tests/test_key_handling/test_key_derivation/test_mnemonic.py b/tests/test_key_handling/test_key_derivation/test_mnemonic.py index d374675f..f2d58e5d 100644 --- a/tests/test_key_handling/test_key_derivation/test_mnemonic.py +++ b/tests/test_key_handling/test_key_derivation/test_mnemonic.py @@ -8,6 +8,7 @@ from eth2deposit.key_handling.key_derivation.mnemonic import ( get_seed, get_mnemonic, + verify_mnemonic, ) @@ -30,3 +31,10 @@ def test_bip39(language: str, test: Sequence[str]) -> None: assert get_mnemonic(language=language, words_path=WORD_LISTS_PATH, entropy=test_entropy) == test_mnemonic assert get_seed(mnemonic=test_mnemonic, password='TREZOR') == test_seed + +@pytest.mark.parametrize( + 'test_mnemonic,is_valid', + [(test_mnemonic[1], True) for _, language_test_vectors in test_vectors.items() for test_mnemonic in language_test_vectors] +) +def test_verify_mnemonic(test_mnemonic: str, is_valid: bool) -> None: + assert verify_mnemonic(test_mnemonic, WORD_LISTS_PATH) == is_valid From 0e172661a8d003f732d0865cb9b1d1791cf98f39 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Aug 2020 13:55:06 +0200 Subject: [PATCH 02/24] Creates new_mnemonic cli options --- eth2deposit/cli/__init__.py | 0 eth2deposit/cli/existing_mnemonic.py | 34 ++++++++ eth2deposit/cli/generate_keys.py | 79 +++++++++++++++++++ eth2deposit/cli/new_mnemonic.py | 46 +++++++++++ eth2deposit/deposit.py | 13 +-- .../key_handling/key_derivation/mnemonic.py | 6 +- 6 files changed, 166 insertions(+), 12 deletions(-) create mode 100644 eth2deposit/cli/__init__.py create mode 100644 eth2deposit/cli/existing_mnemonic.py create mode 100644 eth2deposit/cli/generate_keys.py create mode 100644 eth2deposit/cli/new_mnemonic.py diff --git a/eth2deposit/cli/__init__.py b/eth2deposit/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py new file mode 100644 index 00000000..34e00304 --- /dev/null +++ b/eth2deposit/cli/existing_mnemonic.py @@ -0,0 +1,34 @@ +import click +from typing import ( + Tuple, +) + +from key_handling.key_derivation.mnemonic import ( + verify_mnemonic, +) +from utils.constants import ( + WORD_LISTS_PATH, +) + + +def validate_mnemonic(mnemonic: str) -> str: + if verify_mnemonic(mnemonic, WORD_LISTS_PATH): + return mnemonic + else: + raise click.BadParameter('That is not a valid mnemonic in any language.') + + +@click.command() +@click.option( + '--mnemonic', + prompt='Please enter your mnemonic separated by spaces (" ").', + required=True, + type=str, +) +@click.option( + '--mnemonic-password', + type=str, + defualt='', +) +def existing_mnemonic(mnemonic: str, mnemonic_password: str) -> Tuple[str, str]: + return (mnemonic, mnemonic_password) diff --git a/eth2deposit/cli/generate_keys.py b/eth2deposit/cli/generate_keys.py new file mode 100644 index 00000000..b2b2a768 --- /dev/null +++ b/eth2deposit/cli/generate_keys.py @@ -0,0 +1,79 @@ +import os +import sys +import click + +from eth2deposit.credentials import ( + CredentialList, +) +from eth2deposit.key_handling.key_derivation.mnemonic import ( + get_languages, + get_mnemonic, +) +from eth2deposit.utils.validation import verify_deposit_data_json +from eth2deposit.utils.constants import ( + WORD_LISTS_PATH, + MAX_DEPOSIT_AMOUNT, + DEFAULT_VALIDATOR_KEYS_FOLDER_NAME, +) +from eth2deposit.utils.ascii_art import RHINO_0 +from eth2deposit.settings import ( + ALL_CHAINS, + MAINNET, + get_setting, +) + + +def check_python_version() -> None: + ''' + Checks that the python version running is sufficient and exits if not. + ''' + if sys.version_info < (3, 7): + click.pause('Your python version is insufficient, please install version 3.7 or greater.') + sys.exit() + + +@click.command() +@click.option( + '--num_validators', + prompt='Please choose how many validators you wish to run', + required=True, + type=click.IntRange(0, 2**32)), +) +@click.option( + '--validator_start_index', + type=click.IntRange(0, 2**32)), +) +@click.option( + '--chain', + prompt='Please choose the (mainnet or testnet) network/chain name', + type=click.Choice(ALL_CHAINS.keys(), case_sensitive=False), + default=MAINNET, +) +@click.password_option('--keystore_password', prompt='Type the password that secures your validator keystore(s)') +def main(mnemonic: str, mnemonic_password: str, num_validators: int, chain: str, keystore_password: str) -> None: + check_python_version() + mnemonic = generate_mnemonic(mnemonic_language, WORD_LISTS_PATH) + amounts = [MAX_DEPOSIT_AMOUNT] * num_validators + folder = os.path.join(folder, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) + setting = get_setting(chain) + if not os.path.exists(folder): + os.mkdir(folder) + click.clear() + click.echo(RHINO_0) + click.echo('Creating your keys.') + credentials = CredentialList.from_mnemonic( + mnemonic=mnemonic, + num_keys=num_validators, + amounts=amounts, + fork_version=setting.GENESIS_FORK_VERSION, + ) + click.echo('Saving your keystore(s).') + keystore_filefolders = credentials.export_keystores(password=password, folder=folder) + click.echo('Creating your deposit(s).') + deposits_file = credentials.export_deposit_data_json(folder=folder) + click.echo('Verifying your keystore(s).') + assert credentials.verify_keystores(keystore_filefolders=keystore_filefolders, password=password) + click.echo('Verifying your deposit(s).') + assert verify_deposit_data_json(deposits_file) + click.echo('\nSuccess!\nYour keys can be found at: %s' % folder) + click.pause('\n\nPress any key.') \ No newline at end of file diff --git a/eth2deposit/cli/new_mnemonic.py b/eth2deposit/cli/new_mnemonic.py new file mode 100644 index 00000000..f04d2041 --- /dev/null +++ b/eth2deposit/cli/new_mnemonic.py @@ -0,0 +1,46 @@ +import sys +import click + +from eth2deposit.credentials import ( + CredentialList, +) +from eth2deposit.key_handling.key_derivation.mnemonic import ( + get_languages, + get_mnemonic, +) +from eth2deposit.utils.validation import verify_deposit_data_json +from eth2deposit.utils.constants import ( + WORD_LISTS_PATH, + MAX_DEPOSIT_AMOUNT, + DEFAULT_VALIDATOR_KEYS_FOLDER_NAME, +) +from eth2deposit.utils.ascii_art import RHINO_0 +from eth2deposit.settings import ( + ALL_CHAINS, + MAINNET, + get_setting, +) + +languages = get_languages(WORD_LISTS_PATH) + +@click.command() +@click.option( + '--mnemonic_language', + prompt='Please choose your mnemonic language', + type=click.Choice(languages, case_sensitive=False), + default='english', +) +def generate_mnemonic(mnemonic_language: str) -> str: + mnemonic = get_mnemonic(language=mnemonic_language, words_path=WORD_LISTS_PATH) + test_mnemonic = '' + while mnemonic != test_mnemonic: + click.clear() + click.echo('This is your seed phrase. Write it down and store it safely, it is the ONLY way to retrieve your deposit.') # noqa: E501 + click.echo('\n\n%s\n\n' % mnemonic) + click.pause('Press any key when you have written down your mnemonic.') + + click.clear() + test_mnemonic = click.prompt('Please type your mnemonic (separated by spaces) to confirm you have written it down\n\n') # noqa: E501 + test_mnemonic = test_mnemonic.lower() + click.clear() + return mnemonic diff --git a/eth2deposit/deposit.py b/eth2deposit/deposit.py index 4d4e5995..8d6090d0 100644 --- a/eth2deposit/deposit.py +++ b/eth2deposit/deposit.py @@ -55,18 +55,11 @@ def check_python_version() -> None: '--num_validators', prompt='Please choose how many validators you wish to run', required=True, - type=int, + type=click.IntRange(0, 2**32)), ) @click.option( - '--mnemonic_language', - prompt='Please choose your mnemonic language', - type=click.Choice(languages, case_sensitive=False), - default='english', -) -@click.option( - '--folder', - type=click.Path(exists=True, file_okay=False, dir_okay=True), - default=os.getcwd() + '--validator_start_index', + type=click.IntRange(0, 2**32)), ) @click.option( '--chain', diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index d2ad84a0..48d3088c 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -72,14 +72,15 @@ def get_languages(path: str) -> Tuple[str, ...]: def determine_mnemonic_language(mnemonic: str, words_path:str) -> Sequence[str]: """ - Given a `mnemonic` determine what language it is written in. + Given a `mnemonic` determine what language[s] it is written in. + There are collisions between word-lists, so multiple candidate languages are returned. """ languages = get_languages(words_path) word_language_map = {word: lang for lang in languages for word in _get_word_list(lang, words_path)} try: mnemonic_list = mnemonic.split(' ') word_languages = [word_language_map[word] for word in mnemonic_list] - return set(word_languages) + return list(set(word_languages)) except KeyError: raise ValueError('Word not found in mnemonic word lists for any language.') @@ -95,6 +96,7 @@ def _get_checksum(entropy: bytes) -> int: def verify_mnemonic(mnemonic: str, words_path: str) -> bool: + "Given a mnemonic, verify it against its own checksum." languages = determine_mnemonic_language(mnemonic, words_path) for language in languages: try: From 0fd3c0b1891a2c8021d9aa99997cdf2ea95e5f0d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 28 Sep 2020 16:49:37 +0200 Subject: [PATCH 03/24] Break new and existing mnemonics into separate files with generate_keys subcommand --- eth2deposit/cli/existing_mnemonic.py | 19 ++-- eth2deposit/cli/generate_keys.py | 79 ++++++++------- eth2deposit/cli/new_mnemonic.py | 31 +++--- eth2deposit/credentials.py | 14 ++- eth2deposit/deposit.py | 96 +++---------------- .../key_handling/key_derivation/mnemonic.py | 7 +- .../test_key_derivation/test_mnemonic.py | 5 +- 7 files changed, 100 insertions(+), 151 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 34e00304..6a66746a 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -1,14 +1,18 @@ import click from typing import ( - Tuple, + Any, ) -from key_handling.key_derivation.mnemonic import ( +from eth2deposit.key_handling.key_derivation.mnemonic import ( verify_mnemonic, ) -from utils.constants import ( +from eth2deposit.utils.constants import ( WORD_LISTS_PATH, ) +from .generate_keys import ( + generate_keys, + generate_keys_arguments_wrapper, +) def validate_mnemonic(mnemonic: str) -> str: @@ -19,6 +23,8 @@ def validate_mnemonic(mnemonic: str) -> str: @click.command() +@click.pass_context +@generate_keys_arguments_wrapper @click.option( '--mnemonic', prompt='Please enter your mnemonic separated by spaces (" ").', @@ -28,7 +34,8 @@ def validate_mnemonic(mnemonic: str) -> str: @click.option( '--mnemonic-password', type=str, - defualt='', + default='', ) -def existing_mnemonic(mnemonic: str, mnemonic_password: str) -> Tuple[str, str]: - return (mnemonic, mnemonic_password) +def existing_mnemonic(ctx: click.Context, mnemonic: str, mnemonic_password: str, **kwargs: Any) -> None: + ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': mnemonic_password} + ctx.forward(generate_keys) diff --git a/eth2deposit/cli/generate_keys.py b/eth2deposit/cli/generate_keys.py index b2b2a768..4aafeae0 100644 --- a/eth2deposit/cli/generate_keys.py +++ b/eth2deposit/cli/generate_keys.py @@ -1,17 +1,15 @@ import os -import sys import click +from typing import ( + Any, + Callable, +) from eth2deposit.credentials import ( CredentialList, ) -from eth2deposit.key_handling.key_derivation.mnemonic import ( - get_languages, - get_mnemonic, -) from eth2deposit.utils.validation import verify_deposit_data_json from eth2deposit.utils.constants import ( - WORD_LISTS_PATH, MAX_DEPOSIT_AMOUNT, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME, ) @@ -23,36 +21,47 @@ ) -def check_python_version() -> None: +def generate_keys_arguments_decorator(function: Callable[..., Any]) -> Callable[..., Any]: ''' - Checks that the python version running is sufficient and exits if not. + This is a decorator that, when applied to a parent-command, implements the + to obtain the necessary arguments for the generate_keys() subcommand. ''' - if sys.version_info < (3, 7): - click.pause('Your python version is insufficient, please install version 3.7 or greater.') - sys.exit() + decorators = [ + click.option( + '--num_validators', + prompt='Please choose how many validators you wish to run', + required=True, + type=click.IntRange(0, 2**32), + ), + click.option( + '--validator_start_index', + type=click.IntRange(0, 2**32), + default=0, + ), + click.option( + '--folder', + type=click.Path(exists=True, file_okay=False, dir_okay=True), + default=os.getcwd() + ), + click.option( + '--chain', + prompt='Please choose the (mainnet or testnet) network/chain name', + type=click.Choice(ALL_CHAINS.keys(), case_sensitive=False), + default=MAINNET, + ), + click.password_option('--keystore_password', prompt='Type the password that secures your validator keystore(s)') + ] + for decorator in reversed(decorators): + function = decorator(function) + return function @click.command() -@click.option( - '--num_validators', - prompt='Please choose how many validators you wish to run', - required=True, - type=click.IntRange(0, 2**32)), -) -@click.option( - '--validator_start_index', - type=click.IntRange(0, 2**32)), -) -@click.option( - '--chain', - prompt='Please choose the (mainnet or testnet) network/chain name', - type=click.Choice(ALL_CHAINS.keys(), case_sensitive=False), - default=MAINNET, -) -@click.password_option('--keystore_password', prompt='Type the password that secures your validator keystore(s)') -def main(mnemonic: str, mnemonic_password: str, num_validators: int, chain: str, keystore_password: str) -> None: - check_python_version() - mnemonic = generate_mnemonic(mnemonic_language, WORD_LISTS_PATH) +@click.pass_context +def generate_keys(ctx: click.Context, validator_start_index: int, + num_validators: int, folder: str, chain: str, keystore_password: str, **kwargs: Any) -> None: + mnemonic = ctx.obj['mnemonic'] + mnemonic_password = ctx.obj['mnemonic_password'] amounts = [MAX_DEPOSIT_AMOUNT] * num_validators folder = os.path.join(folder, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) setting = get_setting(chain) @@ -63,17 +72,19 @@ def main(mnemonic: str, mnemonic_password: str, num_validators: int, chain: str, click.echo('Creating your keys.') credentials = CredentialList.from_mnemonic( mnemonic=mnemonic, + mnemonic_password=mnemonic_password, num_keys=num_validators, amounts=amounts, fork_version=setting.GENESIS_FORK_VERSION, + start_index=validator_start_index, ) click.echo('Saving your keystore(s).') - keystore_filefolders = credentials.export_keystores(password=password, folder=folder) + keystore_filefolders = credentials.export_keystores(password=keystore_password, folder=folder) click.echo('Creating your deposit(s).') deposits_file = credentials.export_deposit_data_json(folder=folder) click.echo('Verifying your keystore(s).') - assert credentials.verify_keystores(keystore_filefolders=keystore_filefolders, password=password) + assert credentials.verify_keystores(keystore_filefolders=keystore_filefolders, password=keystore_password) click.echo('Verifying your deposit(s).') assert verify_deposit_data_json(deposits_file) click.echo('\nSuccess!\nYour keys can be found at: %s' % folder) - click.pause('\n\nPress any key.') \ No newline at end of file + click.pause('\n\nPress any key.') diff --git a/eth2deposit/cli/new_mnemonic.py b/eth2deposit/cli/new_mnemonic.py index f04d2041..c6356869 100644 --- a/eth2deposit/cli/new_mnemonic.py +++ b/eth2deposit/cli/new_mnemonic.py @@ -1,36 +1,32 @@ -import sys import click - -from eth2deposit.credentials import ( - CredentialList, +from typing import ( + Any, ) + from eth2deposit.key_handling.key_derivation.mnemonic import ( get_languages, get_mnemonic, ) -from eth2deposit.utils.validation import verify_deposit_data_json -from eth2deposit.utils.constants import ( - WORD_LISTS_PATH, - MAX_DEPOSIT_AMOUNT, - DEFAULT_VALIDATOR_KEYS_FOLDER_NAME, -) -from eth2deposit.utils.ascii_art import RHINO_0 -from eth2deposit.settings import ( - ALL_CHAINS, - MAINNET, - get_setting, +from eth2deposit.utils.constants import WORD_LISTS_PATH + +from .generate_keys import ( + generate_keys, + generate_keys_arguments_wrapper, ) languages = get_languages(WORD_LISTS_PATH) + @click.command() +@click.pass_context @click.option( '--mnemonic_language', prompt='Please choose your mnemonic language', type=click.Choice(languages, case_sensitive=False), default='english', ) -def generate_mnemonic(mnemonic_language: str) -> str: +@generate_keys_arguments_wrapper +def new_mnemonic(ctx: click.Context, mnemonic_language: str, **kwargs: Any) -> None: mnemonic = get_mnemonic(language=mnemonic_language, words_path=WORD_LISTS_PATH) test_mnemonic = '' while mnemonic != test_mnemonic: @@ -43,4 +39,5 @@ def generate_mnemonic(mnemonic_language: str) -> str: test_mnemonic = click.prompt('Please type your mnemonic (separated by spaces) to confirm you have written it down\n\n') # noqa: E501 test_mnemonic = test_mnemonic.lower() click.clear() - return mnemonic + ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': ''} + ctx.forward(generate_keys) diff --git a/eth2deposit/credentials.py b/eth2deposit/credentials.py index 96906b37..2cbcf435 100644 --- a/eth2deposit/credentials.py +++ b/eth2deposit/credentials.py @@ -22,7 +22,7 @@ class Credential: - def __init__(self, *, mnemonic: str, index: int, amount: int, fork_version: bytes): + def __init__(self, *, mnemonic: str, mnemonic_password: str, index: int, amount: int, fork_version: bytes): # Set path as EIP-2334 format # https://eips.ethereum.org/EIPS/eip-2334 purpose = '12381' @@ -32,8 +32,10 @@ def __init__(self, *, mnemonic: str, index: int, amount: int, fork_version: byte self.signing_key_path = f'{withdrawal_key_path}/0' # Do NOT use password for seed generation. - self.withdrawal_sk = mnemonic_and_path_to_key(mnemonic=mnemonic, path=withdrawal_key_path, password='') - self.signing_sk = mnemonic_and_path_to_key(mnemonic=mnemonic, path=self.signing_key_path, password='') + self.withdrawal_sk = mnemonic_and_path_to_key( + mnemonic=mnemonic, path=withdrawal_key_path, password=mnemonic_password) + self.signing_sk = mnemonic_and_path_to_key( + mnemonic=mnemonic, path=self.signing_key_path, password=mnemonic_password) self.amount = amount self.fork_version = fork_version @@ -102,13 +104,15 @@ def __init__(self, credentials: List[Credential]): def from_mnemonic(cls, *, mnemonic: str, + mnemonic_password: str, num_keys: int, amounts: List[int], fork_version: bytes, - start_index: int=0) -> 'CredentialList': + start_index: int) -> 'CredentialList': assert len(amounts) == num_keys key_indices = range(start_index, start_index + num_keys) - return cls([Credential(mnemonic=mnemonic, index=index, amount=amounts[index], fork_version=fork_version) + return cls([Credential(mnemonic=mnemonic, mnemonic_password=mnemonic_password, + index=index, amount=amounts[index], fork_version=fork_version) for index in key_indices]) def export_keystores(self, password: str, folder: str) -> List[str]: diff --git a/eth2deposit/deposit.py b/eth2deposit/deposit.py index 8d6090d0..fae42cee 100644 --- a/eth2deposit/deposit.py +++ b/eth2deposit/deposit.py @@ -1,44 +1,8 @@ -import os import sys import click -from eth2deposit.credentials import ( - CredentialList, -) -from eth2deposit.key_handling.key_derivation.mnemonic import ( - get_languages, - get_mnemonic, -) -from eth2deposit.utils.validation import verify_deposit_data_json -from eth2deposit.utils.constants import ( - WORD_LISTS_PATH, - MAX_DEPOSIT_AMOUNT, - DEFAULT_VALIDATOR_KEYS_FOLDER_NAME, -) -from eth2deposit.utils.ascii_art import RHINO_0 -from eth2deposit.settings import ( - ALL_CHAINS, - MAINNET, - get_setting, -) - -languages = get_languages(WORD_LISTS_PATH) - - -def generate_mnemonic(language: str, words_path: str) -> str: - mnemonic = get_mnemonic(language=language, words_path=words_path) - test_mnemonic = '' - while mnemonic != test_mnemonic: - click.clear() - click.echo('This is your seed phrase. Write it down and store it safely, it is the ONLY way to retrieve your deposit.') # noqa: E501 - click.echo('\n\n%s\n\n' % mnemonic) - click.pause('Press any key when you have written down your mnemonic.') - - click.clear() - test_mnemonic = click.prompt('Please type your mnemonic (separated by spaces) to confirm you have written it down\n\n') # noqa: E501 - test_mnemonic = test_mnemonic.lower() - click.clear() - return mnemonic +from eth2deposit.cli.existing_mnemonic import existing_mnemonic +from eth2deposit.cli.new_mnemonic import new_mnemonic def check_python_version() -> None: @@ -50,52 +14,16 @@ def check_python_version() -> None: sys.exit() -@click.command() -@click.option( - '--num_validators', - prompt='Please choose how many validators you wish to run', - required=True, - type=click.IntRange(0, 2**32)), -) -@click.option( - '--validator_start_index', - type=click.IntRange(0, 2**32)), -) -@click.option( - '--chain', - prompt='Please choose the (mainnet or testnet) network/chain name', - type=click.Choice(ALL_CHAINS.keys(), case_sensitive=False), - default=MAINNET, -) -@click.password_option(prompt='Type the password that secures your validator keystore(s)') -def main(num_validators: int, mnemonic_language: str, folder: str, chain: str, password: str) -> None: - check_python_version() - mnemonic = generate_mnemonic(mnemonic_language, WORD_LISTS_PATH) - amounts = [MAX_DEPOSIT_AMOUNT] * num_validators - folder = os.path.join(folder, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) - setting = get_setting(chain) - if not os.path.exists(folder): - os.mkdir(folder) - click.clear() - click.echo(RHINO_0) - click.echo('Creating your keys.') - credentials = CredentialList.from_mnemonic( - mnemonic=mnemonic, - num_keys=num_validators, - amounts=amounts, - fork_version=setting.GENESIS_FORK_VERSION, - ) - click.echo('Saving your keystore(s).') - keystore_filefolders = credentials.export_keystores(password=password, folder=folder) - click.echo('Creating your deposit(s).') - deposits_file = credentials.export_deposit_data_json(folder=folder) - click.echo('Verifying your keystore(s).') - assert credentials.verify_keystores(keystore_filefolders=keystore_filefolders, password=password) - click.echo('Verifying your deposit(s).') - assert verify_deposit_data_json(deposits_file) - click.echo('\nSuccess!\nYour keys can be found at: %s' % folder) - click.pause('\n\nPress any key.') +# cli = click.CommandCollection(sources=[existing_mnemonic, new_mnemonic]) + +@click.group() +def cli() -> None: + pass +cli.add_command(existing_mnemonic) +cli.add_command(new_mnemonic) + if __name__ == '__main__': - main() + check_python_version() + cli() diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index 48d3088c..21bd1c26 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -70,13 +70,13 @@ def get_languages(path: str) -> Tuple[str, ...]: return languages -def determine_mnemonic_language(mnemonic: str, words_path:str) -> Sequence[str]: +def determine_mnemonic_language(mnemonic: str, words_path: str) -> Sequence[str]: """ Given a `mnemonic` determine what language[s] it is written in. There are collisions between word-lists, so multiple candidate languages are returned. """ languages = get_languages(words_path) - word_language_map = {word: lang for lang in languages for word in _get_word_list(lang, words_path)} + word_language_map = {word: lang for lang in languages for word in _get_word_list(lang, words_path)} try: mnemonic_list = mnemonic.split(' ') word_languages = [word_language_map[word] for word in mnemonic_list] @@ -104,7 +104,7 @@ def verify_mnemonic(mnemonic: str, words_path: str) -> bool: mnemonic_list = mnemonic.split(' ') word_indices = [_word_to_index(word_list, word) for word in mnemonic_list] mnemonic_int = _uint11_array_to_uint(word_indices) - checksum_length = len(mnemonic_list)//3 + checksum_length = len(mnemonic_list) // 3 checksum = mnemonic_int & 2**checksum_length - 1 entropy = (mnemonic_int - checksum) >> checksum_length entropy_bits = entropy.to_bytes(checksum_length * 4, 'big') @@ -114,7 +114,6 @@ def verify_mnemonic(mnemonic: str, words_path: str) -> bool: return False - def get_mnemonic(*, language: str, words_path: str, entropy: Optional[bytes]=None) -> str: """ Return a mnemonic string in a given `language` based on `entropy`. diff --git a/tests/test_key_handling/test_key_derivation/test_mnemonic.py b/tests/test_key_handling/test_key_derivation/test_mnemonic.py index f2d58e5d..010feccf 100644 --- a/tests/test_key_handling/test_key_derivation/test_mnemonic.py +++ b/tests/test_key_handling/test_key_derivation/test_mnemonic.py @@ -32,9 +32,12 @@ def test_bip39(language: str, test: Sequence[str]) -> None: assert get_mnemonic(language=language, words_path=WORD_LISTS_PATH, entropy=test_entropy) == test_mnemonic assert get_seed(mnemonic=test_mnemonic, password='TREZOR') == test_seed + @pytest.mark.parametrize( 'test_mnemonic,is_valid', - [(test_mnemonic[1], True) for _, language_test_vectors in test_vectors.items() for test_mnemonic in language_test_vectors] + [(test_mnemonic[1], True) + for _, language_test_vectors in test_vectors.items() + for test_mnemonic in language_test_vectors] ) def test_verify_mnemonic(test_mnemonic: str, is_valid: bool) -> None: assert verify_mnemonic(test_mnemonic, WORD_LISTS_PATH) == is_valid From c7ec603bcef221154a20ed1d7b7e762cd8fb3b95 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 28 Sep 2020 17:20:08 +0200 Subject: [PATCH 04/24] minor cleanups --- eth2deposit/cli/existing_mnemonic.py | 4 ++-- eth2deposit/cli/new_mnemonic.py | 4 ++-- eth2deposit/deposit.py | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 6a66746a..4246200b 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -11,7 +11,7 @@ ) from .generate_keys import ( generate_keys, - generate_keys_arguments_wrapper, + generate_keys_arguments_decorator, ) @@ -24,7 +24,7 @@ def validate_mnemonic(mnemonic: str) -> str: @click.command() @click.pass_context -@generate_keys_arguments_wrapper +@generate_keys_arguments_decorator @click.option( '--mnemonic', prompt='Please enter your mnemonic separated by spaces (" ").', diff --git a/eth2deposit/cli/new_mnemonic.py b/eth2deposit/cli/new_mnemonic.py index c6356869..2c012ccf 100644 --- a/eth2deposit/cli/new_mnemonic.py +++ b/eth2deposit/cli/new_mnemonic.py @@ -11,7 +11,7 @@ from .generate_keys import ( generate_keys, - generate_keys_arguments_wrapper, + generate_keys_arguments_decorator, ) languages = get_languages(WORD_LISTS_PATH) @@ -25,7 +25,7 @@ type=click.Choice(languages, case_sensitive=False), default='english', ) -@generate_keys_arguments_wrapper +@generate_keys_arguments_decorator def new_mnemonic(ctx: click.Context, mnemonic_language: str, **kwargs: Any) -> None: mnemonic = get_mnemonic(language=mnemonic_language, words_path=WORD_LISTS_PATH) test_mnemonic = '' diff --git a/eth2deposit/deposit.py b/eth2deposit/deposit.py index fae42cee..0d2cd403 100644 --- a/eth2deposit/deposit.py +++ b/eth2deposit/deposit.py @@ -14,8 +14,6 @@ def check_python_version() -> None: sys.exit() -# cli = click.CommandCollection(sources=[existing_mnemonic, new_mnemonic]) - @click.group() def cli() -> None: pass @@ -24,6 +22,7 @@ def cli() -> None: cli.add_command(existing_mnemonic) cli.add_command(new_mnemonic) + if __name__ == '__main__': check_python_version() cli() From d3a6f7d589ce56d84cd3c82ef3a1f6d7feff9ea4 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 29 Sep 2020 16:24:18 +0200 Subject: [PATCH 05/24] Pass cli tests --- eth2deposit/cli/existing_mnemonic.py | 20 +++++++++++++------ eth2deposit/cli/generate_keys.py | 5 ----- eth2deposit/cli/new_mnemonic.py | 1 + eth2deposit/credentials.py | 2 +- .../key_handling/key_derivation/mnemonic.py | 7 ++++++- setup.py | 2 +- test_deposit_script.py | 5 ++--- tests/test_cli.py | 14 ++++++------- 8 files changed, 32 insertions(+), 24 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 4246200b..9c4ac913 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -15,27 +15,35 @@ ) -def validate_mnemonic(mnemonic: str) -> str: +def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: if verify_mnemonic(mnemonic, WORD_LISTS_PATH): return mnemonic else: - raise click.BadParameter('That is not a valid mnemonic in any language.') + raise click.BadParameter('That is not a valid mnemonic, please check for typos') @click.command() @click.pass_context -@generate_keys_arguments_decorator @click.option( '--mnemonic', - prompt='Please enter your mnemonic separated by spaces (" ").', + callback=validate_mnemonic, + prompt='Please enter your mnemonic separated by spaces (" ")', required=True, type=str, ) -@click.option( +@click.password_option( '--mnemonic-password', - type=str, default='', ) +@click.option( + '--validator_start_index', + confirmation_prompt=True, + default=0, + prompt='Enter the index (key number) you wish to start generating more keys from. \ + For example, if you\'ve generated 4 keys in the past, you\'d enter 4 here,', + type=click.IntRange(0, 2**32), +) +@generate_keys_arguments_decorator def existing_mnemonic(ctx: click.Context, mnemonic: str, mnemonic_password: str, **kwargs: Any) -> None: ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': mnemonic_password} ctx.forward(generate_keys) diff --git a/eth2deposit/cli/generate_keys.py b/eth2deposit/cli/generate_keys.py index 4aafeae0..ad7f6206 100644 --- a/eth2deposit/cli/generate_keys.py +++ b/eth2deposit/cli/generate_keys.py @@ -33,11 +33,6 @@ def generate_keys_arguments_decorator(function: Callable[..., Any]) -> Callable[ required=True, type=click.IntRange(0, 2**32), ), - click.option( - '--validator_start_index', - type=click.IntRange(0, 2**32), - default=0, - ), click.option( '--folder', type=click.Path(exists=True, file_okay=False, dir_okay=True), diff --git a/eth2deposit/cli/new_mnemonic.py b/eth2deposit/cli/new_mnemonic.py index 2c012ccf..be22ade5 100644 --- a/eth2deposit/cli/new_mnemonic.py +++ b/eth2deposit/cli/new_mnemonic.py @@ -40,4 +40,5 @@ def new_mnemonic(ctx: click.Context, mnemonic_language: str, **kwargs: Any) -> N test_mnemonic = test_mnemonic.lower() click.clear() ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': ''} + ctx.params['validator_start_index'] = 0 ctx.forward(generate_keys) diff --git a/eth2deposit/credentials.py b/eth2deposit/credentials.py index 2cbcf435..797b189f 100644 --- a/eth2deposit/credentials.py +++ b/eth2deposit/credentials.py @@ -112,7 +112,7 @@ def from_mnemonic(cls, assert len(amounts) == num_keys key_indices = range(start_index, start_index + num_keys) return cls([Credential(mnemonic=mnemonic, mnemonic_password=mnemonic_password, - index=index, amount=amounts[index], fork_version=fork_version) + index=index, amount=amounts[index - start_index], fork_version=fork_version) for index in key_indices]) def export_keystores(self, password: str, folder: str) -> List[str]: diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index 21bd1c26..f084f826 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -97,11 +97,16 @@ def _get_checksum(entropy: bytes) -> int: def verify_mnemonic(mnemonic: str, words_path: str) -> bool: "Given a mnemonic, verify it against its own checksum." - languages = determine_mnemonic_language(mnemonic, words_path) + try: + languages = determine_mnemonic_language(mnemonic, words_path) + except ValueError: + return False for language in languages: try: word_list = _get_word_list(language, words_path) mnemonic_list = mnemonic.split(' ') + if len(mnemonic_list) not in range(12, 27, 3): + return False word_indices = [_word_to_index(word_list, word) for word in mnemonic_list] mnemonic_int = _uint11_array_to_uint(word_indices) checksum_length = len(mnemonic_list) // 3 diff --git a/setup.py b/setup.py index 1ac94ed3..ed887113 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="eth2deposit", - version='0.2.1', + version='0.3.0', py_modules=["eth2deposit"], packages=find_packages(exclude=('tests', 'docs')), python_requires=">=3.7,<4", diff --git a/test_deposit_script.py b/test_deposit_script.py index 4fd94282..e4682642 100755 --- a/test_deposit_script.py +++ b/test_deposit_script.py @@ -24,14 +24,13 @@ async def main(): print('[INFO] Installed') cmd_args = [ - run_script_cmd, + run_script_cmd + ' new-mnemonic', '--num_validators', '1', '--mnemonic_language', 'english', '--chain', 'mainnet', - '--password', 'MyPassword', + '--keystore_password', 'MyPassword', '--folder', my_folder_path, ] - print('[INFO] Creating subprocess 2: deposit-cli') proc = await asyncio.create_subprocess_shell( ' '.join(cmd_args), stdin=asyncio.subprocess.PIPE, diff --git a/tests/test_cli.py b/tests/test_cli.py index 67976f1b..9d435dd1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,8 +9,8 @@ from click.testing import CliRunner -from eth2deposit import deposit -from eth2deposit.deposit import main +from eth2deposit.key_handling.key_derivation import mnemonic as _mnemonic +from eth2deposit.deposit import cli from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME from eth2deposit.key_handling.keystore import Keystore @@ -32,7 +32,7 @@ def test_deposit(monkeypatch) -> None: def get_mnemonic(language: str, words_path: str, entropy: Optional[bytes]=None) -> str: return "fakephrase" - monkeypatch.setattr(deposit, "get_mnemonic", get_mnemonic) + monkeypatch.setattr(_mnemonic, "get_mnemonic", get_mnemonic) # Prepare folder my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') @@ -41,9 +41,9 @@ def get_mnemonic(language: str, words_path: str, entropy: Optional[bytes]=None) os.mkdir(my_folder_path) runner = CliRunner() - inputs = ['5', 'english', 'mainnet', 'MyPassword', 'MyPassword', 'fakephrase'] + inputs = ['abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about', 'TREZOR', 'TREZOR', '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword'] data = '\n'.join(inputs) - result = runner.invoke(main, ['--folder', my_folder_path], input=data) + result = runner.invoke(cli, ['existing-mnemonic', '--folder', my_folder_path], input=data) assert result.exit_code == 0 @@ -84,11 +84,11 @@ async def test_script() -> None: await proc.wait() cmd_args = [ - run_script_cmd, + run_script_cmd + ' new-mnemonic', '--num_validators', '1', '--mnemonic_language', 'english', '--chain', 'mainnet', - '--password', 'MyPassword', + '--keystore_password', 'MyPassword', '--folder', my_folder_path, ] proc = await asyncio.create_subprocess_shell( From 1e04ab2566cedce415354ec6e827eda4674d98da Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 29 Sep 2020 16:30:57 +0200 Subject: [PATCH 06/24] Linting is a thing --- tests/test_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9d435dd1..929a21fb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,7 +41,9 @@ def get_mnemonic(language: str, words_path: str, entropy: Optional[bytes]=None) os.mkdir(my_folder_path) runner = CliRunner() - inputs = ['abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about', 'TREZOR', 'TREZOR', '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword'] + inputs = [ + 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about', + 'TREZOR', 'TREZOR', '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword'] data = '\n'.join(inputs) result = runner.invoke(cli, ['existing-mnemonic', '--folder', my_folder_path], input=data) From 4ff9f2a01f53560fef98f1c9dbd0ce1251f58bed Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 20 Oct 2020 15:50:46 +0200 Subject: [PATCH 07/24] Clarifies the dangers of mnemonic-passwords --- eth2deposit/cli/existing_mnemonic.py | 19 +++++++++++++++++-- tests/test_cli.py | 5 +++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 9c4ac913..008c2631 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -34,16 +34,31 @@ def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: @click.password_option( '--mnemonic-password', default='', + help=('This is almost certainly not the argument you are looking for: it is for mnemonic passwords, not keystore ' + 'passwords. Providing a password here when you didn\'t use one initially, can result in lost keys (and ' + 'therefore funds)! Also note that if you used this tool to generate your mnemonic intially, then you did not ' + 'use a mnemonic password. However, if you are certain you used a password to "increase" the security of your ' + 'mnemonic, this is where you enter it. '), + prompt=False, ) @click.option( '--validator_start_index', confirmation_prompt=True, default=0, - prompt='Enter the index (key number) you wish to start generating more keys from. \ - For example, if you\'ve generated 4 keys in the past, you\'d enter 4 here,', + prompt=('Enter the index (key number) you wish to start generating more keys from. ' + 'For example, if you\'ve generated 4 keys in the past, you\'d enter 4 here,'), type=click.IntRange(0, 2**32), ) @generate_keys_arguments_decorator def existing_mnemonic(ctx: click.Context, mnemonic: str, mnemonic_password: str, **kwargs: Any) -> None: + if mnemonic_password != '': + click.clear() + click.confirm( + ('Are you absolutely certain that you used a mnemonic password? ' + '(This is different from a keystore password!) ' + 'Using one when you are not supposed to can result in loss of funds!'), + abort=True) + click.confirm('Did you generate this mnemonic with this CLI tool initially?', abort=True) + ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': mnemonic_password} ctx.forward(generate_keys) diff --git a/tests/test_cli.py b/tests/test_cli.py index 929a21fb..c398edc7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,9 +43,10 @@ def get_mnemonic(language: str, words_path: str, entropy: Optional[bytes]=None) runner = CliRunner() inputs = [ 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about', - 'TREZOR', 'TREZOR', '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword'] + '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword', 'yes', 'yes'] data = '\n'.join(inputs) - result = runner.invoke(cli, ['existing-mnemonic', '--folder', my_folder_path], input=data) + arguments = ['existing-mnemonic', '--folder', my_folder_path, '--mnemonic-password', 'TREZOR'] + result = runner.invoke(cli, arguments, input=data) assert result.exit_code == 0 From 3891cbada55e529689bfec2a5d12b9c636dfb695 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 21 Oct 2020 16:32:25 +0200 Subject: [PATCH 08/24] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang --- eth2deposit/credentials.py | 1 - .../key_handling/key_derivation/mnemonic.py | 14 ++++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eth2deposit/credentials.py b/eth2deposit/credentials.py index 2f515015..66d82b4a 100644 --- a/eth2deposit/credentials.py +++ b/eth2deposit/credentials.py @@ -40,7 +40,6 @@ def __init__(self, *, mnemonic: str, mnemonic_password: str, index: int, amount: withdrawal_key_path = f'm/{purpose}/{coin_type}/{account}/0' self.signing_key_path = f'{withdrawal_key_path}/0' - # Do NOT use password for seed generation. self.withdrawal_sk = mnemonic_and_path_to_key( mnemonic=mnemonic, path=withdrawal_key_path, password=mnemonic_password) self.signing_sk = mnemonic_and_path_to_key( diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index f6752537..0f1afd4e 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -54,8 +54,8 @@ def _word_to_index(word_list: Sequence[str], word: str) -> int: raise ValueError('Word %s not in BIP39 word-list' % word) -def _uint11_array_to_uint(unit_array: Sequence[int]) -> int: - return sum([x << i * 11 for i, x in enumerate(reversed(unit_array))]) +def _uint11_array_to_uint(uint11_array: Sequence[int]) -> int: + return sum(x << i * 11 for i, x in enumerate(reversed(uint11_array))) def get_seed(*, mnemonic: str, password: str) -> bytes: @@ -102,11 +102,13 @@ def _get_checksum(entropy: bytes) -> int: entropy_length = len(entropy) * 8 assert entropy_length in range(128, 257, 32) checksum_length = (entropy_length // 32) - return int.from_bytes(SHA256(entropy), 'big') >> 256 - checksum_length + return int.from_bytes(SHA256(entropy), 'big') >> (256 - checksum_length) def verify_mnemonic(mnemonic: str, words_path: str) -> bool: - "Given a mnemonic, verify it against its own checksum." + """ + Given a mnemonic, verify it against its own checksum." + """ try: languages = determine_mnemonic_language(mnemonic, words_path) except ValueError: @@ -115,7 +117,7 @@ def verify_mnemonic(mnemonic: str, words_path: str) -> bool: try: word_list = _get_word_list(language, words_path) mnemonic_list = mnemonic.split(' ') - if len(mnemonic_list) not in range(12, 27, 3): + if len(mnemonic_list) not in range(12, 25, 3): return False word_indices = [_word_to_index(word_list, word) for word in mnemonic_list] mnemonic_int = _uint11_array_to_uint(word_indices) @@ -139,7 +141,7 @@ def get_mnemonic(*, language: str, words_path: str, entropy: Optional[bytes]=Non entropy = randbits(256).to_bytes(32, 'big') entropy_length = len(entropy) * 8 if entropy_length not in range(128, 257, 32): - raise IndexError(f"`entropy_length` should be in [128, 160, 192,224, 256]. Got {entropy_length}.") + raise IndexError(f"`entropy_length` should be in [128, 160, 192, 224, 256]. Got {entropy_length}.") checksum_length = (entropy_length // 32) checksum = _get_checksum(entropy) entropy_bits = int.from_bytes(entropy, 'big') << checksum_length From 52b36bcdff1e977a4da2c2eb8f6e540c4bc2bda7 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 26 Oct 2020 17:21:57 +0100 Subject: [PATCH 09/24] Fix indentation error --- eth2deposit/key_handling/key_derivation/mnemonic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index 0f1afd4e..c0e8e81f 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -102,7 +102,7 @@ def _get_checksum(entropy: bytes) -> int: entropy_length = len(entropy) * 8 assert entropy_length in range(128, 257, 32) checksum_length = (entropy_length // 32) - return int.from_bytes(SHA256(entropy), 'big') >> (256 - checksum_length) + return int.from_bytes(SHA256(entropy), 'big') >> (256 - checksum_length) def verify_mnemonic(mnemonic: str, words_path: str) -> bool: From 633409c0594e2ea07cb5170025f0bdb022fcd2ac Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 26 Oct 2020 17:51:38 +0100 Subject: [PATCH 10/24] Implement @hwwhww's code review suggestions --- eth2deposit/cli/existing_mnemonic.py | 3 ++- eth2deposit/cli/new_mnemonic.py | 1 + eth2deposit/key_handling/key_derivation/mnemonic.py | 13 ++++++++----- .../test_key_derivation/test_mnemonic.py | 11 +++++++---- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 008c2631..d885005e 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -3,6 +3,7 @@ Any, ) +from eth2deposit.exceptions import ValidationError from eth2deposit.key_handling.key_derivation.mnemonic import ( verify_mnemonic, ) @@ -19,7 +20,7 @@ def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: if verify_mnemonic(mnemonic, WORD_LISTS_PATH): return mnemonic else: - raise click.BadParameter('That is not a valid mnemonic, please check for typos') + raise ValidationError('That is not a valid mnemonic, please check for typos.') @click.command() diff --git a/eth2deposit/cli/new_mnemonic.py b/eth2deposit/cli/new_mnemonic.py index be22ade5..287b19a1 100644 --- a/eth2deposit/cli/new_mnemonic.py +++ b/eth2deposit/cli/new_mnemonic.py @@ -39,6 +39,7 @@ def new_mnemonic(ctx: click.Context, mnemonic_language: str, **kwargs: Any) -> N test_mnemonic = click.prompt('Please type your mnemonic (separated by spaces) to confirm you have written it down\n\n') # noqa: E501 test_mnemonic = test_mnemonic.lower() click.clear() + # Do NOT use mnemonic_password. ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': ''} ctx.params['validator_start_index'] = 0 ctx.forward(generate_keys) diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index c0e8e81f..756ec5eb 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -95,13 +95,18 @@ def determine_mnemonic_language(mnemonic: str, words_path: str) -> Sequence[str] raise ValueError('Word not found in mnemonic word lists for any language.') +def _validate_entropy_length(entropy: bytes) -> None: + entropy_length = len(entropy) * 8 + if entropy_length not in range(128, 257, 32): + raise IndexError(f"`entropy_length` should be in [128, 160, 192, 224, 256]. Got {entropy_length}.") + + def _get_checksum(entropy: bytes) -> int: """ Determine the index of the checksum word given the entropy """ - entropy_length = len(entropy) * 8 - assert entropy_length in range(128, 257, 32) - checksum_length = (entropy_length // 32) + _validate_entropy_length(entropy) + checksum_length = len(entropy) // 4 return int.from_bytes(SHA256(entropy), 'big') >> (256 - checksum_length) @@ -140,8 +145,6 @@ def get_mnemonic(*, language: str, words_path: str, entropy: Optional[bytes]=Non if entropy is None: entropy = randbits(256).to_bytes(32, 'big') entropy_length = len(entropy) * 8 - if entropy_length not in range(128, 257, 32): - raise IndexError(f"`entropy_length` should be in [128, 160, 192, 224, 256]. Got {entropy_length}.") checksum_length = (entropy_length // 32) checksum = _get_checksum(entropy) entropy_bits = int.from_bytes(entropy, 'big') << checksum_length diff --git a/tests/test_key_handling/test_key_derivation/test_mnemonic.py b/tests/test_key_handling/test_key_derivation/test_mnemonic.py index 8453e787..51f6d335 100644 --- a/tests/test_key_handling/test_key_derivation/test_mnemonic.py +++ b/tests/test_key_handling/test_key_derivation/test_mnemonic.py @@ -48,11 +48,14 @@ def test_verify_mnemonic(test_mnemonic: str, is_valid: bool) -> None: @pytest.mark.parametrize( - 'language, index, valid', + 'language', ['english'] +) +@pytest.mark.parametrize( + 'index, valid', [ - ('english', 0, True), - ('english', 2047, True), - ('english', 2048, False), + (0, True), + (2047, True), + (2048, False), ] ) def test_get_word(language: str, index: int, valid: bool) -> None: From f4d734173d5e73aa4234f19a7f659593e9d538be Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 28 Oct 2020 19:07:13 +0100 Subject: [PATCH 11/24] Adds tests for both new and existing mnemonics via CLI --- eth2deposit/cli/existing_mnemonic.py | 1 - tests/test_cli/__init__.py | 0 tests/test_cli/helpers.py | 15 ++++ .../test_existing_menmonic.py} | 51 +++---------- tests/test_cli/test_new_mnemonic.py | 75 +++++++++++++++++++ 5 files changed, 100 insertions(+), 42 deletions(-) create mode 100644 tests/test_cli/__init__.py create mode 100644 tests/test_cli/helpers.py rename tests/{test_cli.py => test_cli/test_existing_menmonic.py} (64%) create mode 100644 tests/test_cli/test_new_mnemonic.py diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index d885005e..d15a3391 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -59,7 +59,6 @@ def existing_mnemonic(ctx: click.Context, mnemonic: str, mnemonic_password: str, '(This is different from a keystore password!) ' 'Using one when you are not supposed to can result in loss of funds!'), abort=True) - click.confirm('Did you generate this mnemonic with this CLI tool initially?', abort=True) ctx.obj = {'mnemonic': mnemonic, 'mnemonic_password': mnemonic_password} ctx.forward(generate_keys) diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_cli/helpers.py b/tests/test_cli/helpers.py new file mode 100644 index 00000000..9b91d774 --- /dev/null +++ b/tests/test_cli/helpers.py @@ -0,0 +1,15 @@ +import os + +from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME + + +def clean_key_folder(my_folder_path: str) -> None: + validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) + if not os.path.exists(validator_keys_folder_path): + return + + _, _, key_files = next(os.walk(validator_keys_folder_path)) + for key_file_name in key_files: + os.remove(os.path.join(validator_keys_folder_path, key_file_name)) + os.rmdir(validator_keys_folder_path) + os.rmdir(my_folder_path) diff --git a/tests/test_cli.py b/tests/test_cli/test_existing_menmonic.py similarity index 64% rename from tests/test_cli.py rename to tests/test_cli/test_existing_menmonic.py index c398edc7..0965b472 100644 --- a/tests/test_cli.py +++ b/tests/test_cli/test_existing_menmonic.py @@ -3,37 +3,15 @@ import pytest -from typing import ( - Optional, -) - from click.testing import CliRunner -from eth2deposit.key_handling.key_derivation import mnemonic as _mnemonic from eth2deposit.deposit import cli from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME from eth2deposit.key_handling.keystore import Keystore +from.helpers import clean_key_folder -def clean_key_folder(my_folder_path: str) -> None: - validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) - if not os.path.exists(validator_keys_folder_path): - return - - _, _, key_files = next(os.walk(validator_keys_folder_path)) - for key_file_name in key_files: - os.remove(os.path.join(validator_keys_folder_path, key_file_name)) - os.rmdir(validator_keys_folder_path) - os.rmdir(my_folder_path) - - -def test_deposit(monkeypatch) -> None: - # monkeypatch get_mnemonic - def get_mnemonic(language: str, words_path: str, entropy: Optional[bytes]=None) -> str: - return "fakephrase" - - monkeypatch.setattr(_mnemonic, "get_mnemonic", get_mnemonic) - +def test_deposit() -> None: # Prepare folder my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') clean_key_folder(my_folder_path) @@ -43,7 +21,7 @@ def get_mnemonic(language: str, words_path: str, entropy: Optional[bytes]=None) runner = CliRunner() inputs = [ 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about', - '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword', 'yes', 'yes'] + '2', '2', '5', 'mainnet', 'MyPassword', 'MyPassword', 'yes'] data = '\n'.join(inputs) arguments = ['existing-mnemonic', '--folder', my_folder_path, '--mnemonic-password', 'TREZOR'] result = runner.invoke(cli, arguments, input=data) @@ -87,9 +65,12 @@ async def test_script() -> None: await proc.wait() cmd_args = [ - run_script_cmd + ' new-mnemonic', + run_script_cmd, + 'existing-mnemonic', '--num_validators', '1', - '--mnemonic_language', 'english', + '--mnemonic="abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about"', + '--mnemonic-password', 'TREZOR', + '--validator_start_index', '1', '--chain', 'mainnet', '--keystore_password', 'MyPassword', '--folder', my_folder_path, @@ -100,22 +81,10 @@ async def test_script() -> None: stdout=asyncio.subprocess.PIPE, ) - seed_phrase = '' - parsing = False async for out in proc.stdout: output = out.decode('utf-8').rstrip() - if output.startswith("This is your seed phrase."): - parsing = True - elif output.startswith("Please type your mnemonic"): - parsing = False - elif parsing: - seed_phrase += output - if len(seed_phrase) > 0: - encoded_phrase = seed_phrase.encode() - proc.stdin.write(encoded_phrase) - proc.stdin.write(b'\n') - - assert len(seed_phrase) > 0 + if output.startswith('Running deposit-cli...'): + proc.stdin.write(b'y\n') # Check files validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) diff --git a/tests/test_cli/test_new_mnemonic.py b/tests/test_cli/test_new_mnemonic.py new file mode 100644 index 00000000..b7dee4c8 --- /dev/null +++ b/tests/test_cli/test_new_mnemonic.py @@ -0,0 +1,75 @@ +import asyncio +import os + +import pytest + +from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME +from eth2deposit.key_handling.keystore import Keystore +from .helpers import clean_key_folder + + +@pytest.mark.asyncio +async def test_script() -> None: + my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') + if not os.path.exists(my_folder_path): + os.mkdir(my_folder_path) + + if os.name == 'nt': # Windows + run_script_cmd = 'sh deposit.sh' + else: # Mac or Linux + run_script_cmd = './deposit.sh' + + install_cmd = run_script_cmd + ' install' + proc = await asyncio.create_subprocess_shell( + install_cmd, + ) + await proc.wait() + + cmd_args = [ + run_script_cmd + ' new-mnemonic', + '--num_validators', '5', + '--mnemonic_language', 'english', + '--chain', 'mainnet', + '--keystore_password', 'MyPassword', + '--folder', my_folder_path, + ] + proc = await asyncio.create_subprocess_shell( + ' '.join(cmd_args), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + ) + + seed_phrase = '' + parsing = False + async for out in proc.stdout: + output = out.decode('utf-8').rstrip() + if output.startswith("This is your seed phrase."): + parsing = True + elif output.startswith("Please type your mnemonic"): + parsing = False + elif parsing: + seed_phrase += output + if len(seed_phrase) > 0: + encoded_phrase = seed_phrase.encode() + proc.stdin.write(encoded_phrase) + proc.stdin.write(b'\n') + + assert len(seed_phrase) > 0 + + # Check files + validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) + _, _, key_files = next(os.walk(validator_keys_folder_path)) + + def get_uuid(key_file: str) -> str: + keystore = Keystore.from_json(key_file) + return keystore.uuid + + all_uuid = [ + get_uuid(validator_keys_folder_path + '/' + key_file) + for key_file in key_files + if key_file.startswith('keystore') + ] + assert len(set(all_uuid)) == 5 + + # Clean up + clean_key_folder(my_folder_path) From ffcbfbcf5ba70de460670478c6ea0b5270717a8f Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 30 Oct 2020 17:55:35 +0100 Subject: [PATCH 12/24] Adds new-mnemonic and existing-mnemonic commands to README --- README.md | 191 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 141 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 7aa76366..ca1933af 100644 --- a/README.md +++ b/README.md @@ -11,38 +11,41 @@ - [Option 1. Download binary executable file](#option-1-download-binary-executable-file) - [Step 1. Installation](#step-1-installation) - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json) - - [Arguments](#arguments) + - [Commands](#commands) + - [`new-mnemonic` Arguments](#new-mnemonic-arguments) + - [`existing-mnemonic` Arguments](#existing-mnemonic-arguments) - [Successful message](#successful-message) - [Option 2. Build `deposit-cli` with native Python](#option-2-build-deposit-cli-with-native-python) - [Step 0. Python version checking](#step-0-python-version-checking) - [Step 1. Installation](#step-1-installation-1) - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-1) - - [Arguments](#arguments-1) + - [Commands](#commands-1) + - [Arguments](#arguments) - [Successful message](#successful-message-1) - [Option 3. Build `deposit-cli` with `virtualenv`](#option-3-build-deposit-cli-with-virtualenv) - [Step 0. Python version checking](#step-0-python-version-checking-1) - [Step 1. Installation](#step-1-installation-2) - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-2) - - [Arguments](#arguments-2) - - [Successful message](#successful-message-2) + - [Commands](#commands-2) + - [Arguments](#arguments-1) - [For Windows users](#for-windows-users) - [Option 1. Download binary executable file](#option-1-download-binary-executable-file-1) - [Step 1. Installation](#step-1-installation-3) - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-3) - - [Arguments](#arguments-3) - - [Successful message](#successful-message-3) + - [Commands](#commands-3) + - [Arguments](#arguments-2) - [Option 2. Build `deposit-cli` with native Python](#option-2-build-deposit-cli-with-native-python-1) - [Step 0. Python version checking](#step-0-python-version-checking-2) - [Step 1. Installation](#step-1-installation-4) - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-4) - - [Arguments](#arguments-4) - - [Successful message](#successful-message-4) + - [Commands](#commands-4) + - [Arguments](#arguments-3) - [Option 3. Build `deposit-cli` with `virtualenv`](#option-3-build-deposit-cli-with-virtualenv-1) - [Step 0. Python version checking](#step-0-python-version-checking-3) - [Step 1. Installation](#step-1-installation-5) - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-5) - - [Arguments](#arguments-5) - - [Successful message](#successful-message-5) + - [Commands](#commands-5) + - [Arguments](#arguments-4) - [Development](#development) - [Install basic requirements](#install-basic-requirements) - [Install testing requirements](#install-testing-requirements) @@ -71,21 +74,30 @@ See [releases page](https://github.com/ethereum/eth2.0-deposit-cli/releases) to ##### Step 2. Create keys and `deposit_data-*.json` -Run the following command to enter the interactive CLI: +Run the following command to enter the interactive CLI and generate keys from a new mnemonic: ```sh -./deposit +./deposit new-mnemonic ``` -You can also run the tool with optional arguments: +or run the following command to enter the interactive CLI and generate keys from an existing: ```sh -./deposit --num_validators= --mnemonic_language=english --chain= --folder= +./deposit existing-mnemonic ``` -###### Arguments +###### Commands + +The CLI offers different commands depending on what you want to do with the tool. + +| Command | Description | +| ------- | ----------- | +| `new-mnemonic` | (Recommended) If you don't already have a mnemonic that you have securely backed up, or you want to have a separate mnemonic for your eth2 validators, use this option. | +| `existing-mnemonic` | If you have a mnemonic that you already use, then this option allows you to derive new keys from your existing mnemonic. Use this tool, if you have already generated keys with this CLI before, if you want to reuse your mnemonic that you know is secure that you generated elsewhere (reusing your eth1 mnemonic etc), or if you lost your keystores and need to recover your validator/withdrawal keys. | -You can use `--help` flag to see all arguments. +###### `new-mnemonic` Arguments + +You can use `new-mnemonic --help` to see all arguments. Note that if there are missing arguments that the CLI needs, it will ask you for them. | Argument | Type | Description | | -------- | -------- | -------- | @@ -94,11 +106,22 @@ You can use `--help` flag to see all arguments. | `--folder` | String. Pointing to `./validator_keys` by default | The folder path for the keystore(s) and deposit(s) | | `--chain` | String. `mainnet` by default | The chain setting for the signing domain. | +###### `existing-mnemonic` Arguments + +You can use `existing-mnemonic --help` to see all arguments. Note that if there are missing arguments that the CLI needs, it will ask you for them. + +| Argument | Type | Description | +| -------- | -------- | -------- | +| `--validator_start_index` | Non-negative integer | The index of the first validator's keys you wish to generate. If this is your first time generating keys with this mnemonic, use 0. If you have generated keys using this mnemonic before, use the next index from which you want to start generating keys from (eg, if you've generated 4 keys before (keys #0, #1, #2, #3), then enter 4 here.| +| `--num_validators` | Non-negative integer | The number of signing keys you want to generate. Note that the child key(s) are generated via the same master key. | +| `--folder` | String. Pointing to `./validator_keys` by default | The folder path for the keystore(s) and deposit(s) | +| `--chain` | String. `mainnet` by default | The chain setting for the signing domain. | + ###### Successful message You will see the following messages after successfully generated the keystore(s) and the deposit(s): -``` +```text Creating your keys. Saving your keystore(s). Creating your deposit(s). @@ -136,20 +159,36 @@ Or use the helper script: ##### Step 2. Create keys and `deposit_data-*.json` -Run the following command to enter the interactive CLI: +Run one of the following command to enter the interactive CLI: + +```sh +./deposit.sh new-mnemonic +``` + +or ```sh -./deposit.sh +./deposit.sh existing-mnemonic ``` You can also run the tool with optional arguments: ```sh -./deposit.sh --num_validators= --mnemonic_language=english --chain= --folder= +./deposit.sh new-mnemonic --num_validators= --mnemonic_language=english --chain= --folder= +``` + +```sh +./deposit.sh existing-mnemonic --num_validators= --validator_start_index= --chain= --folder= ``` +###### Commands + +See [here](#commands) + ###### Arguments -See [here](#arguments) + +See [here](#new-mnemonic-arguments) for `new-mnemonic` arguments +See [here](#existing-mnemonic-arguments) for `existing-mnemonic` arguments ###### Successful message See [here](#successful-message) @@ -183,23 +222,36 @@ pip3 install -r requirements.txt ##### Step 2. Create keys and `deposit_data-*.json` -Run the following command to enter the interactive CLI: +Run one of the following command to enter the interactive CLI: + +```sh +python3 ./eth2deposit/deposit.py new-mnemonic +``` + +or ```sh -python3 ./eth2deposit/deposit.py +python3 ./eth2deposit/deposit.py existing-mnemonic ``` You can also run the tool with optional arguments: ```sh -python3 ./eth2deposit/deposit.py --num_validators= --mnemonic_language=english --chain= --folder= +python3 ./eth2deposit/deposit.py new-mnemonic --num_validators= --mnemonic_language=english --chain= --folder= +``` + +```sh +python3 ./eth2deposit/deposit.py existing-mnemonic --num_validators= --validator_start_index= --chain= --folder= ``` +###### Commands + +See [here](#commands) + ###### Arguments -See [here](#arguments) -###### Successful message -See [here](#successful-message) +See [here](#new-mnemonic-arguments) for `new-mnemonic` arguments +See [here](#existing-mnemonic-arguments) for `existing-mnemonic` arguments ---- @@ -213,23 +265,36 @@ See [releases page](https://github.com/ethereum/eth2.0-deposit-cli/releases) to ##### Step 2. Create keys and `deposit_data-*.json` -Run the following command to enter the interactive CLI: +Run one of the following command to enter the interactive CLI: + +```sh +deposit.exe new-mnemonic +``` + +or ```sh -deposit.exe +deposit.exe existing-mnemonic ``` You can also run the tool with optional arguments: ```sh -deposit.exe --num_validators= --mnemonic_language=english --chain= --folder= +deposit.exe new-mnemonic --num_validators= --mnemonic_language=english --chain= --folder= +``` + +```sh +deposit.exe existing-mnemonic --num_validators= --validator_start_index= --chain= --folder= ``` +###### Commands + +See [here](#commands) + ###### Arguments -See [here](#arguments) -###### Successful message -See [here](#successful-message) +See [here](#new-mnemonic-arguments) for `new-mnemonic` arguments +See [here](#existing-mnemonic-arguments) for `existing-mnemonic` arguments #### Option 2. Build `deposit-cli` with native Python @@ -258,23 +323,36 @@ sh deposit.sh install ##### Step 2. Create keys and `deposit_data-*.json` -Run the following command to enter the interactive CLI: +Run one of the following command to enter the interactive CLI: ```sh -sh deposit.sh +./deposit.sh new-mnemonic +``` + +or + +```sh +./deposit.sh existing-mnemonic ``` You can also run the tool with optional arguments: ```sh -sh deposit.sh --num_validators= --mnemonic_language=english --chain= --folder= +./deposit.sh new-mnemonic --num_validators= --mnemonic_language=english --chain= --folder= +``` + +```sh +./deposit.sh existing-mnemonic --num_validators= --validator_start_index= --chain= --folder= ``` +###### Commands + +See [here](#commands) + ###### Arguments -See [here](#arguments) -###### Successful message -See [here](#successful-message) +See [here](#new-mnemonic-arguments) for `new-mnemonic` arguments +See [here](#existing-mnemonic-arguments) for `existing-mnemonic` arguments #### Option 3. Build `deposit-cli` with `virtualenv` @@ -282,7 +360,7 @@ See [here](#successful-message) Ensure you are using Python version >= Python3.7 (Assume that you've installed Python 3 as the main Python): -```sh +```cmd python -V ``` @@ -290,7 +368,7 @@ python -V For the [virtualenv](https://virtualenv.pypa.io/en/latest/) users, you can create a new venv: -```sh +```cmd pip3 install virtualenv virtualenv venv .\venv\Scripts\activate @@ -298,30 +376,43 @@ virtualenv venv and install the dependencies: -```sh +```cmd python setup.py install pip3 install -r requirements.txt ``` ##### Step 2. Create keys and `deposit_data-*.json` -Run the following command to enter the interactive CLI: +Run one of the following command to enter the interactive CLI: -```sh -python .\eth2deposit\deposit.py +``cmd +python .\eth2deposit\deposit.py new-mnemonic +``` + +or + +```cmd +python .\eth2deposit\deposit.py existing-mnemonic ``` You can also run the tool with optional arguments: -```sh -python .\eth2deposit\deposit.py --num_validators= --mnemonic_language=english --chain= --folder= +```cmd +python .\eth2deposit\deposit.py new-mnemonic --num_validators= --mnemonic_language=english --chain= --folder= +``` + +```cmd +python .\eth2deposit\deposit.pyexisting-mnemonic --num_validators= --validator_start_index= --chain= --folder= ``` +###### Commands + +See [here](#commands) + ###### Arguments -See [here](#arguments) -###### Successful message -See [here](#successful-message) +See [here](#new-mnemonic-arguments) for `new-mnemonic` arguments +See [here](#existing-mnemonic-arguments) for `existing-mnemonic` arguments ## Development From 2d0813ba7862da75e952d52d8244c3a77bf9e96a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 3 Nov 2020 14:28:19 +0800 Subject: [PATCH 13/24] Fix Makefile and Dockerfile --- Dockerfile | 4 ++++ Makefile | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7333868d..81e2b253 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,8 @@ RUN pip3 install -r requirements.txt RUN python3 setup.py install +ARG cli_command + ENTRYPOINT [ "python3", "./eth2deposit/deposit.py" ] + +CMD [ $cli_command ] diff --git a/Makefile b/Makefile index b8a17863..a1e4ad8f 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ venv_lint: venv_build_test $(VENV_ACTIVATE) && flake8 --config=flake8.ini ./eth2deposit ./tests && mypy --config-file mypy.ini -p eth2deposit venv_deposit: venv_build - $(VENV_ACTIVATE) && python ./eth2deposit/deposit.py + $(VENV_ACTIVATE) && python ./eth2deposit/deposit.py $(filter-out $@,$(MAKECMDGOALS)) build_macos: venv_build ${VENV_NAME}/bin/python -m pip install -r ./build_configs/macos/requirements.txt @@ -57,5 +57,4 @@ build_docker: @docker build --pull -t $(DOCKER_IMAGE) . run_docker: - @docker run -it --rm $(DOCKER_IMAGE) - + @docker run -it --rm $(DOCKER_IMAGE) $(filter-out $@,$(MAKECMDGOALS)) From ad5b96e0b721798a6ce8c0e7b10ae41037b42311 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 3 Nov 2020 14:33:08 +0800 Subject: [PATCH 14/24] Update README --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 91c796ec..24fae7f6 100644 --- a/README.md +++ b/README.md @@ -28,24 +28,29 @@ - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-2) - [Commands](#commands-2) - [Arguments](#arguments-1) + - [Option 4. Use Docker image](#option-4-use-docker-image) + - [Step 1. Build the docker image](#step-1-build-the-docker-image) + - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-3) + - [Arguments](#arguments-2) + - [Successful message](#successful-message-2) - [For Windows users](#for-windows-users) - [Option 1. Download binary executable file](#option-1-download-binary-executable-file-1) - [Step 1. Installation](#step-1-installation-3) - - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-3) + - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-4) - [Commands](#commands-3) - - [Arguments](#arguments-2) + - [Arguments](#arguments-3) - [Option 2. Build `deposit-cli` with native Python](#option-2-build-deposit-cli-with-native-python-1) - [Step 0. Python version checking](#step-0-python-version-checking-2) - [Step 1. Installation](#step-1-installation-4) - - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-4) + - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-5) - [Commands](#commands-4) - - [Arguments](#arguments-3) + - [Arguments](#arguments-4) - [Option 3. Build `deposit-cli` with `virtualenv`](#option-3-build-deposit-cli-with-virtualenv-1) - [Step 0. Python version checking](#step-0-python-version-checking-3) - [Step 1. Installation](#step-1-installation-5) - - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-5) + - [Step 2. Create keys and `deposit_data-*.json`](#step-2-create-keys-and-deposit_data-json-6) - [Commands](#commands-5) - - [Arguments](#arguments-4) + - [Arguments](#arguments-5) - [Development](#development) - [Install basic requirements](#install-basic-requirements) - [Install testing requirements](#install-testing-requirements) @@ -274,13 +279,13 @@ docker run -it --rm -v $(pwd)/validator_keys:/app/validator_keys ethereum/eth2.0 You can also run the tool with optional arguments: ```sh -docker run -it --rm -v $(pwd)/validator_keys:/app/validator_keys ethereum/eth2.0-deposit-cli --num_validators= --mnemonic_language=english --folder= +docker run -it --rm -v $(pwd)/validator_keys:/app/validator_keys ethereum/eth2.0-deposit-cli new-mnemonic --num_validators= --mnemonic_language=english --folder= ``` Example for 1 validator on the [Medalla testnet](https://medalla.launchpad.ethereum.org/) using english: ```sh -docker run -it --rm -v $(pwd)/validator_keys:/app/validator_keys ethereum/eth2.0-deposit-cli --num_validators=1 --mnemonic_language=english --chain=medalla +docker run -it --rm -v $(pwd)/validator_keys:/app/validator_keys ethereum/eth2.0-deposit-cli new-mnemonic --num_validators=1 --mnemonic_language=english --chain=medalla ``` ###### Arguments From 741fa62392d2fdf9469ae6acd6e01f024e9659b2 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 3 Nov 2020 12:29:47 +0100 Subject: [PATCH 15/24] Fix CLI index bounds `type=click.IntRange(0, 2**32)` got corrected to `type=click.IntRange(0, 2**32 - 1)` Co-authored-by: Hsiao-Wei Wang --- eth2deposit/cli/existing_mnemonic.py | 2 +- eth2deposit/cli/generate_keys.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index d15a3391..87f30d79 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -48,7 +48,7 @@ def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: default=0, prompt=('Enter the index (key number) you wish to start generating more keys from. ' 'For example, if you\'ve generated 4 keys in the past, you\'d enter 4 here,'), - type=click.IntRange(0, 2**32), + type=click.IntRange(0, 2**32 - 1), ) @generate_keys_arguments_decorator def existing_mnemonic(ctx: click.Context, mnemonic: str, mnemonic_password: str, **kwargs: Any) -> None: diff --git a/eth2deposit/cli/generate_keys.py b/eth2deposit/cli/generate_keys.py index 8b09e866..729a9a10 100644 --- a/eth2deposit/cli/generate_keys.py +++ b/eth2deposit/cli/generate_keys.py @@ -32,7 +32,7 @@ def generate_keys_arguments_decorator(function: Callable[..., Any]) -> Callable[ '--num_validators', prompt='Please choose how many validators you wish to run', required=True, - type=click.IntRange(0, 2**32), + type=click.IntRange(0, 2**32 - 1), ), click.option( '--folder', From 29f00891fe11cf1027f3bcecdaeca33454d3e507 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 3 Nov 2020 12:51:23 +0100 Subject: [PATCH 16/24] Re-adds password length checks from #138 which were accidentally removed in merge 014e34c --- eth2deposit/cli/generate_keys.py | 43 ++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/eth2deposit/cli/generate_keys.py b/eth2deposit/cli/generate_keys.py index 729a9a10..732fefe8 100644 --- a/eth2deposit/cli/generate_keys.py +++ b/eth2deposit/cli/generate_keys.py @@ -9,7 +9,10 @@ CredentialList, ) from eth2deposit.exceptions import ValidationError -from eth2deposit.utils.validation import verify_deposit_data_json +from eth2deposit.utils.validation import ( + verify_deposit_data_json, + validate_password_strength, +) from eth2deposit.utils.constants import ( MAX_DEPOSIT_AMOUNT, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME, @@ -22,6 +25,38 @@ ) +def get_password(text: str) -> str: + return click.prompt(text, hide_input=True, show_default=False, type=str) + + +def validate_password(cts: click.Context, param: Any, password: str) -> str: + is_valid_password = False + + # The given password has passed confirmation + try: + validate_password_strength(password) + except ValidationError as e: + click.echo(f'Error: {e} Please retype.') + else: + is_valid_password = True + + while not is_valid_password: + password = get_password(text='Type the password that secures your validator keystore(s)') + try: + validate_password_strength(password) + except ValidationError as e: + click.echo(f'Error: {e} Please retype.') + else: + # Confirm password + password_confirmation = get_password(text='Repeat for confirmation') + if password == password_confirmation: + is_valid_password = True + else: + click.echo('Error: the two entered values do not match. Please retype again.') + + return password + + def generate_keys_arguments_decorator(function: Callable[..., Any]) -> Callable[..., Any]: ''' This is a decorator that, when applied to a parent-command, implements the @@ -45,7 +80,11 @@ def generate_keys_arguments_decorator(function: Callable[..., Any]) -> Callable[ type=click.Choice(ALL_CHAINS.keys(), case_sensitive=False), default=MAINNET, ), - click.password_option('--keystore_password', prompt='Type the password that secures your validator keystore(s)') + click.password_option( + '--keystore_password', + callback=validate_password, + prompt='Type the password that secures your validator keystore(s)', + ), ] for decorator in reversed(decorators): function = decorator(function) From 07dce72b842f077b20a1d1227fa17d812d2d228e Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 3 Nov 2020 14:51:55 +0100 Subject: [PATCH 17/24] remove reference to "withdrawal keys" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 24fae7f6..a87ce7d1 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The CLI offers different commands depending on what you want to do with the tool | Command | Description | | ------- | ----------- | | `new-mnemonic` | (Recommended) If you don't already have a mnemonic that you have securely backed up, or you want to have a separate mnemonic for your eth2 validators, use this option. | -| `existing-mnemonic` | If you have a mnemonic that you already use, then this option allows you to derive new keys from your existing mnemonic. Use this tool, if you have already generated keys with this CLI before, if you want to reuse your mnemonic that you know is secure that you generated elsewhere (reusing your eth1 mnemonic etc), or if you lost your keystores and need to recover your validator/withdrawal keys. | +| `existing-mnemonic` | If you have a mnemonic that you already use, then this option allows you to derive new keys from your existing mnemonic. Use this tool, if you have already generated keys with this CLI before, if you want to reuse your mnemonic that you know is secure that you generated elsewhere (reusing your eth1 mnemonic etc), or if you lost your keystores and need to recover your validator keys. | ###### `new-mnemonic` Arguments From d3149c688a30f4af33ff0344730757dbda944f1a Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 3 Nov 2020 15:43:18 +0100 Subject: [PATCH 18/24] Update help messages for all CLI arguments and commands --- eth2deposit/cli/existing_mnemonic.py | 10 ++++++++-- eth2deposit/cli/generate_keys.py | 10 ++++++++-- eth2deposit/cli/new_mnemonic.py | 7 +++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 87f30d79..74ca7ea9 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -23,11 +23,15 @@ def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: raise ValidationError('That is not a valid mnemonic, please check for typos.') -@click.command() +@click.command( + help='Generate (or recover) keys from an existing mnemonic', +) @click.pass_context @click.option( '--mnemonic', callback=validate_mnemonic, + help=('The mnemonic that you used to generate your keys. (It is reccomened not to use this argument, and wait for ' + 'the CLI to ask you for your mnemonic as otherwise it will appear in your shell history.)'), prompt='Please enter your mnemonic separated by spaces (" ")', required=True, type=str, @@ -39,13 +43,15 @@ def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: 'passwords. Providing a password here when you didn\'t use one initially, can result in lost keys (and ' 'therefore funds)! Also note that if you used this tool to generate your mnemonic intially, then you did not ' 'use a mnemonic password. However, if you are certain you used a password to "increase" the security of your ' - 'mnemonic, this is where you enter it. '), + 'mnemonic, this is where you enter it.mnemonic'), prompt=False, ) @click.option( '--validator_start_index', confirmation_prompt=True, default=0, + help=('Enter the index (key number) you wish to start generating more keys from. ' + 'For example, if you\'ve generated 4 keys in the past, you\'d enter 4 here,'), prompt=('Enter the index (key number) you wish to start generating more keys from. ' 'For example, if you\'ve generated 4 keys in the past, you\'d enter 4 here,'), type=click.IntRange(0, 2**32 - 1), diff --git a/eth2deposit/cli/generate_keys.py b/eth2deposit/cli/generate_keys.py index 732fefe8..cfb003bd 100644 --- a/eth2deposit/cli/generate_keys.py +++ b/eth2deposit/cli/generate_keys.py @@ -66,23 +66,29 @@ def generate_keys_arguments_decorator(function: Callable[..., Any]) -> Callable[ click.option( '--num_validators', prompt='Please choose how many validators you wish to run', + help='The number of validators keys you want to generate (you can always generate more later)', required=True, type=click.IntRange(0, 2**32 - 1), ), click.option( '--folder', + default=os.getcwd(), + help='The folder to place the generated keystores and deposit_data.json in', type=click.Path(exists=True, file_okay=False, dir_okay=True), - default=os.getcwd() ), click.option( '--chain', + default=MAINNET, + help='The version of eth2 you are targeting. use "mainnet" if you are depositing ETH', prompt='Please choose the (mainnet or testnet) network/chain name', type=click.Choice(ALL_CHAINS.keys(), case_sensitive=False), - default=MAINNET, ), click.password_option( '--keystore_password', callback=validate_password, + help=('The password that will secure your keystores. You will need to re-enter this to decrypt them when ' + 'you setup your eth2 validators. (It is reccomened not to use this argument, and wait for the CLI ' + 'to ask you for your mnemonic as otherwise it will appear in your shell history.)'), prompt='Type the password that secures your validator keystore(s)', ), ] diff --git a/eth2deposit/cli/new_mnemonic.py b/eth2deposit/cli/new_mnemonic.py index 287b19a1..af278c6c 100644 --- a/eth2deposit/cli/new_mnemonic.py +++ b/eth2deposit/cli/new_mnemonic.py @@ -17,13 +17,16 @@ languages = get_languages(WORD_LISTS_PATH) -@click.command() +@click.command( + help='Generate a new mnemonic and keys', +) @click.pass_context @click.option( '--mnemonic_language', + default='english', + help='The language that your mnemonic is in.', prompt='Please choose your mnemonic language', type=click.Choice(languages, case_sensitive=False), - default='english', ) @generate_keys_arguments_decorator def new_mnemonic(ctx: click.Context, mnemonic_language: str, **kwargs: Any) -> None: From 10bc333ee973d6243e9bfb7fb179b29f373ce7e8 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 3 Nov 2020 17:23:12 +0100 Subject: [PATCH 19/24] Apply @hwwhww's README simplifications --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a87ce7d1..6dfb7ca5 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,8 @@ The CLI offers different commands depending on what you want to do with the tool | Command | Description | | ------- | ----------- | -| `new-mnemonic` | (Recommended) If you don't already have a mnemonic that you have securely backed up, or you want to have a separate mnemonic for your eth2 validators, use this option. | -| `existing-mnemonic` | If you have a mnemonic that you already use, then this option allows you to derive new keys from your existing mnemonic. Use this tool, if you have already generated keys with this CLI before, if you want to reuse your mnemonic that you know is secure that you generated elsewhere (reusing your eth1 mnemonic etc), or if you lost your keystores and need to recover your validator keys. | +| `new-mnemonic` | (Recommended) This command is used to generate keystores with a new mnemonic. | +| `existing-mnemonic` | This command is used to re-generate or derive new keys from your existing mnemonic. Use this command, if (i) you have already generated keys with this CLI before, (ii) you want to reuse your mnemonic that you know is secure that you generated elsewhere (reusing your eth1 mnemonic .etc), or (iii) you lost your keystores and need to recover your keys. | ###### `new-mnemonic` Arguments From 6f54965486657b988dcd42fb0a18aefbc2d74066 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 4 Nov 2020 02:01:18 +0800 Subject: [PATCH 20/24] Add test_regeneration.py --- tests/test_cli/helpers.py | 6 ++ tests/test_cli/test_existing_menmonic.py | 7 +- tests/test_cli/test_new_mnemonic.py | 44 +++++++++++-- tests/test_cli/test_regeneration.py | 82 ++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 12 deletions(-) create mode 100644 tests/test_cli/test_regeneration.py diff --git a/tests/test_cli/helpers.py b/tests/test_cli/helpers.py index 9b91d774..c199c752 100644 --- a/tests/test_cli/helpers.py +++ b/tests/test_cli/helpers.py @@ -1,5 +1,6 @@ import os +from eth2deposit.key_handling.keystore import Keystore from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME @@ -13,3 +14,8 @@ def clean_key_folder(my_folder_path: str) -> None: os.remove(os.path.join(validator_keys_folder_path, key_file_name)) os.rmdir(validator_keys_folder_path) os.rmdir(my_folder_path) + + +def get_uuid(key_file: str) -> str: + keystore = Keystore.from_json(key_file) + return keystore.uuid diff --git a/tests/test_cli/test_existing_menmonic.py b/tests/test_cli/test_existing_menmonic.py index 0965b472..70d18bab 100644 --- a/tests/test_cli/test_existing_menmonic.py +++ b/tests/test_cli/test_existing_menmonic.py @@ -7,8 +7,7 @@ from eth2deposit.deposit import cli from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME -from eth2deposit.key_handling.keystore import Keystore -from.helpers import clean_key_folder +from.helpers import clean_key_folder, get_uuid def test_deposit() -> None: @@ -32,10 +31,6 @@ def test_deposit() -> None: validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) - def get_uuid(key_file: str) -> str: - keystore = Keystore.from_json(key_file) - return keystore.uuid - all_uuid = [ get_uuid(validator_keys_folder_path + '/' + key_file) for key_file in key_files diff --git a/tests/test_cli/test_new_mnemonic.py b/tests/test_cli/test_new_mnemonic.py index b7dee4c8..aa9b8847 100644 --- a/tests/test_cli/test_new_mnemonic.py +++ b/tests/test_cli/test_new_mnemonic.py @@ -3,9 +3,45 @@ import pytest +from click.testing import CliRunner +from eth2deposit.cli import new_mnemonic +from eth2deposit.deposit import cli from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME -from eth2deposit.key_handling.keystore import Keystore -from .helpers import clean_key_folder +from .helpers import clean_key_folder, get_uuid + + +def test_new_mnemonic(monkeypatch) -> None: + # monkeypatch get_mnemonic + def mock_get_mnemonic(language, words_path, entropy=None) -> str: + return "fakephrase" + + monkeypatch.setattr(new_mnemonic, "get_mnemonic", mock_get_mnemonic) + + # Prepare folder + my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') + clean_key_folder(my_folder_path) + if not os.path.exists(my_folder_path): + os.mkdir(my_folder_path) + + runner = CliRunner() + inputs = ['english', '1', 'mainnet', 'MyPassword', 'MyPassword', 'fakephrase'] + data = '\n'.join(inputs) + result = runner.invoke(cli, ['new-mnemonic', '--folder', my_folder_path], input=data) + assert result.exit_code == 0 + + # Check files + validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) + _, _, key_files = next(os.walk(validator_keys_folder_path)) + + all_uuid = [ + get_uuid(validator_keys_folder_path + '/' + key_file) + for key_file in key_files + if key_file.startswith('keystore') + ] + assert len(set(all_uuid)) == 1 + + # Clean up + clean_key_folder(my_folder_path) @pytest.mark.asyncio @@ -60,10 +96,6 @@ async def test_script() -> None: validator_keys_folder_path = os.path.join(my_folder_path, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) _, _, key_files = next(os.walk(validator_keys_folder_path)) - def get_uuid(key_file: str) -> str: - keystore = Keystore.from_json(key_file) - return keystore.uuid - all_uuid = [ get_uuid(validator_keys_folder_path + '/' + key_file) for key_file in key_files diff --git a/tests/test_cli/test_regeneration.py b/tests/test_cli/test_regeneration.py new file mode 100644 index 00000000..684e0f5a --- /dev/null +++ b/tests/test_cli/test_regeneration.py @@ -0,0 +1,82 @@ +import json +import os +from pathlib import Path + +from click.testing import CliRunner +from eth2deposit.cli import new_mnemonic +from eth2deposit.deposit import cli +from eth2deposit.utils.constants import DEFAULT_VALIDATOR_KEYS_FOLDER_NAME +from .helpers import clean_key_folder, get_uuid + + +def test_regeneration(monkeypatch) -> None: + # Part 1: new-mnemonic + + # monkeypatch get_mnemonic + mock_mnemonic = "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about" + + def mock_get_mnemonic(language, words_path, entropy=None) -> str: + return mock_mnemonic + + monkeypatch.setattr(new_mnemonic, "get_mnemonic", mock_get_mnemonic) + + # Prepare folder + folder_path_1 = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER_1') + folder_path_2 = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER_2') + clean_key_folder(folder_path_1) + clean_key_folder(folder_path_2) + if not os.path.exists(folder_path_1): + os.mkdir(folder_path_1) + if not os.path.exists(folder_path_2): + os.mkdir(folder_path_2) + + runner = CliRunner() + # Create index 0 and 1 + my_password = "MyPassword" + inputs = ['english', '2', 'mainnet', my_password, my_password, mock_mnemonic] + data = '\n'.join(inputs) + result = runner.invoke(cli, ['new-mnemonic', '--folder', folder_path_1], input=data) + assert result.exit_code == 0 + + # Check files + validator_keys_folder_path_1 = os.path.join(folder_path_1, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) + _, _, files = next(os.walk(validator_keys_folder_path_1)) + part_1_key_files = sorted([key_file for key_file in files if key_file.startswith('keystore')]) + + all_uuid = [get_uuid(validator_keys_folder_path_1 + '/' + key_file) + for key_file in part_1_key_files] + assert len(set(all_uuid)) == 2 + + # Part 2: + runner = CliRunner() + # Create index 1 and 2 + inputs = [ + mock_mnemonic, + '1', '1', '2', 'mainnet', 'MyPassword', 'MyPassword', 'yes'] + data = '\n'.join(inputs) + arguments = ['existing-mnemonic', '--folder', folder_path_2] + result = runner.invoke(cli, arguments, input=data) + + assert result.exit_code == 0 + + # Check files + validator_keys_folder_path_2 = os.path.join(folder_path_2, DEFAULT_VALIDATOR_KEYS_FOLDER_NAME) + _, _, files = next(os.walk(validator_keys_folder_path_2)) + part_2_key_files = sorted([key_file for key_file in files if key_file.startswith('keystore')]) + + all_uuid = [get_uuid(validator_keys_folder_path_2 + '/' + key_file) + for key_file in part_2_key_files] + assert len(set(all_uuid)) == 2 + + # Finally: + # Check the index=1 files have the same pubkey + assert '1_0_0' in part_1_key_files[1] and '1_0_0' in part_2_key_files[0] + with open(Path(validator_keys_folder_path_1 + '/' + part_1_key_files[1])) as f: + keystore_1_1 = json.load(f) + with open(Path(validator_keys_folder_path_2 + '/' + part_2_key_files[0])) as f: + keystore_2_0 = json.load(f) + assert keystore_1_1['pubkey'] == keystore_2_0['pubkey'] + + # Clean up + clean_key_folder(folder_path_1) + clean_key_folder(folder_path_2) From 120882086492a890f8ddccfa1f082abc9304079f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 4 Nov 2020 02:09:23 +0800 Subject: [PATCH 21/24] Rename `test_deposit` test case to `test_existing_mnemonic` --- tests/test_cli/test_existing_menmonic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli/test_existing_menmonic.py b/tests/test_cli/test_existing_menmonic.py index 70d18bab..145bd55a 100644 --- a/tests/test_cli/test_existing_menmonic.py +++ b/tests/test_cli/test_existing_menmonic.py @@ -10,7 +10,7 @@ from.helpers import clean_key_folder, get_uuid -def test_deposit() -> None: +def test_existing_mnemonic() -> None: # Prepare folder my_folder_path = os.path.join(os.getcwd(), 'TESTING_TEMP_FOLDER') clean_key_folder(my_folder_path) From 8c642d68898befc3f8ca3238d6a9d5ee9d63a3c8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 4 Nov 2020 02:21:54 +0800 Subject: [PATCH 22/24] Fix typo --- eth2deposit/cli/existing_mnemonic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2deposit/cli/existing_mnemonic.py b/eth2deposit/cli/existing_mnemonic.py index 74ca7ea9..5fc1316f 100644 --- a/eth2deposit/cli/existing_mnemonic.py +++ b/eth2deposit/cli/existing_mnemonic.py @@ -43,7 +43,7 @@ def validate_mnemonic(cts: click.Context, param: Any, mnemonic: str) -> str: 'passwords. Providing a password here when you didn\'t use one initially, can result in lost keys (and ' 'therefore funds)! Also note that if you used this tool to generate your mnemonic intially, then you did not ' 'use a mnemonic password. However, if you are certain you used a password to "increase" the security of your ' - 'mnemonic, this is where you enter it.mnemonic'), + 'mnemonic, this is where you enter it.'), prompt=False, ) @click.option( From fb498650f31eb31c661f08c77ffcb29c3987c512 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 4 Nov 2020 02:33:08 +0800 Subject: [PATCH 23/24] Update tests/test_cli/test_regeneration.py --- tests/test_cli/test_regeneration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli/test_regeneration.py b/tests/test_cli/test_regeneration.py index 684e0f5a..ca639112 100644 --- a/tests/test_cli/test_regeneration.py +++ b/tests/test_cli/test_regeneration.py @@ -47,7 +47,7 @@ def mock_get_mnemonic(language, words_path, entropy=None) -> str: for key_file in part_1_key_files] assert len(set(all_uuid)) == 2 - # Part 2: + # Part 2: existing-mnemonic runner = CliRunner() # Create index 1 and 2 inputs = [ From eb37e129c03d40a3ed2b5e3e233c140dc71e1a41 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 4 Nov 2020 03:39:55 +0800 Subject: [PATCH 24/24] Apply suggestions from code review Co-authored-by: Carl Beekhuizen --- tests/test_cli/test_regeneration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli/test_regeneration.py b/tests/test_cli/test_regeneration.py index ca639112..fe1da9a3 100644 --- a/tests/test_cli/test_regeneration.py +++ b/tests/test_cli/test_regeneration.py @@ -13,7 +13,7 @@ def test_regeneration(monkeypatch) -> None: # Part 1: new-mnemonic # monkeypatch get_mnemonic - mock_mnemonic = "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about" + mock_mnemonic = "legal winner thank year wave sausage worth useful legal winner thank yellow" def mock_get_mnemonic(language, words_path, entropy=None) -> str: return mock_mnemonic @@ -76,6 +76,7 @@ def mock_get_mnemonic(language, words_path, entropy=None) -> str: with open(Path(validator_keys_folder_path_2 + '/' + part_2_key_files[0])) as f: keystore_2_0 = json.load(f) assert keystore_1_1['pubkey'] == keystore_2_0['pubkey'] + assert keystore_1_1['path'] == keystore_2_0['path'] # Clean up clean_key_folder(folder_path_1)