Keymanager API for the validator client

AllTests-mainnet.md

@@ -109,21 +109,38 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 + parent sanity                                                                              OK
 ```
 OK: 2/2 Fail: 0/2 Skip: 0/2
-## DeleteKeys requests [Preset: mainnet]
+## DeleteKeys requests [Beacon Node] [Preset: mainnet]
 ```diff
-+ Deleting not existing key [Preset: mainnet]                                                OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
++ Deleting not existing key [Beacon Node] [Preset: mainnet]                                  OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
-## DeleteRemoteKeys requests [Preset: mainnet]
+## DeleteKeys requests [Validator Client] [Preset: mainnet]
 ```diff
-+ Deleting existing local key and remote key [Preset: mainnet]                               OK
-+ Deleting not existing key [Preset: mainnet]                                                OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
++ Deleting not existing key [Validator Client] [Preset: mainnet]                             OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
+```
+OK: 4/4 Fail: 0/4 Skip: 0/4
+## DeleteRemoteKeys requests [Beacon Node] [Preset: mainnet]
+```diff
++ Deleting existing local key and remote key [Beacon Node] [Preset: mainnet]                 OK
++ Deleting not existing key [Beacon Node] [Preset: mainnet]                                  OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
+```
+OK: 5/5 Fail: 0/5 Skip: 0/5
+## DeleteRemoteKeys requests [Validator Client] [Preset: mainnet]
+```diff
++ Deleting existing local key and remote key [Validator Client] [Preset: mainnet]            OK
++ Deleting not existing key [Validator Client] [Preset: mainnet]                             OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
 ```
 OK: 5/5 Fail: 0/5 Skip: 0/5
 ## Diverging hardforks
@@ -169,15 +186,26 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 + addExitMessage/getVoluntaryExitMessage                                                     OK
 ```
 OK: 3/3 Fail: 0/3 Skip: 0/3
-## Fee recipient management [Preset: mainnet]
+## Fee recipient management [Beacon Node] [Preset: mainnet]
+```diff
++ Configuring the fee recpient [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
++ Obtaining the fee recpient of a missing validator returns 404 [Beacon Node] [Preset: mainn OK
++ Obtaining the fee recpient of an unconfigured validator returns the suggested default [Bea OK
++ Setting the fee recipient on a missing validator creates a record for it [Beacon Node] [Pr OK
+```
+OK: 7/7 Fail: 0/7 Skip: 0/7
+## Fee recipient management [Validator Client] [Preset: mainnet]
 ```diff
-+ Configuring the fee recpient [Preset: mainnet]                                             OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
-+ Obtaining the fee recpient of a missing validator returns 404 [Preset: mainnet]            OK
-+ Obtaining the fee recpient of an unconfigured validator returns the suggested default [Pre OK
-+ Setting the fee recipient on a missing validator creates a record for it [Preset: mainnet] OK
++ Configuring the fee recpient [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
++ Obtaining the fee recpient of a missing validator returns 404 [Validator Client] [Preset:  OK
++ Obtaining the fee recpient of an unconfigured validator returns the suggested default [Val OK
++ Setting the fee recipient on a missing validator creates a record for it [Validator Client OK
 ```
 OK: 7/7 Fail: 0/7 Skip: 0/7
 ## FinalizedBlocks [Preset: mainnet]
@@ -235,20 +263,36 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 + is_aggregator                                                                              OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
-## ImportKeystores requests [Preset: mainnet]
+## ImportKeystores requests [Beacon Node] [Preset: mainnet]
 ```diff
-+ ImportKeystores/ListKeystores/DeleteKeystores [Preset: mainnet]                            OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
++ ImportKeystores/ListKeystores/DeleteKeystores [Beacon Node] [Preset: mainnet]              OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
-## ImportRemoteKeys/ListRemoteKeys/DeleteRemoteKeys [Preset: mainnet]
+## ImportKeystores requests [Validator Client] [Preset: mainnet]
 ```diff
-+ Importing list of remote keys [Preset: mainnet]                                            OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
++ ImportKeystores/ListKeystores/DeleteKeystores [Validator Client] [Preset: mainnet]         OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
+```
+OK: 4/4 Fail: 0/4 Skip: 0/4
+## ImportRemoteKeys/ListRemoteKeys/DeleteRemoteKeys [Beacon Node] [Preset: mainnet]
+```diff
++ Importing list of remote keys [Beacon Node] [Preset: mainnet]                              OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
+```
+OK: 4/4 Fail: 0/4 Skip: 0/4
+## ImportRemoteKeys/ListRemoteKeys/DeleteRemoteKeys [Validator Client] [Preset: mainnet]
+```diff
++ Importing list of remote keys [Validator Client] [Preset: mainnet]                         OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
 ## Interop
@@ -268,17 +312,20 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 OK: 4/4 Fail: 0/4 Skip: 0/4
 ## KeyStorage testing suite
 ```diff
++ Load Prysm keystore                                                                        OK
 + Pbkdf2 errors                                                                              OK
 + [PBKDF2] Keystore decryption                                                               OK
++ [PBKDF2] Keystore decryption (requireAllFields, allowUnknownFields)                        OK
 + [PBKDF2] Keystore encryption                                                               OK
 + [PBKDF2] Network Keystore decryption                                                       OK
 + [PBKDF2] Network Keystore encryption                                                       OK
 + [SCRYPT] Keystore decryption                                                               OK
++ [SCRYPT] Keystore decryption (requireAllFields, allowUnknownFields)                        OK
 + [SCRYPT] Keystore encryption                                                               OK
 + [SCRYPT] Network Keystore decryption                                                       OK
 + [SCRYPT] Network Keystore encryption                                                       OK
 ```
-OK: 9/9 Fail: 0/9 Skip: 0/9
+OK: 12/12 Fail: 0/12 Skip: 0/12
 ## Light client [Preset: mainnet]
 ```diff
 + Init from checkpoint                                                                       OK
@@ -302,20 +349,36 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 + Sync (Strict) [Preset: mainnet]                                                            OK
 ```
 OK: 12/12 Fail: 0/12 Skip: 0/12
-## ListKeys requests [Preset: mainnet]
+## ListKeys requests [Beacon Node] [Preset: mainnet]
 ```diff
-+ Correct token provided [Preset: mainnet]                                                   OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
++ Correct token provided [Beacon Node] [Preset: mainnet]                                     OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
-## ListRemoteKeys requests [Preset: mainnet]
+## ListKeys requests [Validator Client] [Preset: mainnet]
 ```diff
-+ Correct token provided [Preset: mainnet]                                                   OK
-+ Invalid Authorization Header [Preset: mainnet]                                             OK
-+ Invalid Authorization Token [Preset: mainnet]                                              OK
-+ Missing Authorization header [Preset: mainnet]                                             OK
++ Correct token provided [Validator Client] [Preset: mainnet]                                OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
+```
+OK: 4/4 Fail: 0/4 Skip: 0/4
+## ListRemoteKeys requests [Beacon Node] [Preset: mainnet]
+```diff
++ Correct token provided [Beacon Node] [Preset: mainnet]                                     OK
++ Invalid Authorization Header [Beacon Node] [Preset: mainnet]                               OK
++ Invalid Authorization Token [Beacon Node] [Preset: mainnet]                                OK
++ Missing Authorization header [Beacon Node] [Preset: mainnet]                               OK
+```
+OK: 4/4 Fail: 0/4 Skip: 0/4
+## ListRemoteKeys requests [Validator Client] [Preset: mainnet]
+```diff
++ Correct token provided [Validator Client] [Preset: mainnet]                                OK
++ Invalid Authorization Header [Validator Client] [Preset: mainnet]                          OK
++ Invalid Authorization Token [Validator Client] [Preset: mainnet]                           OK
++ Missing Authorization header [Validator Client] [Preset: mainnet]                          OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
 ## Message signatures
@@ -358,7 +421,12 @@ OK: 12/12 Fail: 0/12 Skip: 0/12
 + vesion 2 single remote                                                                     OK
 ```
 OK: 3/3 Fail: 0/3 Skip: 0/3
-## Serialization/deserialization  [Preset: mainnet]
+## Serialization/deserialization [Beacon Node] [Preset: mainnet]
+```diff
++ Deserialization test vectors                                                               OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
+## Serialization/deserialization [Validator Client] [Preset: mainnet]
 ```diff
 + Deserialization test vectors                                                               OK
 ```
@@ -587,4 +655,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 328/333 Fail: 0/333 Skip: 5/333
+OK: 364/369 Fail: 0/369 Skip: 5/369

beacon_chain/beacon_node.nim

@@ -11,7 +11,7 @@ import
   std/osproc,
 
   # Nimble packages
-  chronos, json_rpc/servers/httpserver, presto,
+  chronos, json_rpc/servers/httpserver, presto, bearssl/rand,
 
   # Local modules
   "."/[beacon_clock, beacon_chain_db, conf, light_client],
@@ -25,7 +25,8 @@ import
   ./spec/eth2_apis/dynamic_fee_recipients,
   ./sync/[optimistic_sync_light_client, sync_manager, request_manager],
   ./validators/[
-    action_tracker, message_router, validator_monitor, validator_pool],
+    action_tracker, message_router, validator_monitor, validator_pool,
+    keystore_management],
   ./rpc/state_ttl_cache
 
 export
@@ -68,8 +69,8 @@ type
     eth1Monitor*: Eth1Monitor
     payloadBuilderRestClient*: RestClientRef
     restServer*: RestServerRef
+    keymanagerHost*: ref KeymanagerHost
     keymanagerServer*: RestServerRef
-    keymanagerToken*: Option[string]
     eventBus*: EventBus
     vcProcess*: Process
     requestManager*: RequestManager
@@ -102,5 +103,8 @@ template findIt*(s: openArray, predicate: untyped): int =
       break
   res
 
+template rng*(node: BeaconNode): ref HmacDrbgContext =
+  node.network.rng
+
 proc currentSlot*(node: BeaconNode): Slot =
   node.beaconClock.now.slotOrZero

beacon_chain/conf.nim

@@ -807,6 +807,30 @@ type
       desc: "A directory containing validator keystore passwords"
       name: "secrets-dir" .}: Option[InputDir]
 
+    restRequestTimeout* {.
+      defaultValue: 0
+      defaultValueDesc: "infinite"
+      desc: "The number of seconds to wait until complete REST request " &
+            "will be received"
+      name: "rest-request-timeout" .}: Natural
+
+    restMaxRequestBodySize* {.
+      defaultValue: 16_384
+      desc: "Maximum size of REST request body (kilobytes)"
+      name: "rest-max-body-size" .}: Natural
+
+    restMaxRequestHeadersSize* {.
+      defaultValue: 64
+      desc: "Maximum size of REST request headers (kilobytes)"
+      name: "rest-max-headers-size" .}: Natural
+
+    # Same option as appears in Lighthouse and Prysm
+    # https://lighthouse-book.sigmaprime.io/suggested-fee-recipient.html
+    # https://github.com/prysmaticlabs/prysm/pull/10312
+    suggestedFeeRecipient* {.
+      desc: "Suggested fee recipient"
+      name: "suggested-fee-recipient" .}: Option[Address]
+
     keymanagerEnabled* {.
       desc: "Enable the REST keymanager API (BETA version)"
       defaultValue: false
@@ -824,6 +848,11 @@ type
       defaultValueDesc: $defaultAdminListenAddressDesc
       name: "keymanager-address" .}: ValidIpAddress
 
+    keymanagerAllowedOrigin* {.
+      desc: "Limit the access to the Keymanager API to a particular hostname " &
+            "(for CORS-enabled clients such as browsers)"
+      name: "keymanager-allow-origin" .}: Option[string]
+
     keymanagerTokenFile* {.
       desc: "A file specifying the authorizition token required for accessing the keymanager API"
       name: "keymanager-token-file" .}: Option[InputFile]
@@ -1204,6 +1233,13 @@ proc loadEth2Network*(
 template loadEth2Network*(config: BeaconNodeConf): Eth2NetworkMetadata =
   loadEth2Network(config.eth2Network)
 
+func defaultFeeRecipient*(conf: AnyConf): Eth1Address =
+  if conf.suggestedFeeRecipient.isSome:
+    conf.suggestedFeeRecipient.get
+  else:
+    # https://github.com/nim-lang/Nim/issues/19802
+    (static(default(Eth1Address)))
+
 proc loadJwtSecret*(
     rng: var HmacDrbgContext,
     dataDir: string,

beacon_chain/filepath.nim

@@ -10,7 +10,6 @@ when (NimMajor, NimMinor) < (1, 4):
 else:
   {.push raises: [].}
 
-
 import chronicles
 import stew/io2
 import spec/keystore

beacon_chain/networking/eth2_network.nim

@@ -296,11 +296,9 @@ declareGauge nbc_gossipsub_good_fanout,
 declareGauge nbc_gossipsub_healthy_fanout,
   "numbers of topics with dHigh fanout"
 
-const delayBuckets = [1.0, 5.0, 10.0, 20.0, 40.0, 60.0]
-
 declareHistogram nbc_resolve_time,
   "Time(s) used while resolving peer information",
-   buckets = delayBuckets
+   buckets = [1.0, 5.0, 10.0, 20.0, 40.0, 60.0]
 
 const
   libp2p_pki_schemes {.strdefine.} = ""