feat(zk_toolbox): added support for setting attester committee define…

…d in a separate file (#2992)

So far only setting committee to the value in genesis was supported
which is too constraining for real world use. I've added support for
specifying the attester committee in a separate yaml file and passing it
to the set-attester-committee command.

.github/workflows/ci-core-reusable.yml

@@ -351,7 +351,7 @@ jobs:
 
       - name: Setup attester committee for the consensus chain
         run: |
-          ci_run zk_inception consensus set-attester-committee --chain consensus &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
+          ci_run zk_inception consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
 
       - name: Run integration tests
         run: |

core/lib/protobuf_config/src/lib.rs

@@ -71,17 +71,16 @@ pub fn read_optional_repr<P: ProtoRepr>(field: &Option<P>) -> Option<P::Type> {
         .flatten()
 }
 
-pub fn decode_yaml_repr<T: ProtoRepr>(
+/// Reads a yaml file.
+pub fn read_yaml_repr<T: ProtoRepr>(
     path: &PathBuf,
     deny_unknown_fields: bool,
 ) -> anyhow::Result<T::Type> {
     let yaml = std::fs::read_to_string(path).with_context(|| path.display().to_string())?;
-    let d = serde_yaml::Deserializer::from_str(&yaml);
-    let this: T = zksync_protobuf::serde::Deserialize {
+    zksync_protobuf::serde::Deserialize {
         deny_unknown_fields,
     }
-    .proto(d)?;
-    this.read()
+    .proto_repr_from_yaml::<T>(&yaml)
 }
 
 pub fn encode_yaml_repr<T: ProtoRepr>(value: &T::Type) -> anyhow::Result<Vec<u8>> {

core/lib/protobuf_config/src/tests.rs

@@ -2,7 +2,7 @@ use std::{path::PathBuf, str::FromStr};
 
 use zksync_protobuf::testonly::{test_encode_all_formats, ReprConv};
 
-use crate::{decode_yaml_repr, proto};
+use crate::{proto, read_yaml_repr};
 
 /// Tests config <-> proto (boilerplate) conversions.
 #[test]
@@ -60,14 +60,11 @@ fn test_encoding() {
 #[test]
 fn verify_file_parsing() {
     let base_path = PathBuf::from_str("../../../etc/env/file_based/").unwrap();
-    decode_yaml_repr::<proto::general::GeneralConfig>(&base_path.join("general.yaml"), true)
-        .unwrap();
+    read_yaml_repr::<proto::general::GeneralConfig>(&base_path.join("general.yaml"), true).unwrap();
     // It's allowed to have unknown fields in wallets, e.g. we keep private key for fee account
-    decode_yaml_repr::<proto::wallets::Wallets>(&base_path.join("wallets.yaml"), false).unwrap();
-    decode_yaml_repr::<proto::genesis::Genesis>(&base_path.join("genesis.yaml"), true).unwrap();
-    decode_yaml_repr::<proto::contracts::Contracts>(&base_path.join("contracts.yaml"), true)
-        .unwrap();
-    decode_yaml_repr::<proto::secrets::Secrets>(&base_path.join("secrets.yaml"), true).unwrap();
-    decode_yaml_repr::<proto::en::ExternalNode>(&base_path.join("external_node.yaml"), true)
-        .unwrap();
+    read_yaml_repr::<proto::wallets::Wallets>(&base_path.join("wallets.yaml"), false).unwrap();
+    read_yaml_repr::<proto::genesis::Genesis>(&base_path.join("genesis.yaml"), true).unwrap();
+    read_yaml_repr::<proto::contracts::Contracts>(&base_path.join("contracts.yaml"), true).unwrap();
+    read_yaml_repr::<proto::secrets::Secrets>(&base_path.join("secrets.yaml"), true).unwrap();
+    read_yaml_repr::<proto::en::ExternalNode>(&base_path.join("external_node.yaml"), true).unwrap();
 }

zk_toolbox/Cargo.toml

@@ -34,7 +34,9 @@ zksync_protobuf_config = { path = "../core/lib/protobuf_config" }
 zksync_basic_types = { path = "../core/lib/basic_types" }
 zksync_consensus_roles = "=0.3.0"
 zksync_consensus_crypto = "=0.3.0"
+zksync_consensus_utils = "=0.3.0"
 zksync_protobuf = "=0.3.0"
+zksync_protobuf_build = "=0.3.0"
 
 # External dependencies
 anyhow = "1.0.82"
@@ -49,6 +51,7 @@ futures = "0.3.30"
 human-panic = "2.0"
 lazy_static = "1.4.0"
 once_cell = "1.19.0"
+prost = "0.12.1"
 rand = "0.8.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

zk_toolbox/crates/config/src/consensus_secrets.rs

@@ -2,13 +2,13 @@ use std::path::Path;
 
 use xshell::Shell;
 use zksync_config::configs::consensus::ConsensusSecrets;
-use zksync_protobuf_config::decode_yaml_repr;
+use zksync_protobuf_config::read_yaml_repr;
 
 use crate::traits::ReadConfig;
 
 impl ReadConfig for ConsensusSecrets {
     fn read(shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<Self> {
         let path = shell.current_dir().join(path);
-        decode_yaml_repr::<zksync_protobuf_config::proto::secrets::ConsensusSecrets>(&path, false)
+        read_yaml_repr::<zksync_protobuf_config::proto::secrets::ConsensusSecrets>(&path, false)
     }
 }

zk_toolbox/crates/config/src/ecosystem.rs

@@ -146,20 +146,20 @@ impl EcosystemConfig {
             .unwrap_or(self.default_chain.as_ref())
     }
 
-    pub fn load_chain(&self, name: Option<String>) -> Option<ChainConfig> {
+    pub fn load_chain(&self, name: Option<String>) -> anyhow::Result<ChainConfig> {
         let name = name.unwrap_or(self.default_chain.clone());
         self.load_chain_inner(&name)
     }
 
-    pub fn load_current_chain(&self) -> Option<ChainConfig> {
+    pub fn load_current_chain(&self) -> anyhow::Result<ChainConfig> {
         self.load_chain_inner(self.current_chain())
     }
 
-    fn load_chain_inner(&self, name: &str) -> Option<ChainConfig> {
+    fn load_chain_inner(&self, name: &str) -> anyhow::Result<ChainConfig> {
         let path = self.chains.join(name).join(CONFIG_NAME);
-        let config = ChainConfigInternal::read(self.get_shell(), path.clone()).ok()?;
+        let config = ChainConfigInternal::read(self.get_shell(), path.clone())?;
 
-        Some(ChainConfig {
+        Ok(ChainConfig {
             id: config.id,
             name: config.name,
             chain_id: config.chain_id,

zk_toolbox/crates/config/src/external_node.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use xshell::Shell;
 pub use zksync_config::configs::en_config::ENConfig;
-use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr};
+use zksync_protobuf_config::{encode_yaml_repr, read_yaml_repr};
 
 use crate::{
     consts::EN_CONFIG_FILE,
@@ -23,6 +23,6 @@ impl SaveConfig for ENConfig {
 impl ReadConfig for ENConfig {
     fn read(shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<Self> {
         let path = shell.current_dir().join(path);
-        decode_yaml_repr::<zksync_protobuf_config::proto::en::ExternalNode>(&path, false)
+        read_yaml_repr::<zksync_protobuf_config::proto::en::ExternalNode>(&path, false)
     }
 }

zk_toolbox/crates/config/src/general.rs

@@ -6,7 +6,7 @@ use url::Url;
 use xshell::Shell;
 use zksync_config::configs::object_store::ObjectStoreMode;
 pub use zksync_config::configs::GeneralConfig;
-use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr};
+use zksync_protobuf_config::{encode_yaml_repr, read_yaml_repr};
 
 use crate::{
     consts::GENERAL_FILE,
@@ -137,7 +137,7 @@ impl SaveConfig for GeneralConfig {
 impl ReadConfig for GeneralConfig {
     fn read(shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<Self> {
         let path = shell.current_dir().join(path);
-        decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&path, false)
+        read_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&path, false)
     }
 }
 

zk_toolbox/crates/config/src/genesis.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 use xshell::Shell;
 use zksync_basic_types::L1ChainId;
 pub use zksync_config::GenesisConfig;
-use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr};
+use zksync_protobuf_config::{encode_yaml_repr, read_yaml_repr};
 
 use crate::{
     consts::GENESIS_FILE,
@@ -32,6 +32,6 @@ impl SaveConfig for GenesisConfig {
 impl ReadConfig for GenesisConfig {
     fn read(shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<Self> {
         let path = shell.current_dir().join(path);
-        decode_yaml_repr::<zksync_protobuf_config::proto::genesis::Genesis>(&path, false)
+        read_yaml_repr::<zksync_protobuf_config::proto::genesis::Genesis>(&path, false)
     }
 }

zk_toolbox/crates/config/src/lib.rs

@@ -10,7 +10,7 @@ pub use manipulations::*;
 pub use secrets::*;
 pub use wallet_creation::*;
 pub use wallets::*;
-pub use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr};
+pub use zksync_protobuf_config::{encode_yaml_repr, read_yaml_repr};
 
 mod apps;
 mod chain;

zk_toolbox/crates/config/src/secrets.rs

@@ -5,7 +5,7 @@ use common::db::DatabaseConfig;
 use xshell::Shell;
 use zksync_basic_types::url::SensitiveUrl;
 pub use zksync_config::configs::Secrets as SecretsConfig;
-use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr};
+use zksync_protobuf_config::{encode_yaml_repr, read_yaml_repr};
 
 use crate::{
     consts::SECRETS_FILE,
@@ -59,6 +59,6 @@ impl SaveConfig for SecretsConfig {
 impl ReadConfig for SecretsConfig {
     fn read(shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<Self> {
         let path = shell.current_dir().join(path);
-        decode_yaml_repr::<zksync_protobuf_config::proto::secrets::Secrets>(&path, false)
+        read_yaml_repr::<zksync_protobuf_config::proto::secrets::Secrets>(&path, false)
     }
 }

zk_toolbox/crates/zk_inception/Cargo.toml

@@ -35,8 +35,16 @@ zksync_basic_types.workspace = true
 clap-markdown.workspace = true
 zksync_consensus_roles.workspace = true
 zksync_consensus_crypto.workspace = true
+zksync_protobuf.workspace = true
+zksync_protobuf_config.workspace = true
+prost.workspace = true
 secrecy.workspace = true
 
+[dev-dependencies]
+rand.workspace = true
+zksync_consensus_utils.workspace = true
+
 [build-dependencies]
 eyre.workspace = true
 ethers.workspace = true
+zksync_protobuf_build.workspace = true

zk_toolbox/crates/zk_inception/README.md

@@ -17,8 +17,12 @@ This document contains the help content for the `zk_inception` command-line prog
 - [`zk_inception chain initialize-bridges`↴](#zk_inception-chain-initialize-bridges)
 - [`zk_inception chain deploy-l2-contracts`↴](#zk_inception-chain-deploy-l2-contracts)
 - [`zk_inception chain upgrader`↴](#zk_inception-chain-upgrader)
+- [`zk_inception chain deploy-consensus-registry`↴](#zk_inception-chain-deploy-consensus-registry)
+- [`zk_inception chain deploy-multicall3`↴](#zk_inception-chain-deploy-multicall3)
 - [`zk_inception chain deploy-paymaster`↴](#zk_inception-chain-deploy-paymaster)
 - [`zk_inception chain update-token-multiplier-setter`↴](#zk_inception-chain-update-token-multiplier-setter)
+- [`zk_inception consensus set-attester-committee`↴](#zk_inception-consensus-set-attester-committee)
+- [`zk_inception consensus get-attester-committee`↴](#zk_inception-consensus-get-attester-committee)
 - [`zk_inception prover`↴](#zk_inception-prover)
 - [`zk_inception prover init`↴](#zk_inception-prover-init)
 - [`zk_inception prover setup-keys`↴](#zk_inception-prover-setup-keys)
@@ -364,6 +368,18 @@ Deploy Default Upgrader
 
   e.g.: `zk_inception init -a --private-key=<PRIVATE_KEY>`
 
+## `zk_inception chain deploy-consensus-registry`
+
+Deploy Consensus Registry smart contract
+
+**Usage:** `zk_inception chain deploy-consensus-registry`
+
+## `zk_inception chain deploy-multicall3`
+
+Deploy Multicall3 smart contract
+
+**Usage:** `zk_inception chain deploy-multicall3`
+
 ## `zk_inception chain deploy-paymaster`
 
 Deploy paymaster smart contract
@@ -414,6 +430,47 @@ Update Token Multiplier Setter address on L1
 
   e.g.: `zk_inception init -a --private-key=<PRIVATE_KEY>`
 
+## `zk_inception consensus`
+
+Consensus related commands
+
+**Usage:** `zk_inception consensus <COMMAND>`
+
+###### **Subcommands:**
+
+- `set-attester-committee` — Set attester committee
+- `get-attester-committee` — Get attester committee
+
+## `zk_inception consensus set-attester-committee`
+
+Set attester committee in the consensus registry smart contract. Requires `consensus_registry` and `multicall3`
+contracts to be deployed.
+
+**Usage:** `zk_inception consensus set-attester-committee [OPTIONS]`
+
+###### **Options:**
+
+- `--from-genesis` — Set attester committee to `consensus.genesis_spec.attesters` in general.yaml Mutually exclusive
+  with `--from-file`.
+- `--from-file <PATH>` — Set attester committee to committee specified in yaml file at `PATH`.  
+  Mutually exclusive with `--from-genesis`. File format is specified in
+  `zk_inception/src/commands/consensus/proto/mod.proto`. Example:
+
+  ```yaml
+  attesters:
+    - key: attester:public:secp256k1:0339d4b0cdd9896d3929631a4e5e9a5b4919f52592bec571d70bb0e50a3a824714
+      weight: 1
+    - key: attester:public:secp256k1:024897d8c10d7a57d108cfe2a724d7824c657f219ef5d9f7674810a6746c19fa7b
+      weight: 1
+  ```
+
+## `zk_inception consensus get-attester-committee`
+
+Requires `consensus_registry` and `multicall3` contracts to be deployed. Fetches attester committee from the consensus
+registry contract and prints it.
+
+**Usage:** `zk_inception consensus get-attester-committee`
+
 ## `zk_inception prover`
 
 Prover related commands

zk_toolbox/crates/zk_inception/build.rs

@@ -7,5 +7,15 @@ fn main() -> eyre::Result<()> {
     Abigen::new("ConsensusRegistry", "abi/ConsensusRegistry.json")?
         .generate()?
         .write_to_file(outdir.join("consensus_registry_abi.rs"))?;
+
+    zksync_protobuf_build::Config {
+        input_root: "src/commands/consensus/proto".into(),
+        proto_root: "zksync/toolbox/consensus".into(),
+        dependencies: vec!["::zksync_protobuf_config::proto".parse().unwrap()],
+        protobuf_crate: "::zksync_protobuf".parse().unwrap(),
+        is_public: false,
+    }
+    .generate()
+    .unwrap();
     Ok(())
 }