diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index d5019d978..a21420ed6 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "3b6f4e6a48e1285d47063867a49f78b534b79328b33a12ee0397984b3b9fca0c", + "checksum": "55992251f00c2300cc56994973d1ed4168f2ba18d456153ffd49c49e49f9a6db", "crates": { "actix-codec 0.5.1": { "name": "actix-codec", @@ -10805,6 +10805,10 @@ "id": "ic-canister-client 0.9.0", "target": "ic_canister_client" }, + { + "id": "ic-interfaces-registry 0.9.0", + "target": "ic_interfaces_registry" + }, { "id": "ic-nns-constants 0.9.0", "target": "ic_nns_constants" @@ -10813,6 +10817,14 @@ "id": "ic-nns-governance 0.9.0", "target": "ic_nns_governance" }, + { + "id": "ic-protobuf 0.9.0", + "target": "ic_protobuf" + }, + { + "id": "ic-registry-local-registry 0.9.0", + "target": "ic_registry_local_registry" + }, { "id": "ic-sys 0.9.0", "target": "ic_sys" @@ -10853,6 +10865,10 @@ "id": "sha2 0.10.8", "target": "sha2" }, + { + "id": "shellexpand 3.1.0", + "target": "shellexpand" + }, { "id": "socket2 0.5.5", "target": "socket2" @@ -10873,6 +10889,10 @@ "id": "tabular 0.2.0", "target": "tabular" }, + { + "id": "tempdir 0.3.7", + "target": "tempdir" + }, { "id": "tokio 1.35.1", "target": "tokio" @@ -10880,6 +10900,10 @@ { "id": "url 2.5.0", "target": "url" + }, + { + "id": "uuid 1.7.0", + "target": "uuid" } ], "selects": {} @@ -12565,6 +12589,36 @@ }, "license": "MIT/Apache-2.0" }, + "fuchsia-cprng 0.1.1": { + "name": "fuchsia-cprng", + "version": "0.1.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/fuchsia-cprng/0.1.1/download", + "sha256": "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + } + }, + "targets": [ + { + "Library": { + "crate_name": "fuchsia_cprng", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "fuchsia_cprng", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.1.1" + }, + "license": null + }, "funty 2.0.0": { "name": "funty", "version": "2.0.0", @@ -34008,6 +34062,77 @@ }, "license": "MIT" }, + "rand 0.4.6": { + "name": "rand", + "version": "0.4.6", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/rand/0.4.6/download", + "sha256": "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rand", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rand", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "libc", + "std" + ], + "selects": {} + }, + "deps": { + "common": [], + "selects": { + "cfg(target_env = \"sgx\")": [ + { + "id": "rand_core 0.3.1", + "target": "rand_core" + }, + { + "id": "rdrand 0.4.0", + "target": "rdrand" + } + ], + "cfg(target_os = \"fuchsia\")": [ + { + "id": "fuchsia-cprng 0.1.1", + "target": "fuchsia_cprng" + } + ], + "cfg(unix)": [ + { + "id": "libc 0.2.152", + "target": "libc" + } + ], + "cfg(windows)": [ + { + "id": "winapi 0.3.9", + "target": "winapi" + } + ] + } + }, + "edition": "2015", + "version": "0.4.6" + }, + "license": "MIT/Apache-2.0" + }, "rand 0.7.3": { "name": "rand", "version": "0.7.3", @@ -34244,6 +34369,75 @@ }, "license": "MIT OR Apache-2.0" }, + "rand_core 0.3.1": { + "name": "rand_core", + "version": "0.3.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/rand_core/0.3.1/download", + "sha256": "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rand_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rand_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "rand_core 0.4.2", + "target": "rand_core" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.3.1" + }, + "license": "MIT/Apache-2.0" + }, + "rand_core 0.4.2": { + "name": "rand_core", + "version": "0.4.2", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/rand_core/0.4.2/download", + "sha256": "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rand_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rand_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.4.2" + }, + "license": "MIT/Apache-2.0" + }, "rand_core 0.5.1": { "name": "rand_core", "version": "0.5.1", @@ -34446,6 +34640,45 @@ }, "license": "MIT/Apache-2.0" }, + "rdrand 0.4.0": { + "name": "rdrand", + "version": "0.4.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/rdrand/0.4.0/download", + "sha256": "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rdrand", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rdrand", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "rand_core 0.3.1", + "target": "rand_core" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.4.0" + }, + "license": "ISC" + }, "redox_syscall 0.4.1": { "name": "redox_syscall", "version": "0.4.1", @@ -35068,6 +35301,47 @@ }, "license": null }, + "remove_dir_all 0.5.3": { + "name": "remove_dir_all", + "version": "0.5.3", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/remove_dir_all/0.5.3/download", + "sha256": "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" + } + }, + "targets": [ + { + "Library": { + "crate_name": "remove_dir_all", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "remove_dir_all", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(windows)": [ + { + "id": "winapi 0.3.9", + "target": "winapi" + } + ] + } + }, + "edition": "2015", + "version": "0.5.3" + }, + "license": "MIT/Apache-2.0" + }, "rend 0.4.1": { "name": "rend", "version": "0.4.1", @@ -38324,6 +38598,54 @@ }, "license": "MIT/Apache-2.0" }, + "shellexpand 3.1.0": { + "name": "shellexpand", + "version": "3.1.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/shellexpand/3.1.0/download", + "sha256": "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "shellexpand", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "shellexpand", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "base-0", + "default", + "dirs", + "tilde" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "dirs 5.0.1", + "target": "dirs" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "3.1.0" + }, + "license": "MIT/Apache-2.0" + }, "signal-hook-registry 1.4.1": { "name": "signal-hook-registry", "version": "1.4.1", @@ -40634,6 +40956,49 @@ }, "license": "MIT" }, + "tempdir 0.3.7": { + "name": "tempdir", + "version": "0.3.7", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/tempdir/0.3.7/download", + "sha256": "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "tempdir", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "tempdir", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "rand 0.4.6", + "target": "rand" + }, + { + "id": "remove_dir_all 0.5.3", + "target": "remove_dir_all" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.3.7" + }, + "license": "MIT/Apache-2.0" + }, "tempfile 3.9.0": { "name": "tempfile", "version": "3.9.0", @@ -44874,6 +45239,7 @@ "objbase", "processenv", "processthreadsapi", + "profileapi", "shlobj", "std", "sysinfoapi", @@ -48700,6 +49066,7 @@ "wasm32-unknown-unknown", "wasm32-wasi" ], + "cfg(target_env = \"sgx\")": [], "cfg(target_family = \"unix\")": [ "aarch64-apple-darwin", "aarch64-apple-ios", @@ -48735,6 +49102,10 @@ "i686-unknown-freebsd", "x86_64-unknown-freebsd" ], + "cfg(target_os = \"fuchsia\")": [ + "aarch64-fuchsia", + "x86_64-fuchsia" + ], "cfg(target_os = \"haiku\")": [], "cfg(target_os = \"hermit\")": [], "cfg(target_os = \"ios\")": [ diff --git a/Cargo.lock b/Cargo.lock index 541a9c97a..3338d6b08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2156,10 +2156,13 @@ dependencies = [ "ic-base-types", "ic-canister-client", "ic-canisters", + "ic-interfaces-registry", "ic-management-backend", "ic-management-types", "ic-nns-constants", "ic-nns-governance", + "ic-protobuf", + "ic-registry-local-registry", "ic-sys", "itertools 0.12.0", "keyring", @@ -2170,14 +2173,17 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.8", + "shellexpand", "socket2 0.5.5", "spinners", "strum 0.25.0", "tabled", "tabular", + "tempdir", "tempfile", "tokio", "url", + "uuid", "wiremock", ] @@ -2507,6 +2513,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "2.0.0" @@ -6771,6 +6783,19 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.7.3" @@ -6815,6 +6840,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -6857,6 +6897,15 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "977b1e897f9d764566891689e642653e5ed90c6895106acd005eb4c1d0203991" +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -6970,6 +7019,15 @@ dependencies = [ "url", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rend" version = "0.4.1" @@ -7589,6 +7647,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shellexpand" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -8027,6 +8094,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.9.0" diff --git a/rs/cli/Cargo.toml b/rs/cli/Cargo.toml index fbd73313f..e6ac745c7 100644 --- a/rs/cli/Cargo.toml +++ b/rs/cli/Cargo.toml @@ -48,6 +48,12 @@ tabled = { workspace = true } tabular = { workspace = true } tokio = { workspace = true } url = { workspace = true } +tempdir = "0.3" +uuid = "1.7.0" +shellexpand = "3.1.0" +ic-registry-local-registry = { workspace = true } +ic-protobuf = { workspace = true } +ic-interfaces-registry = { workspace = true } [dev-dependencies] tempfile = "3.3.0" diff --git a/rs/cli/src/cli.rs b/rs/cli/src/cli.rs index 528dcc390..643e15971 100644 --- a/rs/cli/src/cli.rs +++ b/rs/cli/src/cli.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use clap::{Parser, Subcommand}; use clap_num::maybe_hex; use ic_base_types::PrincipalId; @@ -120,6 +122,16 @@ pub(crate) enum Commands { /// all subnets subnet_ids: Vec, }, + + /// Dump registry + DumpRegistry { + /// Version to dump. If value is less than 0 will dump the latest version + #[clap(long, default_value = "-1")] + version: i64, + /// Optional path to cached registry + #[clap(long, env = "LOCAL_REGISTRY_PATH")] + path: Option, + }, } pub(crate) mod subnet { diff --git a/rs/cli/src/main.rs b/rs/cli/src/main.rs index be81c2875..60fef14be 100644 --- a/rs/cli/src/main.rs +++ b/rs/cli/src/main.rs @@ -20,6 +20,7 @@ mod detect_neuron; mod general; mod ic_admin; mod ops_subnet_node_replace; +mod registry_dump; mod runner; const STAGING_NEURON_ID: u64 = 49; @@ -277,6 +278,10 @@ async fn main() -> Result<(), anyhow::Error> { Some(neuron) => neuron, None => return Err(anyhow::anyhow!("Neuron required for this command")), }, cli.get_nns_url()).await + }, + + cli::Commands::DumpRegistry { version, path } => { + registry_dump::dump_registry(path, version, cli_opts.network).await } } }) diff --git a/rs/cli/src/registry_dump.rs b/rs/cli/src/registry_dump.rs new file mode 100644 index 000000000..a110655a6 --- /dev/null +++ b/rs/cli/src/registry_dump.rs @@ -0,0 +1,74 @@ +use std::{path::PathBuf, str::FromStr, time::Duration}; + +use anyhow::Error; +use ic_base_types::RegistryVersion; +use ic_interfaces_registry::RegistryClient; +use ic_management_backend::registry::{local_registry_path, sync_local_store, RegistryFamilyEntries}; +use ic_management_types::Network; +use ic_protobuf::registry::{ + dc::v1::DataCenterRecord, node::v1::NodeRecord, node_operator::v1::NodeOperatorRecord, subnet::v1::SubnetRecord, +}; +use ic_registry_local_registry::LocalRegistry; +use uuid::Uuid; + +pub async fn dump_registry(path: &Option, version: &i64, network: Network) -> Result<(), Error> { + let (path, should_dispose) = match path { + Some(p) => (p.clone(), false), + None => { + let uuid = Uuid::new_v4(); + let binding = format!("~/tmp/{}", uuid); + let local_temp = shellexpand::tilde(&binding); + ( + PathBuf::from_str(&local_temp).map_err(|e| anyhow::anyhow!("Couldn't create path: {:?}", e))?, + true, + ) + } + }; + + std::env::set_var("LOCAL_REGISTRY_PATH", path.clone()); + + sync_local_store(network.clone()).await?; + + let local_registry = LocalRegistry::new(local_registry_path(network), Duration::from_secs(10)) + .map_err(|e| anyhow::anyhow!("Couldn't create local registry client instance: {:?}", e))?; + + // determine desired version + let version = { + if *version >= 0 { + RegistryVersion::new(*version as u64) + } else { + local_registry.get_latest_version() + } + }; + + let nodes = local_registry + .get_family_entries_of_version::(version) + .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; + let nodes = serde_json::to_string(&nodes).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + + let subnets = local_registry + .get_family_entries_of_version::(version) + .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; + let subnets = serde_json::to_string(&subnets).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + + let dcs = local_registry + .get_family_entries_of_version::(version) + .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; + let dcs = serde_json::to_string(&dcs).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + + let node_operators = local_registry + .get_family_entries_of_version::(version) + .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; + let node_operators = + serde_json::to_string(&node_operators).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + + println!( + "{{ \"nodes\": {}, \"subnets\": {}, \"dcs\": {}, \"node_operators\": {} }}", + nodes, subnets, dcs, node_operators + ); + + if should_dispose { + std::fs::remove_dir_all(path).map_err(|e| anyhow::anyhow!("Error removing created dir: {:?}", e))? + } + Ok(()) +} diff --git a/rs/ic-management-backend/src/registry.rs b/rs/ic-management-backend/src/registry.rs index 381a653da..4b0511881 100644 --- a/rs/ic-management-backend/src/registry.rs +++ b/rs/ic-management-backend/src/registry.rs @@ -82,7 +82,7 @@ pub struct RegistryState { hostos_releases: ArtifactReleases, ic_repo: Option, } -trait RegistryEntry: RegistryValue { +pub trait RegistryEntry: RegistryValue { const KEY_PREFIX: &'static str; } @@ -102,9 +102,13 @@ impl RegistryEntry for SubnetRecord { const KEY_PREFIX: &'static str = SUBNET_RECORD_KEY_PREFIX; } -trait RegistryFamilyEntries { +pub trait RegistryFamilyEntries { fn get_family_entries(&self) -> Result>; fn get_family_entries_versioned(&self) -> Result>; + fn get_family_entries_of_version( + &self, + version: RegistryVersion, + ) -> Result>; } impl RegistryFamilyEntries for LocalRegistry { @@ -127,12 +131,19 @@ impl RegistryFamilyEntries for LocalRegistry { } fn get_family_entries_versioned(&self) -> Result> { + self.get_family_entries_of_version(self.get_latest_version()) + } + + fn get_family_entries_of_version( + &self, + version: RegistryVersion, + ) -> Result> { let prefix_length = T::KEY_PREFIX.len(); Ok(self - .get_key_family(T::KEY_PREFIX, self.get_latest_version())? + .get_key_family(T::KEY_PREFIX, version)? .iter() .filter_map(|key| { - self.get_versioned_value(key, self.get_latest_version()) + self.get_versioned_value(key, version) .map(|r| { r.value.as_ref().map(|v| { (