From 1dd4984fdcde951df08ea1ba427ad21ef641ee40 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 4 Mar 2024 16:15:35 +0100 Subject: [PATCH 01/26] started work on calculating progress --- rs/rollout-controller/src/main.rs | 4 +- rs/rollout-controller/src/rollout_schedule.rs | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 rs/rollout-controller/src/rollout_schedule.rs diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index e77c629d7..87f74fade 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -11,6 +11,7 @@ use crate::{git_sync::sync_git, registry_wrappers::sync_wrap}; mod git_sync; mod registry_wrappers; +mod rollout_schedule; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -42,7 +43,6 @@ async fn main() -> anyhow::Result<()> { } should_sleep = true; - // Sync registry info!(logger, "Syncing registry for network '{:?}'", args.network); match sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()).await { Ok(()) => info!(logger, "Syncing registry completed"), @@ -63,6 +63,8 @@ async fn main() -> anyhow::Result<()> { } } + info!(logger, "Checking the progress of current release"); + // Read prometheus // Read last iteration from disk diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs new file mode 100644 index 000000000..14dbabca9 --- /dev/null +++ b/rs/rollout-controller/src/rollout_schedule.rs @@ -0,0 +1,50 @@ +use std::path::PathBuf; + +use slog::Logger; + +pub async fn calculate_progress(logger: &Logger, release_index: &PathBuf) -> anyhow::Result<()> { + // Deserialize the index + + // Check if the desired rollout version is elected + + // Iterate over stages and compare desired state from file and the state it is live + // Take into consideration: + // 0. Check if the plan is paused. If it is do nothing + // 1. If the subnet is on the desired version, proceed + // 2. If the subnet isn't on the desired version: + // 2.1. Check if there is an open proposal for upgrading, if there isn't place one + // 2.2. If the proposal is executed check when it was upgraded and query prometheus + // to see if there were any alerts and if the bake time has passed. If the bake + // time didn't pass don't do anything. If there were alerts don't do anything. +} + +pub struct Index { + pub rollout: Rollout, + pub features: Vec, + pub releases: Vec, +} + +pub struct Rollout { + pub rc_name: String, + pub pause: bool, + pub skip_days: Vec, + pub stages: Vec, +} + +pub struct Stage { + pub subnets: Vec, + pub bake_time: String, + pub wait_for_next_week: bool, +} + +pub struct Feature { + pub name: String, + pub subnets: Vec, +} + +pub struct Release { + pub name: String, + pub version: String, + pub publish: String, + pub features: Vec, +} From 17092252e365612845e369282eaba4c75f46a6b6 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Tue, 5 Mar 2024 11:16:19 +0100 Subject: [PATCH 02/26] saving prog --- Cargo.lock | 7 + rs/ic-management-backend/src/registry.rs | 4 +- rs/ic-management-types/src/lib.rs | 2 +- rs/rollout-controller/Cargo.toml | 7 + rs/rollout-controller/src/main.rs | 48 +++-- rs/rollout-controller/src/rollout_schedule.rs | 184 ++++++++++++++++-- 6 files changed, 218 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afea731de..d917aa7b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7377,11 +7377,18 @@ name = "rollout-controller" version = "0.2.1" dependencies = [ "anyhow", + "chrono", "clap 4.4.18", "crossbeam", "crossbeam-channel", "humantime", + "ic-management-backend", "ic-management-types", + "ic-registry-keys", + "ic-registry-local-registry", + "serde", + "serde_json", + "serde_yaml 0.9.32", "service-discovery", "slog", "slog-async", diff --git a/rs/ic-management-backend/src/registry.rs b/rs/ic-management-backend/src/registry.rs index 4b0511881..544a4ba0b 100644 --- a/rs/ic-management-backend/src/registry.rs +++ b/rs/ic-management-backend/src/registry.rs @@ -1,4 +1,4 @@ -use crate::config::{get_nns_url_string_from_target_network, get_nns_url_vec_from_target_network}; +use crate::config::get_nns_url_vec_from_target_network; use crate::factsdb; use crate::git_ic_repo::IcRepo; use crate::proposal; @@ -195,7 +195,7 @@ impl ReleasesOps for ArtifactReleases { impl RegistryState { pub async fn new(network: Network, without_update_loop: bool) -> Self { - let nns_url = get_nns_url_string_from_target_network(&network); + let nns_url = network.get_url().to_string(); sync_local_store(network.clone()) .await diff --git a/rs/ic-management-types/src/lib.rs b/rs/ic-management-types/src/lib.rs index 7db159723..e47d3cb3d 100644 --- a/rs/ic-management-types/src/lib.rs +++ b/rs/ic-management-types/src/lib.rs @@ -587,7 +587,7 @@ impl Network { match self { Network::Mainnet => Url::from_str("https://ic0.app").unwrap(), // Workaround for staging boundary node not working properly (503 Service unavailable) - Network::Staging => Url::from_str("https://[2600:3000:6100:200:5000:b0ff:fe8e:6b7b]:8080").unwrap(), + Network::Staging => Url::from_str("http://[2600:3000:6100:200:5000:b0ff:fe8e:6b7b]:8080").unwrap(), Self::Url(url) => url.clone(), } } diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index 879fede1e..05164e1f2 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -21,3 +21,10 @@ crossbeam = { workspace = true } crossbeam-channel = { workspace = true } humantime = { workspace = true } tokio-util = "0.6.10" +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml = "0.9.32" +ic-registry-keys = { workspace = true } +ic-registry-local-registry = { workspace = true } +ic-management-backend = { workspace = true } +chrono = { version = "0.4", features = [ "serde" ] } diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index 87f74fade..53bb1e3d6 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -7,7 +7,7 @@ use slog::{info, o, warn, Drain, Level, Logger}; use tokio::select; use tokio_util::sync::CancellationToken; -use crate::{git_sync::sync_git, registry_wrappers::sync_wrap}; +use crate::{git_sync::sync_git, registry_wrappers::sync_wrap, rollout_schedule::calculate_progress}; mod git_sync; mod registry_wrappers; @@ -43,28 +43,42 @@ async fn main() -> anyhow::Result<()> { } should_sleep = true; - info!(logger, "Syncing registry for network '{:?}'", args.network); - match sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()).await { - Ok(()) => info!(logger, "Syncing registry completed"), + // info!(logger, "Syncing registry for network '{:?}'", args.network); + // match sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()).await { + // Ok(()) => info!(logger, "Syncing registry completed"), + // Err(e) => { + // warn!(logger, "{:?}", e); + // should_sleep = false; + // continue; + // } + // }; + + // info!(logger, "Syncing git repo"); + // match sync_git(&logger, &args.git_repo_path, &args.git_repo_url, &args.release_index).await { + // Ok(()) => info!(logger, "Syncing git repo completed"), + // Err(e) => { + // warn!(logger, "{:?}", e); + // should_sleep = false; + // continue; + // } + // } + + info!(logger, "Calculating the progress of the current release"); + match calculate_progress( + &logger, + &args.git_repo_path.join(&args.release_index), + &args.network, + token.clone(), + ) + .await + { + Ok(()) => info!(logger, "Calculating completed"), Err(e) => { warn!(logger, "{:?}", e); - should_sleep = false; - continue; - } - }; - - info!(logger, "Syncing git repo"); - match sync_git(&logger, &args.git_repo_path, &args.git_repo_url, &args.release_index).await { - Ok(()) => info!(logger, "Syncing git repo completed"), - Err(e) => { - warn!(logger, "{:?}", e); - should_sleep = false; continue; } } - info!(logger, "Checking the progress of current release"); - // Read prometheus // Read last iteration from disk diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs index 14dbabca9..b3952679d 100644 --- a/rs/rollout-controller/src/rollout_schedule.rs +++ b/rs/rollout-controller/src/rollout_schedule.rs @@ -1,50 +1,206 @@ -use std::path::PathBuf; +use std::{collections::BTreeMap, path::PathBuf}; -use slog::Logger; +use anyhow::Ok; +use chrono::{Local, NaiveDate}; +use ic_management_backend::registry::RegistryState; +use ic_management_types::Network; +use serde::Deserialize; +use slog::{debug, info, Logger}; +use tokio::{fs::File, io::AsyncReadExt, select}; +use tokio_util::sync::CancellationToken; -pub async fn calculate_progress(logger: &Logger, release_index: &PathBuf) -> anyhow::Result<()> { +pub async fn calculate_progress( + logger: &Logger, + release_index: &PathBuf, + network: &Network, + token: CancellationToken, +) -> anyhow::Result<()> { // Deserialize the index + debug!(logger, "Deserializing index"); + let mut index = String::from(""); + let mut rel_index = File::open(release_index) + .await + .map_err(|e| anyhow::anyhow!("Couldn't open release index: {:?}", e))?; + rel_index + .read_to_string(&mut index) + .await + .map_err(|e| anyhow::anyhow!("Couldn't read release index: {:?}", e))?; + let index: Index = + serde_yaml::from_str(&index).map_err(|e| anyhow::anyhow!("Couldn't parse release indes: {:?}", e))?; + + // Check if the plan is paused + if index.rollout.pause { + info!(logger, "Release is paused, no progress to be made."); + return Ok(()); + } + + // Check if this day should be skipped + let today = Local::now().date_naive(); + if index.rollout.skip_days.iter().any(|f| f.eq(&today)) { + info!(logger, "'{}' should be skipped as per rollout skip days", today); + return Ok(()); + } // Check if the desired rollout version is elected + debug!(logger, "Creating registry"); + let mut registry_state = select! { + res = RegistryState::new(network.clone(), true) => res, + _ = token.cancelled() => { + info!(logger, "Received shutdown while creating registry"); + return Ok(()) + } + }; + debug!(logger, "Updating registry with data"); + let node_provider_data = vec![]; + select! { + res = registry_state.update_node_details(&node_provider_data) => res?, + _ = token.cancelled() => { + info!(logger, "Received shutdown while creating registry"); + return Ok(()) + } + } + debug!( + logger, + "Created registry with latest version: '{}'", + registry_state.version() + ); + debug!(logger, "Fetching elected versions"); + let elected_versions = registry_state.get_blessed_replica_versions().await?; + let current_release = match index.releases.first() { + Some(release) => release, + None => return Err(anyhow::anyhow!("Expected to have atleast one release")), + }; + + debug!(logger, "Checking if versions are elected"); + let mut current_release_feature_spec: BTreeMap> = BTreeMap::new(); + let mut current_version = String::from(""); + for version in ¤t_release.versions { + if !elected_versions.contains(&version.version) { + return Err(anyhow::anyhow!( + "Version '{}' is not blessed and is required for release '{}' with build name: {}", + version.version, + current_release.rc_name, + version.name + )); + } + + if version.name.eq(¤t_release.rc_name) { + current_version = version.version.to_string(); + continue; + } + + if !version.name.eq(¤t_release.rc_name) && version.subnets.is_empty() { + return Err(anyhow::anyhow!( + "Feature build '{}' doesn't have subnets specified", + version.name + )); + } + + current_release_feature_spec.insert(version.version.to_string(), version.subnets.clone()); + } + + for (i, stage) in index.rollout.stages.iter().enumerate() { + info!(logger, "### Checking stage {}", i); + + if stage.update_unassigned_nodes { + debug!(logger, "Unassigned nodes stage"); + + let unassigned_nodes_version = registry_state.get_unassigned_nodes_replica_version().await?; + + if unassigned_nodes_version.eq(¤t_version) { + info!(logger, "Unassigned version already at version '{}'", current_version); + continue; + } + } + + debug!(logger, "Regular nodes stage"); + for subnet_short in &stage.subnets { + let desired_version = match current_release_feature_spec + .iter() + .find(|(_, subnets)| subnets.contains(&subnet_short)) + { + Some((name, _)) => name, + None => ¤t_version, + }; + debug!( + logger, + "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version + ); + + let subnet = match registry_state + .subnets() + .into_iter() + .find(|(key, _)| key.to_string().starts_with(subnet_short)) + { + Some((_, subnet)) => subnet, + None => { + return Err(anyhow::anyhow!( + "Couldn't find subnet that starts with '{}'", + subnet_short + )) + } + }; + + if subnet.replica_version.eq(desired_version) { + info!(logger, "Subnet {} is at desired version", subnet_short); + // Check bake time + continue; + } + + info!(logger, "Subnet {} is not at desired version", subnet_short) + // Check if there is an open proposal => if yes, wait; if false, place and exit + } + info!(logger, "### Stage {} completed", i) + } // Iterate over stages and compare desired state from file and the state it is live // Take into consideration: - // 0. Check if the plan is paused. If it is do nothing // 1. If the subnet is on the desired version, proceed // 2. If the subnet isn't on the desired version: // 2.1. Check if there is an open proposal for upgrading, if there isn't place one // 2.2. If the proposal is executed check when it was upgraded and query prometheus // to see if there were any alerts and if the bake time has passed. If the bake // time didn't pass don't do anything. If there were alerts don't do anything. + + Ok(()) } +#[derive(Deserialize)] pub struct Index { pub rollout: Rollout, - pub features: Vec, pub releases: Vec, } +#[derive(Deserialize)] pub struct Rollout { - pub rc_name: String, + #[serde(default)] pub pause: bool, - pub skip_days: Vec, + pub skip_days: Vec, pub stages: Vec, } +#[derive(Deserialize, Default)] +#[serde(default)] + pub struct Stage { pub subnets: Vec, pub bake_time: String, pub wait_for_next_week: bool, + update_unassigned_nodes: bool, } -pub struct Feature { - pub name: String, - pub subnets: Vec, +#[derive(Deserialize)] +pub struct Release { + pub rc_name: String, + pub versions: Vec, } -pub struct Release { - pub name: String, +#[derive(Deserialize)] +pub struct Version { pub version: String, - pub publish: String, - pub features: Vec, + pub name: String, + #[serde(default)] + pub release_notes_read: bool, + #[serde(default)] + pub subnets: Vec, } From bdcd01c194d918f7a98f478bb6a6f46b10fd2295 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Tue, 5 Mar 2024 14:26:22 +0100 Subject: [PATCH 03/26] progressing --- Cargo.lock | 13 +++ rs/rollout-controller/Cargo.toml | 3 + rs/rollout-controller/src/main.rs | 40 ++++++-- rs/rollout-controller/src/rollout_schedule.rs | 99 +++++++++++++++++-- 4 files changed, 139 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d917aa7b3..68ae5dc9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3221,6 +3221,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + [[package]] name = "hyper" version = "0.14.28" @@ -7382,10 +7392,12 @@ dependencies = [ "crossbeam", "crossbeam-channel", "humantime", + "humantime-serde", "ic-management-backend", "ic-management-types", "ic-registry-keys", "ic-registry-local-registry", + "prometheus-http-query", "serde", "serde_json", "serde_yaml 0.9.32", @@ -7395,6 +7407,7 @@ dependencies = [ "slog-term", "tokio", "tokio-util 0.6.10", + "url", ] [[package]] diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index 05164e1f2..b0cadd90d 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -28,3 +28,6 @@ ic-registry-keys = { workspace = true } ic-registry-local-registry = { workspace = true } ic-management-backend = { workspace = true } chrono = { version = "0.4", features = [ "serde" ] } +url = { workspace = true } +prometheus-http-query = { workspace = true } +humantime-serde = "1.1.1" diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index 53bb1e3d6..2cf16c8c8 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -1,11 +1,13 @@ -use std::{path::PathBuf, time::Duration}; +use std::{path::PathBuf, str::FromStr, time::Duration}; use clap::Parser; use humantime::parse_duration; use ic_management_types::Network; +use prometheus_http_query::Client; use slog::{info, o, warn, Drain, Level, Logger}; use tokio::select; use tokio_util::sync::CancellationToken; +use url::Url; use crate::{git_sync::sync_git, registry_wrappers::sync_wrap, rollout_schedule::calculate_progress}; @@ -17,6 +19,19 @@ mod rollout_schedule; async fn main() -> anyhow::Result<()> { let args = Cli::parse(); let logger = make_logger(args.log_level.clone().into()); + let prometheus_endpoint = match &args.network { + Network::Mainnet => Url::from_str("https://victoria.ch1-obs1.dfinity.network") + .map_err(|e| anyhow::anyhow!("Couldn't parse url: {:?}", e))?, + Network::Staging => Url::from_str("https://victoria.ch1-obsstage1.dfinity.network") + .map_err(|e| anyhow::anyhow!("Couldn't parse url: {:?}", e))?, + Network::Url(url) => url.clone(), + }; + let prometheus_endpoint = prometheus_endpoint + .join("select/0/prometheus") + .map_err(|e| anyhow::anyhow!("Couldn't append victoria prometheus endpoint: {:?}", e))?; + + let client = Client::try_from(prometheus_endpoint.to_string()) + .map_err(|e| anyhow::anyhow!("Couldn't create prometheus client: {:?}", e))?; let shutdown = tokio::signal::ctrl_c(); let token = CancellationToken::new(); @@ -63,12 +78,14 @@ async fn main() -> anyhow::Result<()> { // } // } + // Calculate what should be done info!(logger, "Calculating the progress of the current release"); match calculate_progress( &logger, &args.git_repo_path.join(&args.release_index), &args.network, token.clone(), + &client, ) .await { @@ -79,15 +96,7 @@ async fn main() -> anyhow::Result<()> { } } - // Read prometheus - - // Read last iteration from disk - - // Calculate what should be done - // Apply changes - - // Serialize new state to disk } info!(logger, "Shutdown complete"); shutdown_handle.await.unwrap(); @@ -182,6 +191,19 @@ The fully qualified name of release index file in the git repositry. "# )] release_index: String, + + #[clap( + long = "prometheus-endpoint", + help = r#" +Optional url of prometheus endpoint to use for querying bake time. +If not specified it will take following based on 'Network' values: + 1. Mainnet => https://victoria.ch1-obs1.dfinity.network + 2. Staging => https://victoria.ch1-obsstage1.dfinity.network + 3. arbitrary nns url => must be specified or will error + +"# + )] + victoria_url: Option, } #[derive(Debug, Clone)] diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs index b3952679d..20bb83bbc 100644 --- a/rs/rollout-controller/src/rollout_schedule.rs +++ b/rs/rollout-controller/src/rollout_schedule.rs @@ -1,9 +1,11 @@ -use std::{collections::BTreeMap, path::PathBuf}; +use std::{collections::BTreeMap, path::PathBuf, time::Duration}; use anyhow::Ok; -use chrono::{Local, NaiveDate}; +use chrono::{Days, Local, NaiveDate}; +use humantime::format_duration; use ic_management_backend::registry::RegistryState; use ic_management_types::Network; +use prometheus_http_query::Client; use serde::Deserialize; use slog::{debug, info, Logger}; use tokio::{fs::File, io::AsyncReadExt, select}; @@ -14,6 +16,7 @@ pub async fn calculate_progress( release_index: &PathBuf, network: &Network, token: CancellationToken, + prometheus_client: &Client, ) -> anyhow::Result<()> { // Deserialize the index debug!(logger, "Deserializing index"); @@ -35,7 +38,7 @@ pub async fn calculate_progress( } // Check if this day should be skipped - let today = Local::now().date_naive(); + let today = Local::now().to_utc().date_naive(); if index.rollout.skip_days.iter().any(|f| f.eq(&today)) { info!(logger, "'{}' should be skipped as per rollout skip days", today); return Ok(()); @@ -98,7 +101,6 @@ pub async fn calculate_progress( current_release_feature_spec.insert(version.version.to_string(), version.subnets.clone()); } - for (i, stage) in index.rollout.stages.iter().enumerate() { info!(logger, "### Checking stage {}", i); @@ -111,6 +113,7 @@ pub async fn calculate_progress( info!(logger, "Unassigned version already at version '{}'", current_version); continue; } + // Update unassigned nodes } debug!(logger, "Regular nodes stage"); @@ -144,9 +147,90 @@ pub async fn calculate_progress( if subnet.replica_version.eq(desired_version) { info!(logger, "Subnet {} is at desired version", subnet_short); // Check bake time - continue; - } + let now = Local::now().to_utc(); + let principal = subnet.principal.to_string(); + let result = prometheus_client + .query_range( + format!( + r#" +last_over_time( + (timestamp( + sum by(ic_active_version,ic_subnet) (ic_replica_info{{ic_subnet="{0}", ic_active_version="{1}"}}) + ))[7d:1m] +) +"#, + principal, desired_version + ), + now.checked_sub_days(Days::new(7)).unwrap().timestamp(), + now.timestamp(), + 10.0, + ) + .get() + .await?; + + if let Some(range) = result.data().as_matrix() { + let first = match range.first() { + Some(val) => match val.samples().first() { + Some(val) => val.timestamp(), + None => { + return Err(anyhow::anyhow!( + "Expected samples to have first data for bake time for subnet '{}'", + subnet_short, + )) + } + }, + None => { + return Err(anyhow::anyhow!( + "Expected range vector to have first data for bake time for subnet '{}'", + subnet_short + )) + } + }; + + let last = match range.last() { + Some(val) => match val.samples().last() { + Some(val) => val.timestamp(), + None => { + return Err(anyhow::anyhow!( + "Expected samples to have last data for bake time for subnet '{}'", + subnet_short + )) + } + }, + None => { + return Err(anyhow::anyhow!( + "Expected range vector to have last data for bake time for subnet '{}'", + subnet_short + )) + } + }; + let diff = last - first; + + debug!( + logger, + "For subnet '{}' comparing bake time - {} {} - diff", + subnet_short, + stage.bake_time.as_secs_f64(), + diff + ); + + if diff > stage.bake_time.as_secs_f64() { + info!(logger, "Subnet {} is baked", subnet_short); + continue; + } else { + let remaining = Duration::from_secs_f64(stage.bake_time.as_secs_f64() - diff); + let formatted = format_duration(remaining); + info!( + logger, + "Waiting for subnet {} to bake, pending {}", subnet_short, formatted + ); + return Ok(()); + } + } else { + return Err(anyhow::anyhow!("Expected result data to be a matrix")); + } + } info!(logger, "Subnet {} is not at desired version", subnet_short) // Check if there is an open proposal => if yes, wait; if false, place and exit } @@ -184,7 +268,8 @@ pub struct Rollout { pub struct Stage { pub subnets: Vec, - pub bake_time: String, + #[serde(with = "humantime_serde")] + pub bake_time: Duration, pub wait_for_next_week: bool, update_unassigned_nodes: bool, } From c67c63ac63f9db3e73cc4e803fbe150006bc0232 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Tue, 5 Mar 2024 16:02:22 +0100 Subject: [PATCH 04/26] implemented base --- rs/ic-management-backend/src/registry.rs | 8 +- rs/rollout-controller/src/main.rs | 39 ++-- rs/rollout-controller/src/rollout_schedule.rs | 170 +++++++++--------- 3 files changed, 112 insertions(+), 105 deletions(-) diff --git a/rs/ic-management-backend/src/registry.rs b/rs/ic-management-backend/src/registry.rs index 544a4ba0b..25ed27582 100644 --- a/rs/ic-management-backend/src/registry.rs +++ b/rs/ic-management-backend/src/registry.rs @@ -1,7 +1,7 @@ use crate::config::get_nns_url_vec_from_target_network; use crate::factsdb; use crate::git_ic_repo::IcRepo; -use crate::proposal; +use crate::proposal::{self, SubnetUpdateProposal}; use crate::public_dashboard::query_ic_dashboard_list; use async_trait::async_trait; use decentralization::network::{AvailableNodesQuerier, SubnetQuerier, SubnetQueryBy}; @@ -718,6 +718,12 @@ impl RegistryState { } } + pub async fn open_subnet_upgrade_proposals(&self) -> Result> { + let proposal_agent = proposal::ProposalAgent::new(self.nns_url.clone()); + + proposal_agent.list_update_subnet_version_proposals().await + } + async fn retireable_hostos_versions(&self) -> Result> { let active_releases = self.hostos_releases.get_active_branches(); let hostos_versions: BTreeSet = self.nodes.values().map(|s| s.hostos_version.clone()).collect(); diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index 2cf16c8c8..c8ab1a31e 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -40,9 +40,11 @@ async fn main() -> anyhow::Result<()> { let shutdown_logger = logger.clone(); let shutdown_token = token.clone(); let shutdown_handle = tokio::spawn(async move { - shutdown.await.unwrap(); + select! { + _ = shutdown => shutdown_token.cancel(), + _ = shutdown_token.cancelled() => {} + } info!(shutdown_logger, "Received shutdown"); - shutdown_token.cancel(); }); let mut interval = tokio::time::interval(args.poll_interval); @@ -50,23 +52,23 @@ async fn main() -> anyhow::Result<()> { loop { if should_sleep { select! { - tick = interval.tick() => info!(logger, "Running loop @ {:?}", tick), _ = token.cancelled() => break, + tick = interval.tick() => info!(logger, "Running loop @ {:?}", tick), } } else if token.is_cancelled() { break; } should_sleep = true; - // info!(logger, "Syncing registry for network '{:?}'", args.network); - // match sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()).await { - // Ok(()) => info!(logger, "Syncing registry completed"), - // Err(e) => { - // warn!(logger, "{:?}", e); - // should_sleep = false; - // continue; - // } - // }; + info!(logger, "Syncing registry for network '{:?}'", args.network); + match sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()).await { + Ok(()) => info!(logger, "Syncing registry completed"), + Err(e) => { + warn!(logger, "{:?}", e); + should_sleep = false; + continue; + } + }; // info!(logger, "Syncing git repo"); // match sync_git(&logger, &args.git_repo_path, &args.git_repo_url, &args.release_index).await { @@ -80,7 +82,7 @@ async fn main() -> anyhow::Result<()> { // Calculate what should be done info!(logger, "Calculating the progress of the current release"); - match calculate_progress( + let actions = match calculate_progress( &logger, &args.git_repo_path.join(&args.release_index), &args.network, @@ -89,14 +91,21 @@ async fn main() -> anyhow::Result<()> { ) .await { - Ok(()) => info!(logger, "Calculating completed"), + Ok(actions) => actions, Err(e) => { warn!(logger, "{:?}", e); continue; } - } + }; + info!(logger, "Calculating completed"); + if actions.is_empty() { + info!(logger, "No actions needed, sleeping"); + continue; + } + info!(logger, "Calculated actions: {:#?}", actions); // Apply changes + token.cancel(); } info!(logger, "Shutdown complete"); shutdown_handle.await.unwrap(); diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs index 20bb83bbc..7ebcc29d9 100644 --- a/rs/rollout-controller/src/rollout_schedule.rs +++ b/rs/rollout-controller/src/rollout_schedule.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, path::PathBuf, time::Duration}; use anyhow::Ok; -use chrono::{Days, Local, NaiveDate}; +use chrono::{Local, NaiveDate}; use humantime::format_duration; use ic_management_backend::registry::RegistryState; use ic_management_types::Network; @@ -17,7 +17,7 @@ pub async fn calculate_progress( network: &Network, token: CancellationToken, prometheus_client: &Client, -) -> anyhow::Result<()> { +) -> anyhow::Result> { // Deserialize the index debug!(logger, "Deserializing index"); let mut index = String::from(""); @@ -34,14 +34,14 @@ pub async fn calculate_progress( // Check if the plan is paused if index.rollout.pause { info!(logger, "Release is paused, no progress to be made."); - return Ok(()); + return Ok(vec![]); } // Check if this day should be skipped let today = Local::now().to_utc().date_naive(); if index.rollout.skip_days.iter().any(|f| f.eq(&today)) { info!(logger, "'{}' should be skipped as per rollout skip days", today); - return Ok(()); + return Ok(vec![]); } // Check if the desired rollout version is elected @@ -50,7 +50,7 @@ pub async fn calculate_progress( res = RegistryState::new(network.clone(), true) => res, _ = token.cancelled() => { info!(logger, "Received shutdown while creating registry"); - return Ok(()) + return Ok(vec![]) } }; debug!(logger, "Updating registry with data"); @@ -59,7 +59,7 @@ pub async fn calculate_progress( res = registry_state.update_node_details(&node_provider_data) => res?, _ = token.cancelled() => { info!(logger, "Received shutdown while creating registry"); - return Ok(()) + return Ok(vec![]) } } debug!( @@ -101,8 +101,38 @@ pub async fn calculate_progress( current_release_feature_spec.insert(version.version.to_string(), version.subnets.clone()); } + + let mut last_bake_status: BTreeMap = BTreeMap::new(); + let result = prometheus_client + .query( + r#" + time() - max(last_over_time( + (timestamp( + sum by(ic_active_version,ic_subnet) (ic_replica_info) + ))[21d:1m] + ) unless (sum by (ic_active_version, ic_subnet) (ic_replica_info))) by (ic_subnet) + "#, + ) + .get() + .await?; + + let last = match result.data().clone().into_vector().into_iter().last() { + Some(data) => data, + None => return Err(anyhow::anyhow!("There should be data regarding ic_replica_info")), + }; + + for vector in last.iter() { + let subnet = vector.metric().get("ic_subnet").expect("To have ic_subnet key"); + let last_update = vector.sample().value(); + last_bake_status.insert(subnet.to_string(), last_update); + } + + let subnet_update_proposals = registry_state.open_subnet_upgrade_proposals().await?; + for (i, stage) in index.rollout.stages.iter().enumerate() { info!(logger, "### Checking stage {}", i); + let mut actions = vec![]; + let mut stage_baked = true; if stage.update_unassigned_nodes { debug!(logger, "Unassigned nodes stage"); @@ -113,7 +143,9 @@ pub async fn calculate_progress( info!(logger, "Unassigned version already at version '{}'", current_version); continue; } - // Update unassigned nodes + // Should update unassigned nodes + actions.push(format!("Update unassigned nodes to version: {}", current_version)); + return Ok(actions); } debug!(logger, "Regular nodes stage"); @@ -147,93 +179,53 @@ pub async fn calculate_progress( if subnet.replica_version.eq(desired_version) { info!(logger, "Subnet {} is at desired version", subnet_short); // Check bake time - let now = Local::now().to_utc(); - let principal = subnet.principal.to_string(); - let result = prometheus_client - .query_range( - format!( - r#" -last_over_time( - (timestamp( - sum by(ic_active_version,ic_subnet) (ic_replica_info{{ic_subnet="{0}", ic_active_version="{1}"}}) - ))[7d:1m] -) -"#, - principal, desired_version - ), - now.checked_sub_days(Days::new(7)).unwrap().timestamp(), - now.timestamp(), - 10.0, - ) - .get() - .await?; - - if let Some(range) = result.data().as_matrix() { - let first = match range.first() { - Some(val) => match val.samples().first() { - Some(val) => val.timestamp(), - None => { - return Err(anyhow::anyhow!( - "Expected samples to have first data for bake time for subnet '{}'", - subnet_short, - )) - } - }, - None => { - return Err(anyhow::anyhow!( - "Expected range vector to have first data for bake time for subnet '{}'", - subnet_short - )) - } - }; - - let last = match range.last() { - Some(val) => match val.samples().last() { - Some(val) => val.timestamp(), - None => { - return Err(anyhow::anyhow!( - "Expected samples to have last data for bake time for subnet '{}'", - subnet_short - )) - } - }, - None => { - return Err(anyhow::anyhow!( - "Expected range vector to have last data for bake time for subnet '{}'", - subnet_short - )) - } - }; - - let diff = last - first; - - debug!( + let bake = last_bake_status + .get(&subnet.principal.to_string()) + .expect("Should have key"); + if bake.gt(&stage.bake_time.as_secs_f64()) { + info!(logger, "Subnet {} is baked", subnet_short); + continue; + } else { + let remaining = Duration::from_secs_f64(stage.bake_time.as_secs_f64() - bake); + let formatted = format_duration(remaining); + info!( logger, - "For subnet '{}' comparing bake time - {} {} - diff", - subnet_short, - stage.bake_time.as_secs_f64(), - diff + "Waiting for subnet {} to bake, pending {}", subnet_short, formatted ); - - if diff > stage.bake_time.as_secs_f64() { - info!(logger, "Subnet {} is baked", subnet_short); - continue; - } else { - let remaining = Duration::from_secs_f64(stage.bake_time.as_secs_f64() - diff); - let formatted = format_duration(remaining); - info!( - logger, - "Waiting for subnet {} to bake, pending {}", subnet_short, formatted - ); - return Ok(()); - } - } else { - return Err(anyhow::anyhow!("Expected result data to be a matrix")); + stage_baked |= false; } } - info!(logger, "Subnet {} is not at desired version", subnet_short) + info!(logger, "Subnet {} is not at desired version", subnet_short); // Check if there is an open proposal => if yes, wait; if false, place and exit + + if let Some(proposal) = subnet_update_proposals.iter().find(|proposal| { + proposal.payload.subnet_id == subnet.principal + && proposal.payload.replica_version_id.eq(desired_version) + && !proposal.info.executed + }) { + info!( + logger, + "For subnet '{}' found open proposal 'https://dashboard.internetcomputer.org/proposal/{}'", + subnet_short, + proposal.info.id + ); + continue; + } + + actions.push(format!( + "Should update subnet '{}' to version '{}'", + subnet_short, desired_version + )) } + + if !stage_baked { + return Ok(vec![]); + } + + if !actions.is_empty() { + return Ok(actions); + } + info!(logger, "### Stage {} completed", i) } @@ -246,7 +238,7 @@ last_over_time( // to see if there were any alerts and if the bake time has passed. If the bake // time didn't pass don't do anything. If there were alerts don't do anything. - Ok(()) + Ok(vec![]) } #[derive(Deserialize)] From 60b3e4b7022acabb879fd75de45c9123bfe7bb8e Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 8 Mar 2024 13:44:17 +0100 Subject: [PATCH 05/26] adding traits and tidying up fetcher logic --- Cargo.lock | 1 + rs/rollout-controller/Cargo.toml | 2 + .../src/fetching/curl_fetcher.rs | 42 ++++++++ rs/rollout-controller/src/fetching/mod.rs | 44 +++++++++ .../src/fetching/sparse_checkout_fetcher.rs | 98 +++++++++++++++++++ rs/rollout-controller/src/git_sync.rs | 63 ------------ rs/rollout-controller/src/main.rs | 45 ++++++--- rs/rollout-controller/src/rollout_schedule.rs | 2 +- rust-toolchain.toml | 2 +- 9 files changed, 223 insertions(+), 76 deletions(-) create mode 100644 rs/rollout-controller/src/fetching/curl_fetcher.rs create mode 100644 rs/rollout-controller/src/fetching/mod.rs create mode 100644 rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs delete mode 100644 rs/rollout-controller/src/git_sync.rs diff --git a/Cargo.lock b/Cargo.lock index 68ae5dc9d..947b0f91e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7398,6 +7398,7 @@ dependencies = [ "ic-registry-keys", "ic-registry-local-registry", "prometheus-http-query", + "reqwest", "serde", "serde_json", "serde_yaml 0.9.32", diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index b0cadd90d..c1c092ad6 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -31,3 +31,5 @@ chrono = { version = "0.4", features = [ "serde" ] } url = { workspace = true } prometheus-http-query = { workspace = true } humantime-serde = "1.1.1" +reqwest = { workspace = true } + diff --git a/rs/rollout-controller/src/fetching/curl_fetcher.rs b/rs/rollout-controller/src/fetching/curl_fetcher.rs new file mode 100644 index 000000000..2d000dcc7 --- /dev/null +++ b/rs/rollout-controller/src/fetching/curl_fetcher.rs @@ -0,0 +1,42 @@ +use reqwest::Client; +use slog::{debug, Logger}; + +use super::RolloutScheduleFetcher; + +#[derive(Clone)] +pub struct CurlFetcher { + logger: Logger, + client: Client, + url: String, +} + +impl CurlFetcher { + pub fn new(logger: Logger, url: String) -> anyhow::Result { + Ok(Self { + client: Client::new(), + logger, + url, + }) + } +} + +impl RolloutScheduleFetcher for CurlFetcher { + async fn fetch(&self) -> anyhow::Result { + debug!(self.logger, "Fetching rollout index"); + + let response = self + .client + .get(&self.url) + .send() + .await + .map_err(|e| anyhow::anyhow!("Error fetching rollout schedule: {:?}", e))?; + + let bytes = response + .bytes() + .await + .map_err(|e| anyhow::anyhow!("Error converting body to bytes: {:?}", e))?; + + serde_yaml::from_slice(bytes.to_vec().as_slice()) + .map_err(|e| anyhow::anyhow!("Couldn't parse release index: {:?}", e)) + } +} diff --git a/rs/rollout-controller/src/fetching/mod.rs b/rs/rollout-controller/src/fetching/mod.rs new file mode 100644 index 000000000..bfc70742f --- /dev/null +++ b/rs/rollout-controller/src/fetching/mod.rs @@ -0,0 +1,44 @@ +use std::path::PathBuf; + +use slog::Logger; + +use crate::rollout_schedule::Index; + +use self::{curl_fetcher::CurlFetcher, sparse_checkout_fetcher::SparseCheckoutFetcher}; + +pub mod curl_fetcher; +pub mod sparse_checkout_fetcher; + +pub trait RolloutScheduleFetcher { + async fn fetch(&self) -> anyhow::Result; +} + +pub enum RolloutScheduleFetcherImplementation { + Curl(CurlFetcher), + Git(SparseCheckoutFetcher), +} + +pub async fn resolve( + mode: String, + logger: Logger, + path: PathBuf, + url: String, + release_index: String, +) -> anyhow::Result { + match mode.to_lowercase().as_str() { + "git" => SparseCheckoutFetcher::new(logger, path, url, release_index) + .await + .map(RolloutScheduleFetcherImplementation::Git), + "curl" => CurlFetcher::new(logger, url).map(RolloutScheduleFetcherImplementation::Curl), + _ => Err(anyhow::anyhow!("Couldn't construct index fetcher for mode '{}'", mode)), + } +} + +impl RolloutScheduleFetcherImplementation { + pub async fn fetch(&self) -> anyhow::Result { + match self { + RolloutScheduleFetcherImplementation::Curl(implementation) => implementation.fetch().await, + RolloutScheduleFetcherImplementation::Git(implementation) => implementation.fetch().await, + } + } +} diff --git a/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs b/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs new file mode 100644 index 000000000..911e383cd --- /dev/null +++ b/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs @@ -0,0 +1,98 @@ +use std::path::PathBuf; + +use slog::{debug, info, Logger}; +use tokio::{ + fs::{create_dir_all, File}, + io::{AsyncReadExt, AsyncWriteExt}, + process::Command, +}; + +use crate::rollout_schedule::Index; + +use super::RolloutScheduleFetcher; + +#[derive(Clone)] +pub struct SparseCheckoutFetcher { + logger: Logger, + path: PathBuf, + release_index: String, +} + +impl SparseCheckoutFetcher { + pub async fn new(logger: Logger, path: PathBuf, url: String, release_index: String) -> anyhow::Result { + let fetcher = Self { + logger, + path, + release_index, + }; + + if !fetcher.path.exists() { + info!(fetcher.logger, "Git directory not found. Creating..."); + create_dir_all(&fetcher.path) + .await + .map_err(|e| anyhow::anyhow!("Couldn't create directory for git repo: {:?}", e))?; + debug!( + fetcher.logger, + "Created directory for github repo at: '{}'", + fetcher.path.display() + ); + + fetcher.configure_git_repo(&url).await?; + debug!(fetcher.logger, "Repo configured") + } + + Ok(fetcher) + } + + async fn configure_git_repo(&self, url: &String) -> anyhow::Result<()> { + debug!(self.logger, "Initializing repository on path '{}'", self.path.display()); + Self::execute_git_command(&self.path, &["init"]).await?; + debug!(self.logger, "Configuring sparse checkout"); + Self::execute_git_command(&self.path, &["config", "core.sparseCheckout", "true"]).await?; + debug!(self.logger, "Setting up remote"); + Self::execute_git_command(&self.path, &["remote", "add", "-f", "origin", url]).await?; + + debug!(self.logger, "Creating file for sparse checkout paths"); + let mut file = File::create(self.path.join(".git/info/sparse-checkout")) + .await + .map_err(|e| anyhow::anyhow!("Couldn't create sparse-checkout file: {:?}", e))?; + + debug!(self.logger, "Writing release index path to sparse checkout"); + file.write_all(self.release_index.as_bytes()) + .await + .map_err(|e| anyhow::anyhow!("Couldn't write to sparse-checkout file: {:?}", e))?; + + debug!(self.logger, "Checking out 'main'"); + Self::execute_git_command(&self.path, &["checkout", "main"]).await?; + + Ok(()) + } + + async fn execute_git_command(path: &PathBuf, args: &[&str]) -> anyhow::Result<()> { + let mut cmd = Command::new("git"); + cmd.current_dir(path); + cmd.args(args.iter()); + cmd.output() + .await + .map_err(|e| anyhow::anyhow!("Couldn't execute command 'git {}': {:?}", args.join(" "), e))?; + Ok(()) + } +} + +impl RolloutScheduleFetcher for SparseCheckoutFetcher { + async fn fetch(&self) -> anyhow::Result { + debug!(self.logger, "Syncing git repo"); + SparseCheckoutFetcher::execute_git_command(&self.path, &["pull"]).await?; + + debug!(self.logger, "Deserializing index"); + let mut index = String::from(""); + let mut rel_index = File::open(self.path.join(&self.release_index)) + .await + .map_err(|e| anyhow::anyhow!("Couldn't open release index: {:?}", e))?; + rel_index + .read_to_string(&mut index) + .await + .map_err(|e| anyhow::anyhow!("Couldn't read release index: {:?}", e))?; + serde_yaml::from_str(&index).map_err(|e| anyhow::anyhow!("Couldn't parse release index: {:?}", e)) + } +} diff --git a/rs/rollout-controller/src/git_sync.rs b/rs/rollout-controller/src/git_sync.rs deleted file mode 100644 index 04c3460a7..000000000 --- a/rs/rollout-controller/src/git_sync.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::path::PathBuf; - -use slog::{debug, info, Logger}; -use tokio::{ - fs::{create_dir_all, File}, - io::AsyncWriteExt, - process::Command, -}; - -pub async fn sync_git(logger: &Logger, path: &PathBuf, url: &String, release_index: &String) -> anyhow::Result<()> { - if !path.exists() { - info!(logger, "Git directory not found. Creating..."); - create_dir_all(path) - .await - .map_err(|e| anyhow::anyhow!("Couldn't create directory for git repo: {:?}", e))?; - debug!(logger, "Created directory for github repo at: '{}'", path.display()); - - configure_git_repo(path, url, &logger, release_index).await?; - debug!(logger, "Repo configured") - } - - debug!(logger, "Syncing git repo"); - execute_git_command(&path, &["pull"]).await -} - -async fn configure_git_repo( - path: &PathBuf, - url: &String, - logger: &Logger, - release_index: &String, -) -> anyhow::Result<()> { - debug!(logger, "Initializing repository on path '{}'", path.display()); - execute_git_command(path, &["init"]).await?; - debug!(logger, "Configuring sparse checkout"); - execute_git_command(path, &["config", "core.sparseCheckout", "true"]).await?; - debug!(logger, "Setting up remote"); - execute_git_command(path, &["remote", "add", "-f", "origin", url]).await?; - - debug!(logger, "Creating file for sparse checkout paths"); - let mut file = File::create(path.join(".git/info/sparse-checkout")) - .await - .map_err(|e| anyhow::anyhow!("Couldn't create sparse-checkout file: {:?}", e))?; - - debug!(logger, "Writing release index path to sparse checkout"); - file.write_all(release_index.as_bytes()) - .await - .map_err(|e| anyhow::anyhow!("Couldn't write to sparse-checkout file: {:?}", e))?; - - debug!(logger, "Checking out 'main'"); - execute_git_command(path, &["checkout", "main"]).await?; - - Ok(()) -} - -async fn execute_git_command(path: &PathBuf, args: &[&str]) -> anyhow::Result<()> { - let mut cmd = Command::new("git"); - cmd.current_dir(path); - cmd.args(args.iter()); - cmd.output() - .await - .map_err(|e| anyhow::anyhow!("Couldn't execute command 'git {}': {:?}", args.join(" "), e))?; - Ok(()) -} diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index c8ab1a31e..c0dfd7a11 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -9,9 +9,9 @@ use tokio::select; use tokio_util::sync::CancellationToken; use url::Url; -use crate::{git_sync::sync_git, registry_wrappers::sync_wrap, rollout_schedule::calculate_progress}; +use crate::{registry_wrappers::sync_wrap, rollout_schedule::calculate_progress}; -mod git_sync; +mod fetching; mod registry_wrappers; mod rollout_schedule; @@ -47,6 +47,15 @@ async fn main() -> anyhow::Result<()> { info!(shutdown_logger, "Received shutdown"); }); + let fetcher = fetching::resolve( + args.mode, + logger.clone(), + args.git_repo_path.clone(), + args.git_repo_url.clone(), + args.release_index.clone(), + ) + .await?; + let mut interval = tokio::time::interval(args.poll_interval); let mut should_sleep = false; loop { @@ -70,15 +79,18 @@ async fn main() -> anyhow::Result<()> { } }; - // info!(logger, "Syncing git repo"); - // match sync_git(&logger, &args.git_repo_path, &args.git_repo_url, &args.release_index).await { - // Ok(()) => info!(logger, "Syncing git repo completed"), - // Err(e) => { - // warn!(logger, "{:?}", e); - // should_sleep = false; - // continue; - // } - // } + info!(logger, "Fetching rollout index"); + let index = match fetcher.fetch().await { + Ok(index) => { + info!(logger, "Fetching of new index complete"); + index + } + Err(e) => { + warn!(logger, "{:?}", e); + should_sleep = false; + continue; + } + }; // Calculate what should be done info!(logger, "Calculating the progress of the current release"); @@ -213,6 +225,17 @@ If not specified it will take following based on 'Network' values: "# )] victoria_url: Option, + + #[clap( + long = "mode", + help = r#" +Mode for fetching the release index. Available modes: + 1. git => runs using 'sparse checkout' of a file + 2. curl => runs using fetching of raw file + +"# + )] + mode: String, } #[derive(Debug, Clone)] diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs index 7ebcc29d9..1fa7d163b 100644 --- a/rs/rollout-controller/src/rollout_schedule.rs +++ b/rs/rollout-controller/src/rollout_schedule.rs @@ -152,7 +152,7 @@ pub async fn calculate_progress( for subnet_short in &stage.subnets { let desired_version = match current_release_feature_spec .iter() - .find(|(_, subnets)| subnets.contains(&subnet_short)) + .find(|(_, subnets)| subnets.contains(subnet_short)) { Some((name, _)) => name, None => ¤t_version, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f15816914..0ff3a2bbb 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] # Make sure docker/Dockerfile has the same Rust toolchain version -channel = "1.74.0" +channel = "1.75.0" profile = "default" components = ["rls"] From ffaea557ec45de29943b4ac960bbcfdfe95294dc Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 8 Mar 2024 14:13:08 +0100 Subject: [PATCH 06/26] refactoring calculation to enhance testability --- .../src/fetching/curl_fetcher.rs | 14 ++++ rs/rollout-controller/src/fetching/mod.rs | 30 +++---- .../src/fetching/sparse_checkout_fetcher.rs | 33 ++++++++ rs/rollout-controller/src/main.rs | 78 +++++-------------- .../src/registry_wrappers.rs | 33 ++++++-- rs/rollout-controller/src/rollout_schedule.rs | 46 +---------- 6 files changed, 113 insertions(+), 121 deletions(-) diff --git a/rs/rollout-controller/src/fetching/curl_fetcher.rs b/rs/rollout-controller/src/fetching/curl_fetcher.rs index 2d000dcc7..9846f21a0 100644 --- a/rs/rollout-controller/src/fetching/curl_fetcher.rs +++ b/rs/rollout-controller/src/fetching/curl_fetcher.rs @@ -1,8 +1,22 @@ +use clap::Parser; use reqwest::Client; use slog::{debug, Logger}; use super::RolloutScheduleFetcher; +#[derive(Parser, Clone, Debug)] +pub struct CurlFetcherConfig { + #[clap( + long = "url", + default_value = "https://raw.githubusercontent.com/dfinity/dre/main/release-index.yaml", + help = r#" +The url of the raw file in github + +"# + )] + pub url: String, +} + #[derive(Clone)] pub struct CurlFetcher { logger: Logger, diff --git a/rs/rollout-controller/src/fetching/mod.rs b/rs/rollout-controller/src/fetching/mod.rs index bfc70742f..b475bcce8 100644 --- a/rs/rollout-controller/src/fetching/mod.rs +++ b/rs/rollout-controller/src/fetching/mod.rs @@ -1,10 +1,11 @@ -use std::path::PathBuf; - use slog::Logger; -use crate::rollout_schedule::Index; +use crate::{rollout_schedule::Index, Commands}; -use self::{curl_fetcher::CurlFetcher, sparse_checkout_fetcher::SparseCheckoutFetcher}; +use self::{ + curl_fetcher::{CurlFetcher, CurlFetcherConfig}, + sparse_checkout_fetcher::{SparseCheckoutFetcher, SparseCheckoutFetcherConfig}, +}; pub mod curl_fetcher; pub mod sparse_checkout_fetcher; @@ -18,19 +19,18 @@ pub enum RolloutScheduleFetcherImplementation { Git(SparseCheckoutFetcher), } -pub async fn resolve( - mode: String, - logger: Logger, - path: PathBuf, - url: String, - release_index: String, -) -> anyhow::Result { - match mode.to_lowercase().as_str() { - "git" => SparseCheckoutFetcher::new(logger, path, url, release_index) +pub async fn resolve(subcmd: Commands, logger: Logger) -> anyhow::Result { + match subcmd { + Commands::Git(SparseCheckoutFetcherConfig { + repo_url, + release_index, + repo_path, + }) => SparseCheckoutFetcher::new(logger, repo_path, repo_url, release_index) .await .map(RolloutScheduleFetcherImplementation::Git), - "curl" => CurlFetcher::new(logger, url).map(RolloutScheduleFetcherImplementation::Curl), - _ => Err(anyhow::anyhow!("Couldn't construct index fetcher for mode '{}'", mode)), + Commands::Curl(CurlFetcherConfig { url }) => { + CurlFetcher::new(logger, url).map(RolloutScheduleFetcherImplementation::Curl) + } } } diff --git a/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs b/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs index 911e383cd..103b5d580 100644 --- a/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs +++ b/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use clap::Parser; use slog::{debug, info, Logger}; use tokio::{ fs::{create_dir_all, File}, @@ -11,6 +12,38 @@ use crate::rollout_schedule::Index; use super::RolloutScheduleFetcher; +#[derive(Parser, Clone, Debug)] +pub struct SparseCheckoutFetcherConfig { + #[clap( + long = "repo-path", + help = r#" +The path to the directory that will be used for git sync + +"# + )] + pub repo_path: PathBuf, + + #[clap( + long = "repo-url", + default_value = "git@github.com:dfinity/dre.git", + help = r#" +The url of the repository with which we should sync. + +"# + )] + pub repo_url: String, + + #[clap( + long = "release-file-name", + default_value = "release-index.yaml", + help = r#" +The fully qualified name of release index file in the git repositry. + +"# + )] + pub release_index: String, +} + #[derive(Clone)] pub struct SparseCheckoutFetcher { logger: Logger, diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index c0dfd7a11..8ce71c3b1 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -1,6 +1,7 @@ use std::{path::PathBuf, str::FromStr, time::Duration}; -use clap::Parser; +use clap::{Parser, Subcommand}; +use fetching::{curl_fetcher::CurlFetcherConfig, sparse_checkout_fetcher::SparseCheckoutFetcherConfig}; use humantime::parse_duration; use ic_management_types::Network; use prometheus_http_query::Client; @@ -47,14 +48,7 @@ async fn main() -> anyhow::Result<()> { info!(shutdown_logger, "Received shutdown"); }); - let fetcher = fetching::resolve( - args.mode, - logger.clone(), - args.git_repo_path.clone(), - args.git_repo_url.clone(), - args.release_index.clone(), - ) - .await?; + let fetcher = fetching::resolve(args.subcommand, logger.clone()).await?; let mut interval = tokio::time::interval(args.poll_interval); let mut should_sleep = false; @@ -70,8 +64,15 @@ async fn main() -> anyhow::Result<()> { should_sleep = true; info!(logger, "Syncing registry for network '{:?}'", args.network); - match sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()).await { - Ok(()) => info!(logger, "Syncing registry completed"), + let maybe_registry_state = select! { + res = sync_wrap(logger.clone(), args.targets_dir.clone(), args.network.clone()) => res, + _ = token.cancelled() => break, + }; + let registry_state = match maybe_registry_state { + Ok(state) => { + info!(logger, "Syncing registry completed"); + state + } Err(e) => { warn!(logger, "{:?}", e); should_sleep = false; @@ -94,15 +95,7 @@ async fn main() -> anyhow::Result<()> { // Calculate what should be done info!(logger, "Calculating the progress of the current release"); - let actions = match calculate_progress( - &logger, - &args.git_repo_path.join(&args.release_index), - &args.network, - token.clone(), - &client, - ) - .await - { + let actions = match calculate_progress(&logger, index, &client, registry_state).await { Ok(actions) => actions, Err(e) => { warn!(logger, "{:?}", e); @@ -184,35 +177,6 @@ The interval at which ICs are polled for updates. )] poll_interval: Duration, - #[clap( - long = "git-repo-path", - help = r#" -The path to the directory that will be used for git sync - -"# - )] - git_repo_path: PathBuf, - - #[clap( - long = "git-repo-url", - default_value = "git@github.com:dfinity/dre.git", - help = r#" -The url of the repository with which we should sync. - -"# - )] - git_repo_url: String, - - #[clap( - long = "release-file-name", - default_value = "release-index.yaml", - help = r#" -The fully qualified name of release index file in the git repositry. - -"# - )] - release_index: String, - #[clap( long = "prometheus-endpoint", help = r#" @@ -226,16 +190,14 @@ If not specified it will take following based on 'Network' values: )] victoria_url: Option, - #[clap( - long = "mode", - help = r#" -Mode for fetching the release index. Available modes: - 1. git => runs using 'sparse checkout' of a file - 2. curl => runs using fetching of raw file + #[clap(subcommand)] + pub(crate) subcommand: Commands, +} -"# - )] - mode: String, +#[derive(Subcommand, Clone, Debug)] +enum Commands { + Git(SparseCheckoutFetcherConfig), + Curl(CurlFetcherConfig), } #[derive(Debug, Clone)] diff --git a/rs/rollout-controller/src/registry_wrappers.rs b/rs/rollout-controller/src/registry_wrappers.rs index 6a3170f17..1a10bde65 100644 --- a/rs/rollout-controller/src/registry_wrappers.rs +++ b/rs/rollout-controller/src/registry_wrappers.rs @@ -1,13 +1,36 @@ use std::path::PathBuf; +use ic_management_backend::registry::RegistryState; use ic_management_types::Network; use service_discovery::registry_sync::sync_local_registry; -use slog::Logger; +use slog::{debug, Logger}; -pub async fn sync_wrap(logger: Logger, targets_dir: PathBuf, network: Network) -> anyhow::Result<()> { +pub async fn sync_wrap(logger: Logger, targets_dir: PathBuf, network: Network) -> anyhow::Result { let (_, stop_signal) = crossbeam::channel::bounded::<()>(0); - sync_local_registry(logger, targets_dir, vec![network.get_url()], false, None, &stop_signal) - .await - .map_err(|e| anyhow::anyhow!("Error during syncing registry: {:?}", e)) + sync_local_registry( + logger.clone(), + targets_dir, + vec![network.get_url()], + false, + None, + &stop_signal, + ) + .await + .map_err(|e| anyhow::anyhow!("Error during syncing registry: {:?}", e))?; + + // Check if the desired rollout version is elected + debug!(logger, "Creating registry"); + let mut registry_state = RegistryState::new(network.clone(), true).await; + + debug!(logger, "Updating registry with data"); + let node_provider_data = vec![]; + registry_state.update_node_details(&node_provider_data).await?; + debug!( + logger, + "Created registry with latest version: '{}'", + registry_state.version() + ); + + Ok(registry_state) } diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs index 1fa7d163b..aa6ab12b9 100644 --- a/rs/rollout-controller/src/rollout_schedule.rs +++ b/rs/rollout-controller/src/rollout_schedule.rs @@ -1,36 +1,19 @@ -use std::{collections::BTreeMap, path::PathBuf, time::Duration}; +use std::{collections::BTreeMap, time::Duration}; use anyhow::Ok; use chrono::{Local, NaiveDate}; use humantime::format_duration; use ic_management_backend::registry::RegistryState; -use ic_management_types::Network; use prometheus_http_query::Client; use serde::Deserialize; use slog::{debug, info, Logger}; -use tokio::{fs::File, io::AsyncReadExt, select}; -use tokio_util::sync::CancellationToken; pub async fn calculate_progress( logger: &Logger, - release_index: &PathBuf, - network: &Network, - token: CancellationToken, + index: Index, prometheus_client: &Client, + registry_state: RegistryState, ) -> anyhow::Result> { - // Deserialize the index - debug!(logger, "Deserializing index"); - let mut index = String::from(""); - let mut rel_index = File::open(release_index) - .await - .map_err(|e| anyhow::anyhow!("Couldn't open release index: {:?}", e))?; - rel_index - .read_to_string(&mut index) - .await - .map_err(|e| anyhow::anyhow!("Couldn't read release index: {:?}", e))?; - let index: Index = - serde_yaml::from_str(&index).map_err(|e| anyhow::anyhow!("Couldn't parse release indes: {:?}", e))?; - // Check if the plan is paused if index.rollout.pause { info!(logger, "Release is paused, no progress to be made."); @@ -44,29 +27,6 @@ pub async fn calculate_progress( return Ok(vec![]); } - // Check if the desired rollout version is elected - debug!(logger, "Creating registry"); - let mut registry_state = select! { - res = RegistryState::new(network.clone(), true) => res, - _ = token.cancelled() => { - info!(logger, "Received shutdown while creating registry"); - return Ok(vec![]) - } - }; - debug!(logger, "Updating registry with data"); - let node_provider_data = vec![]; - select! { - res = registry_state.update_node_details(&node_provider_data) => res?, - _ = token.cancelled() => { - info!(logger, "Received shutdown while creating registry"); - return Ok(vec![]) - } - } - debug!( - logger, - "Created registry with latest version: '{}'", - registry_state.version() - ); debug!(logger, "Fetching elected versions"); let elected_versions = registry_state.get_blessed_replica_versions().await?; let current_release = match index.releases.first() { From a9b6c5f77a82c59f30a9b069df67082a970ce7fe Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 8 Mar 2024 14:28:24 +0100 Subject: [PATCH 07/26] repinning --- Cargo.Bazel.lock | 85 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index 733c4674d..21c247bd1 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "4cbc45073b367bdd54509e9c2c2a37238dcc5ab9c68dc9258cc18bd1975156f7", + "checksum": "a5133e8350a1044ca018fd8519fc227babb0d02b5c2b8b2c263dc054d6dd51ff", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -16178,6 +16178,49 @@ }, "license": "MIT/Apache-2.0" }, + "humantime-serde 1.1.1": { + "name": "humantime-serde", + "version": "1.1.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/humantime-serde/1.1.1/download", + "sha256": "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "humantime_serde", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "humantime_serde", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "humantime 2.1.0", + "target": "humantime" + }, + { + "id": "serde 1.0.197", + "target": "serde" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.1.1" + }, + "license": "MIT OR Apache-2.0" + }, "hyper 0.14.28": { "name": "hyper", "version": "0.14.28", @@ -37098,6 +37141,10 @@ "id": "anyhow 1.0.80", "target": "anyhow" }, + { + "id": "chrono 0.4.34", + "target": "chrono" + }, { "id": "clap 4.4.18", "target": "clap" @@ -37114,6 +37161,38 @@ "id": "humantime 2.1.0", "target": "humantime" }, + { + "id": "humantime-serde 1.1.1", + "target": "humantime_serde" + }, + { + "id": "ic-registry-keys 0.9.0", + "target": "ic_registry_keys" + }, + { + "id": "ic-registry-local-registry 0.9.0", + "target": "ic_registry_local_registry" + }, + { + "id": "prometheus-http-query 0.8.2", + "target": "prometheus_http_query" + }, + { + "id": "reqwest 0.11.24", + "target": "reqwest" + }, + { + "id": "serde 1.0.197", + "target": "serde" + }, + { + "id": "serde_json 1.0.114", + "target": "serde_json" + }, + { + "id": "serde_yaml 0.9.32", + "target": "serde_yaml" + }, { "id": "slog 2.7.0", "target": "slog" @@ -37133,6 +37212,10 @@ { "id": "tokio-util 0.6.10", "target": "tokio_util" + }, + { + "id": "url 2.5.0", + "target": "url" } ], "selects": {} From c1a5a731a038b167c8ba1c23ccef7b713e39900a Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 8 Mar 2024 14:33:18 +0100 Subject: [PATCH 08/26] updating release index schema --- release-index-shema.json | 59 +++++++++++--------------- release-index.yaml | 89 +++++++++++++++++++--------------------- 2 files changed, 65 insertions(+), 83 deletions(-) diff --git a/release-index-shema.json b/release-index-shema.json index ddb953ad1..24f2de626 100644 --- a/release-index-shema.json +++ b/release-index-shema.json @@ -1,20 +1,14 @@ { "$schema": "http://json-schema.org/draft-06/schema#", - "$ref": "#/definitions/Welcome3", + "$ref": "#/definitions/Welcome8", "definitions": { - "Welcome3": { + "Welcome8": { "type": "object", "additionalProperties": false, "properties": { "rollout": { "$ref": "#/definitions/Rollout" }, - "features": { - "type": "array", - "items": { - "$ref": "#/definitions/Feature" - } - }, "releases": { "type": "array", "items": { @@ -23,47 +17,45 @@ } }, "required": [ - "features", "releases", "rollout" ], - "title": "Welcome3" + "title": "Welcome8" }, - "Feature": { + "Release": { "type": "object", "additionalProperties": false, "properties": { - "name": { + "rc_name": { "type": "string" }, - "subnets": { + "versions": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/Version" } } }, "required": [ - "name", - "subnets" + "rc_name", + "versions" ], - "title": "Feature" + "title": "Release" }, - "Release": { + "Version": { "type": "object", "additionalProperties": false, "properties": { - "rc_name": { + "version": { "type": "string" }, - "version": { + "name": { "type": "string" }, - "publish": { - "type": "string", - "format": "date-time" + "release_notes_ready": { + "type": "boolean" }, - "features": { + "subnets": { "type": "array", "items": { "type": "string" @@ -71,20 +63,16 @@ } }, "required": [ - "features", - "publish", - "rc_name", + "name", + "release_notes_ready", "version" ], - "title": "Release" + "title": "Version" }, "Rollout": { "type": "object", "additionalProperties": false, "properties": { - "rc_name": { - "type": "string" - }, "pause": { "type": "boolean" }, @@ -104,7 +92,6 @@ }, "required": [ "pause", - "rc_name", "skip_days", "stages" ], @@ -123,14 +110,14 @@ "bake_time": { "type": "string" }, + "update_unassigned_nodes": { + "type": "boolean" + }, "wait_for_next_week": { "type": "boolean" } }, - "required": [ - "bake_time", - "subnets" - ], + "required": [], "title": "Stage" } } diff --git a/release-index.yaml b/release-index.yaml index 10ce5edf9..eb4c369f2 100644 --- a/release-index.yaml +++ b/release-index.yaml @@ -1,6 +1,5 @@ rollout: - rc_name: rc--2024-02-21_23-01 # changed in step 4 & 5 - pause: true + pause: false skip_days: - 2024-02-26 stages: @@ -12,7 +11,7 @@ rollout: bake_time: 4h - subnets: [snjp4, w4asl, qxesv] bake_time: 2h - - subnets: [4zbus, ejbmu, 2fq7c] + - subnets: [4zbus, ejbmu, 2fq7c] bake_time: 2h - subnets: [pae4o, 5kdm2, csyj4] bake_time: 2h @@ -20,7 +19,7 @@ rollout: bake_time: 2h - subnets: [k44fs, cv73p, 4ecnw] bake_time: 1h - - subnets: [opn46, lspz2, o3ow2] + - subnets: [opn46, lspz2, o3ow2] bake_time: 1h - subnets: [w4rem, 6pbhf, e66qm] bake_time: 1h @@ -28,52 +27,48 @@ rollout: bake_time: 30m - subnets: [mpubz, x33ed, pzp6e] bake_time: 30m - - subnets: [3hhby, nl6hn, gmq5v] + - subnets: [3hhby, nl6hn, gmq5v] bake_time: 30m - - subnets: [tdb26] + - update_unassigned_nodes: true + - subnets: [tdb26] bake_time: 2h wait_for_next_week: true -features: # can be changed in step 4 $ 5 - - name: p2p - subnets: - - shefu - - qdvhd - - bkfrj - - snjp4 - - qxesv - - w4asl - - 4zbus - - ejbmu - - 2fq7c - - pae4o - - csyj4 - - 5kdm2 - - eq6en - - brlsh - - k44fs - - cv73p - - 4ecnw - - opn46 - - lspz2 - - o3ow2 - - w4rem - - 6pbhf - - e66qm - - jtdsg - - fuqsr - - mpubz - - x33ed - - 3hhby - - nl6hn - releases: # add new release in step 2 - rc_name: rc--2024-02-21_23-01 - version: 85bd56a70e55b2cea75cae6405ae11243e5fdad8 - publish: 2024-02-26T14:20:00+00:00 # changed in step 3 - features: - - p2p - - rc_name: rc--2024-02-21_23-01 - version: 2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f - publish: 2024-02-26T14:20:00+00:00 # changed in step 3 - features: [] \ No newline at end of file + versions: + - version: 2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f + name: rc--2024-02-21_23-01 + release_notes_ready: false + - version: 85bd56a70e55b2cea75cae6405ae11243e5fdad8 + name: p2p + release_notes_ready: false + subnets: + - shefu + - qdvhd + - bkfrj + - snjp4 + - qxesv + - w4asl + - 4zbus + - ejbmu + - 2fq7c + - pae4o + - csyj4 + - 5kdm2 + - eq6en + - brlsh + - k44fs + - cv73p + - 4ecnw + - opn46 + - lspz2 + - o3ow2 + - 6pbhf + - e66qm + - jtdsg + - fuqsr + - mpubz + - x33ed + - 3hhby + - nl6hn \ No newline at end of file From e0f23c0cc831e7c073f56c729757f3371631023b Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 8 Mar 2024 14:54:10 +0100 Subject: [PATCH 09/26] fixing bazel --- Cargo.Bazel.lock | 2 +- WORKSPACE.bazel | 2 +- rs/rollout-controller/BUILD.bazel | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index 47b7eafd7..2cd2b1b38 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "7902360c8c51d31ad0c9e2d339a47134e1b9f6a0c7ae83f75d7f4e1db07d9a5f", + "checksum": "f592b7f11408309a32e1101beaa290cf3b72fc911291d8bae0589e0655ec91d4", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index aac354efc..8e7ba6a07 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -87,7 +87,7 @@ rust_analyzer_dependencies() rust_register_toolchains( edition = "2021", versions = [ - "1.71.1", + "1.75.0", ], ) diff --git a/rs/rollout-controller/BUILD.bazel b/rs/rollout-controller/BUILD.bazel index 1f9d876a3..1e0e57130 100644 --- a/rs/rollout-controller/BUILD.bazel +++ b/rs/rollout-controller/BUILD.bazel @@ -6,7 +6,7 @@ load("@rules_rust//rust:defs.bzl", "rust_binary") DEPS = [ "//rs/ic-observability/service-discovery", "//rs/ic-management-types", -] + "//rs/ic-management-backend:ic-management-backend-lib",] rust_binary( From 0c4ef34a0fe2da47fc6090318bde4a6af9238369 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 11 Mar 2024 12:22:54 +0100 Subject: [PATCH 10/26] started refactoring calculations --- Cargo.Bazel.lock | 6 +- Cargo.lock | 1 + rs/rollout-controller/Cargo.toml | 1 + .../src/calculation/current_release_finder.rs | 97 +++++++++++++++++++ rs/rollout-controller/src/calculation/mod.rs | 70 +++++++++++++ .../src/calculation/should_proceed.rs | 47 +++++++++ .../src/fetching/curl_fetcher.rs | 2 +- rs/rollout-controller/src/fetching/mod.rs | 2 +- .../src/fetching/sparse_checkout_fetcher.rs | 2 +- rs/rollout-controller/src/main.rs | 4 +- 10 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 rs/rollout-controller/src/calculation/current_release_finder.rs create mode 100644 rs/rollout-controller/src/calculation/mod.rs create mode 100644 rs/rollout-controller/src/calculation/should_proceed.rs diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index 2cd2b1b38..eab002829 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "f592b7f11408309a32e1101beaa290cf3b72fc911291d8bae0589e0655ec91d4", + "checksum": "1752fa655d1b64c6a41f586b4679a9dc4e0ebd622211e259c04151e41d5a3edd", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -37224,6 +37224,10 @@ "id": "prometheus-http-query 0.8.2", "target": "prometheus_http_query" }, + { + "id": "regex 1.10.3", + "target": "regex" + }, { "id": "reqwest 0.11.24", "target": "reqwest" diff --git a/Cargo.lock b/Cargo.lock index 463cd52e6..c1e855546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7408,6 +7408,7 @@ dependencies = [ "ic-registry-keys", "ic-registry-local-registry", "prometheus-http-query", + "regex", "reqwest", "serde", "serde_json", diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index c1c092ad6..aea758d3c 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -32,4 +32,5 @@ url = { workspace = true } prometheus-http-query = { workspace = true } humantime-serde = "1.1.1" reqwest = { workspace = true } +regex = { workspace = true } diff --git a/rs/rollout-controller/src/calculation/current_release_finder.rs b/rs/rollout-controller/src/calculation/current_release_finder.rs new file mode 100644 index 000000000..c9ed30d2a --- /dev/null +++ b/rs/rollout-controller/src/calculation/current_release_finder.rs @@ -0,0 +1,97 @@ +use chrono::NaiveDateTime; +use regex::Regex; + +use super::{Index, Release}; + +pub fn find_latest_release(index: &Index) -> anyhow::Result { + let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); + + let mut mapped: Vec<(Release, NaiveDateTime)> = index + .releases + .iter() + .cloned() + .filter_map(|release| { + let captures = match regex.captures(&release.rc_name) { + Some(captures) => captures, + None => return None, + }; + let datetime_str = captures.name("datetime").unwrap().as_str(); + let datetime = match NaiveDateTime::parse_from_str(datetime_str, "%Y-%m-%d_%H-%M") { + Ok(val) => val, + Err(_) => return None, + }; + Some((release, datetime)) + }) + .collect(); + + mapped.sort_by_key(|(_, datetime)| *datetime); + mapped.reverse(); + + match mapped.first() { + Some((found, _)) => Ok(found.clone()), + None => Err(anyhow::anyhow!("There aren't any releases that match the criteria")), + } +} + +#[cfg(test)] +mod find_latest_release_tests { + use super::*; + + #[test] + fn should_not_find_release_none_match_regex() { + let index = Index { + releases: vec![ + Release { + rc_name: String::from("bad-name"), + versions: Default::default(), + }, + Release { + rc_name: String::from("rc--kind-of-ok_no-no"), + versions: Default::default(), + }, + ], + ..Default::default() + }; + + let latest = find_latest_release(&index); + + assert!(latest.is_err()); + } + + #[test] + fn should_return_latest_correct_release() { + let index = Index { + releases: vec![ + Release { + rc_name: String::from("rc--kind-of-ok_no-no"), + ..Default::default() + }, + Release { + rc_name: String::from("rc--2024-03-09_23-01"), + ..Default::default() + }, + Release { + rc_name: String::from("rc--2024-03-10_23-01"), + ..Default::default() + }, + ], + ..Default::default() + }; + + let latest = find_latest_release(&index); + + assert!(latest.is_ok()); + let latest = latest.unwrap(); + + assert_eq!(latest.rc_name, String::from("rc--2024-03-10_23-01")) + } + + #[test] + fn should_not_return_release_empty_index() { + let index = Index { ..Default::default() }; + + let latest = find_latest_release(&index); + + assert!(latest.is_err()) + } +} diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs new file mode 100644 index 000000000..f9c1a1649 --- /dev/null +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -0,0 +1,70 @@ +use std::time::Duration; + +use crate::calculation::should_proceed::should_proceed; +use chrono::{Local, NaiveDate}; +use ic_management_backend::registry::RegistryState; +use prometheus_http_query::Client; +use serde::Deserialize; +use slog::{info, Logger}; + +use self::current_release_finder::find_latest_release; + +mod current_release_finder; +mod should_proceed; + +#[derive(Deserialize, Clone, Default)] +pub struct Index { + pub rollout: Rollout, + pub releases: Vec, +} + +#[derive(Deserialize, Clone, Default)] +pub struct Rollout { + #[serde(default)] + pub pause: bool, + pub skip_days: Vec, + pub stages: Vec, +} + +#[derive(Deserialize, Default, Clone)] +#[serde(default)] + +pub struct Stage { + pub subnets: Vec, + #[serde(with = "humantime_serde")] + pub bake_time: Duration, + pub wait_for_next_week: bool, + update_unassigned_nodes: bool, +} + +#[derive(Deserialize, Clone, Default)] +pub struct Release { + pub rc_name: String, + pub versions: Vec, +} + +#[derive(Deserialize, Clone, Default)] +pub struct Version { + pub version: String, + pub name: String, + #[serde(default)] + pub release_notes_read: bool, + #[serde(default)] + pub subnets: Vec, +} + +pub async fn calculate_progress( + logger: &Logger, + index: Index, + _prometheus_client: &Client, + _registry_state: RegistryState, +) -> anyhow::Result> { + if !should_proceed(&index, Local::now().to_utc().date_naive()) { + info!(logger, "Rollout controller paused or should skip this day."); + return Ok(vec![]); + } + + let _latest_release = find_latest_release(&index)?; + + Ok(vec![]) +} diff --git a/rs/rollout-controller/src/calculation/should_proceed.rs b/rs/rollout-controller/src/calculation/should_proceed.rs new file mode 100644 index 000000000..5a2d6157a --- /dev/null +++ b/rs/rollout-controller/src/calculation/should_proceed.rs @@ -0,0 +1,47 @@ +use chrono::{Local, NaiveDate}; + +use super::Index; + +pub fn should_proceed(index: &Index, today: NaiveDate) -> bool { + // Check if the plan is paused + if index.rollout.pause { + return false; + } + + // Check if this day should be skipped + if index.rollout.skip_days.iter().any(|f| f.eq(&today)) { + return false; + } + + true +} + +#[cfg(test)] +mod should_proceed_tests { + use std::str::FromStr; + + use crate::calculation::Rollout; + + use super::*; + + #[test] + fn should_proceed_not_blocked_and_not_skipped() { + let index = Index::default(); + + assert!(should_proceed(&index, Local::now().to_utc().date_naive())) + } + + #[test] + fn should_not_proceed_skipped_day() { + let day = NaiveDate::from_str("2024-03-11").unwrap(); + let index = Index { + rollout: Rollout { + skip_days: vec![day.clone()], + ..Default::default() + }, + ..Default::default() + }; + + assert!(!should_proceed(&index, day)) + } +} diff --git a/rs/rollout-controller/src/fetching/curl_fetcher.rs b/rs/rollout-controller/src/fetching/curl_fetcher.rs index 9846f21a0..84765b5db 100644 --- a/rs/rollout-controller/src/fetching/curl_fetcher.rs +++ b/rs/rollout-controller/src/fetching/curl_fetcher.rs @@ -35,7 +35,7 @@ impl CurlFetcher { } impl RolloutScheduleFetcher for CurlFetcher { - async fn fetch(&self) -> anyhow::Result { + async fn fetch(&self) -> anyhow::Result { debug!(self.logger, "Fetching rollout index"); let response = self diff --git a/rs/rollout-controller/src/fetching/mod.rs b/rs/rollout-controller/src/fetching/mod.rs index b475bcce8..3fb9b18df 100644 --- a/rs/rollout-controller/src/fetching/mod.rs +++ b/rs/rollout-controller/src/fetching/mod.rs @@ -1,6 +1,6 @@ use slog::Logger; -use crate::{rollout_schedule::Index, Commands}; +use crate::{calculation::Index, Commands}; use self::{ curl_fetcher::{CurlFetcher, CurlFetcherConfig}, diff --git a/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs b/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs index 103b5d580..1724befdd 100644 --- a/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs +++ b/rs/rollout-controller/src/fetching/sparse_checkout_fetcher.rs @@ -8,7 +8,7 @@ use tokio::{ process::Command, }; -use crate::rollout_schedule::Index; +use crate::calculation::Index; use super::RolloutScheduleFetcher; diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index 8ce71c3b1..bc2e319e6 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -10,11 +10,11 @@ use tokio::select; use tokio_util::sync::CancellationToken; use url::Url; -use crate::{registry_wrappers::sync_wrap, rollout_schedule::calculate_progress}; +use crate::{calculation::calculate_progress, registry_wrappers::sync_wrap}; +mod calculation; mod fetching; mod registry_wrappers; -mod rollout_schedule; #[tokio::main] async fn main() -> anyhow::Result<()> { From cd4bfe9ef908ff69dbdd8decf7d03bbe5533f387 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 11 Mar 2024 14:34:28 +0100 Subject: [PATCH 11/26] moved calculations to a new place --- .../src/calculation/current_release_finder.rs | 97 ---- rs/rollout-controller/src/calculation/mod.rs | 62 ++- .../src/calculation/release_actions.rs | 452 ++++++++++++++++++ .../src/calculation/should_proceed.rs | 4 +- .../src/calculation/stage_checks.rs | 145 ++++++ rs/rollout-controller/src/rollout_schedule.rs | 243 ---------- 6 files changed, 655 insertions(+), 348 deletions(-) delete mode 100644 rs/rollout-controller/src/calculation/current_release_finder.rs create mode 100644 rs/rollout-controller/src/calculation/release_actions.rs create mode 100644 rs/rollout-controller/src/calculation/stage_checks.rs delete mode 100644 rs/rollout-controller/src/rollout_schedule.rs diff --git a/rs/rollout-controller/src/calculation/current_release_finder.rs b/rs/rollout-controller/src/calculation/current_release_finder.rs deleted file mode 100644 index c9ed30d2a..000000000 --- a/rs/rollout-controller/src/calculation/current_release_finder.rs +++ /dev/null @@ -1,97 +0,0 @@ -use chrono::NaiveDateTime; -use regex::Regex; - -use super::{Index, Release}; - -pub fn find_latest_release(index: &Index) -> anyhow::Result { - let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); - - let mut mapped: Vec<(Release, NaiveDateTime)> = index - .releases - .iter() - .cloned() - .filter_map(|release| { - let captures = match regex.captures(&release.rc_name) { - Some(captures) => captures, - None => return None, - }; - let datetime_str = captures.name("datetime").unwrap().as_str(); - let datetime = match NaiveDateTime::parse_from_str(datetime_str, "%Y-%m-%d_%H-%M") { - Ok(val) => val, - Err(_) => return None, - }; - Some((release, datetime)) - }) - .collect(); - - mapped.sort_by_key(|(_, datetime)| *datetime); - mapped.reverse(); - - match mapped.first() { - Some((found, _)) => Ok(found.clone()), - None => Err(anyhow::anyhow!("There aren't any releases that match the criteria")), - } -} - -#[cfg(test)] -mod find_latest_release_tests { - use super::*; - - #[test] - fn should_not_find_release_none_match_regex() { - let index = Index { - releases: vec![ - Release { - rc_name: String::from("bad-name"), - versions: Default::default(), - }, - Release { - rc_name: String::from("rc--kind-of-ok_no-no"), - versions: Default::default(), - }, - ], - ..Default::default() - }; - - let latest = find_latest_release(&index); - - assert!(latest.is_err()); - } - - #[test] - fn should_return_latest_correct_release() { - let index = Index { - releases: vec![ - Release { - rc_name: String::from("rc--kind-of-ok_no-no"), - ..Default::default() - }, - Release { - rc_name: String::from("rc--2024-03-09_23-01"), - ..Default::default() - }, - Release { - rc_name: String::from("rc--2024-03-10_23-01"), - ..Default::default() - }, - ], - ..Default::default() - }; - - let latest = find_latest_release(&index); - - assert!(latest.is_ok()); - let latest = latest.unwrap(); - - assert_eq!(latest.rc_name, String::from("rc--2024-03-10_23-01")) - } - - #[test] - fn should_not_return_release_empty_index() { - let index = Index { ..Default::default() }; - - let latest = find_latest_release(&index); - - assert!(latest.is_err()) - } -} diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index f9c1a1649..b7773e812 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{collections::BTreeMap, time::Duration}; use crate::calculation::should_proceed::should_proceed; use chrono::{Local, NaiveDate}; @@ -7,10 +7,14 @@ use prometheus_http_query::Client; use serde::Deserialize; use slog::{info, Logger}; -use self::current_release_finder::find_latest_release; +use self::{ + release_actions::{create_current_release_feature_spec, find_latest_release}, + stage_checks::check_stages, +}; -mod current_release_finder; +mod release_actions; mod should_proceed; +mod stage_checks; #[derive(Deserialize, Clone, Default)] pub struct Index { @@ -56,15 +60,59 @@ pub struct Version { pub async fn calculate_progress( logger: &Logger, index: Index, - _prometheus_client: &Client, - _registry_state: RegistryState, + prometheus_client: &Client, + registry_state: RegistryState, ) -> anyhow::Result> { if !should_proceed(&index, Local::now().to_utc().date_naive()) { info!(logger, "Rollout controller paused or should skip this day."); return Ok(vec![]); } - let _latest_release = find_latest_release(&index)?; + let latest_release = find_latest_release(&index)?; + let elected_versions = registry_state.get_blessed_replica_versions().await?; - Ok(vec![]) + let (current_version, current_feature_spec) = + create_current_release_feature_spec(&latest_release, elected_versions) + .map_err(|e| anyhow::anyhow!(e.to_string()))?; + + let mut last_bake_status: BTreeMap = BTreeMap::new(); + let result = prometheus_client + .query( + r#" + time() - max(last_over_time( + (timestamp( + sum by(ic_active_version,ic_subnet) (ic_replica_info) + ))[21d:1m] + ) unless (sum by (ic_active_version, ic_subnet) (ic_replica_info))) by (ic_subnet) + "#, + ) + .get() + .await?; + + let last = match result.data().clone().into_vector().into_iter().last() { + Some(data) => data, + None => return Err(anyhow::anyhow!("There should be data regarding ic_replica_info")), + }; + + for vector in last.iter() { + let subnet = vector.metric().get("ic_subnet").expect("To have ic_subnet key"); + let last_update = vector.sample().value(); + last_bake_status.insert(subnet.to_string(), last_update); + } + + let subnet_update_proposals = registry_state.open_subnet_upgrade_proposals().await?; + let unassigned_nodes_version = registry_state.get_unassigned_nodes_replica_version().await?; + + let actions = check_stages( + ¤t_version, + current_feature_spec, + last_bake_status, + subnet_update_proposals, + &index.rollout.stages, + Some(&logger), + &unassigned_nodes_version, + ®istry_state.subnets().into_values().collect(), + )?; + + Ok(actions) } diff --git a/rs/rollout-controller/src/calculation/release_actions.rs b/rs/rollout-controller/src/calculation/release_actions.rs new file mode 100644 index 000000000..4202a7ac0 --- /dev/null +++ b/rs/rollout-controller/src/calculation/release_actions.rs @@ -0,0 +1,452 @@ +use std::{collections::BTreeMap, fmt::Display}; + +use chrono::NaiveDateTime; +use regex::Regex; + +use super::{Index, Release}; + +pub fn find_latest_release(index: &Index) -> anyhow::Result { + let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); + + let mut mapped: Vec<(Release, NaiveDateTime)> = index + .releases + .iter() + .cloned() + .filter_map(|release| { + let captures = match regex.captures(&release.rc_name) { + Some(captures) => captures, + None => return None, + }; + let datetime_str = captures.name("datetime").unwrap().as_str(); + let datetime = match NaiveDateTime::parse_from_str(datetime_str, "%Y-%m-%d_%H-%M") { + Ok(val) => val, + Err(_) => return None, + }; + Some((release, datetime)) + }) + .collect(); + + mapped.sort_by_key(|(_, datetime)| *datetime); + mapped.reverse(); + + match mapped.first() { + Some((found, _)) => Ok(found.clone()), + None => Err(anyhow::anyhow!("There aren't any releases that match the criteria")), + } +} + +pub fn create_current_release_feature_spec( + current_release: &Release, + blessed_versions: Vec, +) -> Result<(String, BTreeMap>), CreateCurrentReleaseFeatureSpecError> { + let mut current_release_feature_spec: BTreeMap> = BTreeMap::new(); + let mut current_release_version = "".to_string(); + + for version in ¤t_release.versions { + if !blessed_versions.contains(&version.version) { + return Err(CreateCurrentReleaseFeatureSpecError::VersionNotBlessed { + rc: current_release.rc_name.to_string(), + version_name: version.name.to_string(), + version: version.version.to_string(), + }); + } + + if version.name.eq(¤t_release.rc_name) { + if current_release_version.is_empty() { + current_release_version = version.version.to_string(); + continue; + } + + // Version override attempt. Shouldn't be possible + return Err(CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { + rc: current_release.rc_name.to_string(), + version_name: version.name.to_string(), + }); + } + + if !version.name.eq(¤t_release.rc_name) && version.subnets.is_empty() { + return Err(CreateCurrentReleaseFeatureSpecError::FeatureBuildNoSubnets { + rc: current_release.rc_name.to_string(), + version_name: version.name.to_string(), + }); + } + + if current_release_feature_spec.contains_key(&version.version) { + return Err(CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { + rc: current_release.rc_name.to_string(), + version_name: version.name.to_string(), + }); + } + + current_release_feature_spec.insert(version.version.to_string(), version.subnets.clone()); + } + + if current_release_version.is_empty() { + return Err(CreateCurrentReleaseFeatureSpecError::CurrentVersionNotFound { + rc: current_release.rc_name.to_string(), + }); + } + + Ok((current_release_version, current_release_feature_spec)) +} + +#[derive(PartialEq, Debug)] +pub enum CreateCurrentReleaseFeatureSpecError { + CurrentVersionNotFound { + rc: String, + }, + FeatureBuildNoSubnets { + rc: String, + version_name: String, + }, + VersionNotBlessed { + rc: String, + version_name: String, + version: String, + }, + VersionSpecifiedTwice { + rc: String, + version_name: String, + }, +} + +impl Display for CreateCurrentReleaseFeatureSpecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CreateCurrentReleaseFeatureSpecError::CurrentVersionNotFound { rc } => f.write_fmt(format_args!( + "Regular release version not found for release named: {}", + rc + )), + CreateCurrentReleaseFeatureSpecError::FeatureBuildNoSubnets { rc, version_name } => { + f.write_fmt(format_args!( + "Feature build '{}' that is part of rc '{}' doesn't have subnets specified", + version_name, rc + )) + } + CreateCurrentReleaseFeatureSpecError::VersionNotBlessed { + rc, + version, + version_name, + } => f.write_fmt(format_args!( + "Version '{}', named '{}' that is part of rc '{}' is not blessed", + version, version_name, rc + )), + CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { rc, version_name } => f.write_fmt( + format_args!("Version '{}' is defined twice within rc named '{}'", version_name, rc), + ), + } + } +} + +#[cfg(test)] +mod find_latest_release_tests { + use super::*; + + #[test] + fn should_not_find_release_none_match_regex() { + let index = Index { + releases: vec![ + Release { + rc_name: String::from("bad-name"), + versions: Default::default(), + }, + Release { + rc_name: String::from("rc--kind-of-ok_no-no"), + versions: Default::default(), + }, + ], + ..Default::default() + }; + + let latest = find_latest_release(&index); + + assert!(latest.is_err()); + } + + #[test] + fn should_return_latest_correct_release() { + let index = Index { + releases: vec![ + Release { + rc_name: String::from("rc--kind-of-ok_no-no"), + ..Default::default() + }, + Release { + rc_name: String::from("rc--2024-03-09_23-01"), + ..Default::default() + }, + Release { + rc_name: String::from("rc--2024-03-10_23-01"), + ..Default::default() + }, + ], + ..Default::default() + }; + + let latest = find_latest_release(&index); + + assert!(latest.is_ok()); + let latest = latest.unwrap(); + + assert_eq!(latest.rc_name, String::from("rc--2024-03-10_23-01")) + } + + #[test] + fn should_not_return_release_empty_index() { + let index = Index { ..Default::default() }; + + let latest = find_latest_release(&index); + + assert!(latest.is_err()) + } +} + +#[cfg(test)] +mod create_current_release_feature_spec_tests { + use crate::calculation::Version; + + use super::*; + + #[test] + fn should_create_map() { + let blessed_versions = vec![ + "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", + "85bd56a70e55b2cea75cae6405ae11243e5fdad8", + ] + .iter() + .map(|v| v.to_string()) + .collect::>(); + let current_release = Release { + rc_name: "rc--2024-03-10_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-03-10_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01-p2p".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), + ..Default::default() + }, + ], + }; + + let response = create_current_release_feature_spec(¤t_release, blessed_versions); + + assert!(response.is_ok()); + let (current_version, feat_map) = response.unwrap(); + assert_eq!(current_version, "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string()); + assert_eq!(feat_map.len(), 1); + + let (version, subnets) = feat_map.first_key_value().unwrap(); + assert_eq!(version, "85bd56a70e55b2cea75cae6405ae11243e5fdad8"); + assert_eq!(subnets.len(), 1); + let subnet = subnets.first().unwrap(); + assert_eq!(subnet, "shefu"); + } + + #[test] + fn shouldnt_create_map_regular_version_not_blessed() { + let blessed_versions = vec!["85bd56a70e55b2cea75cae6405ae11243e5fdad8"] + .iter() + .map(|v| v.to_string()) + .collect::>(); + + let current_release = Release { + rc_name: "rc--2024-03-10_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-03-10_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01-p2p".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), + ..Default::default() + }, + ], + }; + + let response = create_current_release_feature_spec(¤t_release, blessed_versions); + + assert!(response.is_err()); + let error = response.err().unwrap(); + assert_eq!( + error, + CreateCurrentReleaseFeatureSpecError::VersionNotBlessed { + rc: "rc--2024-03-10_23-01".to_string(), + version_name: "rc--2024-03-10_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string() + } + ) + } + + #[test] + fn shouldnt_create_map_version_override() { + let blessed_versions = vec![ + "85bd56a70e55b2cea75cae6405ae11243e5fdad8", + "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", + ] + .iter() + .map(|v| v.to_string()) + .collect::>(); + + let current_release = Release { + rc_name: "rc--2024-03-10_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-03-10_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), + ..Default::default() + }, + ], + }; + + let response = create_current_release_feature_spec(¤t_release, blessed_versions); + + assert!(response.is_err()); + let error = response.err().unwrap(); + assert_eq!( + error, + CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { + rc: "rc--2024-03-10_23-01".to_string(), + version_name: "rc--2024-03-10_23-01".to_string() + } + ) + } + + #[test] + fn shouldnt_create_map_version_override_for_feature_builds() { + let blessed_versions = vec![ + "85bd56a70e55b2cea75cae6405ae11243e5fdad8", + "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", + ] + .iter() + .map(|v| v.to_string()) + .collect::>(); + + let current_release = Release { + rc_name: "rc--2024-03-10_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-03-10_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01-p2p".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01-p2p2".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), + ..Default::default() + }, + ], + }; + + let response = create_current_release_feature_spec(¤t_release, blessed_versions); + + assert!(response.is_err()); + let error = response.err().unwrap(); + assert_eq!( + error, + CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { + rc: "rc--2024-03-10_23-01".to_string(), + version_name: "rc--2024-03-10_23-01-p2p2".to_string() + } + ) + } + + #[test] + fn shouldnt_create_map_version_no_subnets_for_feature() { + let blessed_versions = vec![ + "85bd56a70e55b2cea75cae6405ae11243e5fdad8", + "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", + ] + .iter() + .map(|v| v.to_string()) + .collect::>(); + + let current_release = Release { + rc_name: "rc--2024-03-10_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-03-10_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01-p2p".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }, + ], + }; + + let response = create_current_release_feature_spec(¤t_release, blessed_versions); + + assert!(response.is_err()); + let error = response.err().unwrap(); + assert_eq!( + error, + CreateCurrentReleaseFeatureSpecError::FeatureBuildNoSubnets { + rc: "rc--2024-03-10_23-01".to_string(), + version_name: "rc--2024-03-10_23-01-p2p".to_string() + } + ) + } + + #[test] + fn shouldnt_create_map_version_no_regular_build() { + let blessed_versions = vec![ + "85bd56a70e55b2cea75cae6405ae11243e5fdad8", + "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", + ] + .iter() + .map(|v| v.to_string()) + .collect::>(); + + let current_release = Release { + rc_name: "rc--2024-03-10_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-03-10_23-01-notregular".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + subnets: vec!["shefu".to_string()], + ..Default::default() + }, + Version { + name: "rc--2024-03-10_23-01-p2p".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + subnets: vec!["qdvhd".to_string()], + ..Default::default() + }, + ], + }; + + let response = create_current_release_feature_spec(¤t_release, blessed_versions); + + assert!(response.is_err()); + let error = response.err().unwrap(); + assert_eq!( + error, + CreateCurrentReleaseFeatureSpecError::CurrentVersionNotFound { + rc: "rc--2024-03-10_23-01".to_string(), + } + ) + } +} diff --git a/rs/rollout-controller/src/calculation/should_proceed.rs b/rs/rollout-controller/src/calculation/should_proceed.rs index 5a2d6157a..c4b9c3d3b 100644 --- a/rs/rollout-controller/src/calculation/should_proceed.rs +++ b/rs/rollout-controller/src/calculation/should_proceed.rs @@ -1,4 +1,4 @@ -use chrono::{Local, NaiveDate}; +use chrono::NaiveDate; use super::Index; @@ -20,6 +20,8 @@ pub fn should_proceed(index: &Index, today: NaiveDate) -> bool { mod should_proceed_tests { use std::str::FromStr; + use chrono::Local; + use crate::calculation::Rollout; use super::*; diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs new file mode 100644 index 000000000..7febf3fbb --- /dev/null +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -0,0 +1,145 @@ +use std::{collections::BTreeMap, time::Duration}; + +use humantime::format_duration; +use ic_management_backend::proposal::SubnetUpdateProposal; +use ic_management_types::Subnet; +use slog::{debug, info, Logger}; + +use super::Stage; + +pub fn check_stages( + current_version: &String, + current_release_feature_spec: BTreeMap>, + last_bake_status: BTreeMap, + subnet_update_proposals: Vec, + stages: &Vec, + logger: Option<&Logger>, + unassigned_version: &String, + subnets: &Vec, +) -> anyhow::Result> { + for (i, stage) in stages.iter().enumerate() { + if let Some(logger) = logger { + info!(logger, "### Checking stage {}", i); + } + let mut actions = vec![]; + let mut stage_baked = true; + + if stage.update_unassigned_nodes { + if let Some(logger) = logger { + debug!(logger, "Unassigned nodes stage"); + } + + let unassigned_nodes_version = unassigned_version; + + if unassigned_nodes_version.eq(current_version) { + if let Some(logger) = logger { + info!(logger, "Unassigned version already at version '{}'", current_version); + } + continue; + } + // Should update unassigned nodes + actions.push(format!("Update unassigned nodes to version: {}", current_version)); + return Ok(actions); + } + + if let Some(logger) = logger { + debug!(logger, "Regular nodes stage"); + } + for subnet_short in &stage.subnets { + let desired_version = match current_release_feature_spec + .iter() + .find(|(_, subnets)| subnets.contains(subnet_short)) + { + Some((name, _)) => name, + None => current_version, + }; + + if let Some(logger) = logger { + debug!( + logger, + "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version + ); + } + + let subnet = match subnets + .into_iter() + .find(|key| key.principal.to_string().starts_with(subnet_short)) + { + Some(subnet) => subnet, + None => { + return Err(anyhow::anyhow!( + "Couldn't find subnet that starts with '{}'", + subnet_short + )) + } + }; + + if subnet.replica_version.eq(desired_version) { + if let Some(logger) = logger { + info!(logger, "Subnet {} is at desired version", subnet_short); + } + // Check bake time + let bake = last_bake_status + .get(&subnet.principal.to_string()) + .expect("Should have key"); + if bake.gt(&stage.bake_time.as_secs_f64()) { + if let Some(logger) = logger { + info!(logger, "Subnet {} is baked", subnet_short); + } + continue; + } else { + let remaining = Duration::from_secs_f64(stage.bake_time.as_secs_f64() - bake); + let formatted = format_duration(remaining); + if let Some(logger) = logger { + info!( + logger, + "Waiting for subnet {} to bake, pending {}", subnet_short, formatted + ); + } + stage_baked |= false; + } + } + if let Some(logger) = logger { + info!(logger, "Subnet {} is not at desired version", subnet_short); + } + // Check if there is an open proposal => if yes, wait; if false, place and exit + + if let Some(proposal) = subnet_update_proposals.iter().find(|proposal| { + proposal.payload.subnet_id == subnet.principal + && proposal.payload.replica_version_id.eq(desired_version) + && !proposal.info.executed + }) { + if let Some(logger) = logger { + info!( + logger, + "For subnet '{}' found open proposal 'https://dashboard.internetcomputer.org/proposal/{}'", + subnet_short, + proposal.info.id + ); + } + continue; + } + + actions.push(format!( + "Should update subnet '{}' to version '{}'", + subnet_short, desired_version + )) + } + + if !stage_baked { + return Ok(vec![]); + } + + if !actions.is_empty() { + return Ok(actions); + } + + if let Some(logger) = logger { + info!(logger, "### Stage {} completed", i) + } + } + if let Some(logger) = logger { + info!(logger, "The current rollout '{}' is completed.", current_version); + } + Ok(vec![]) +} diff --git a/rs/rollout-controller/src/rollout_schedule.rs b/rs/rollout-controller/src/rollout_schedule.rs deleted file mode 100644 index aa6ab12b9..000000000 --- a/rs/rollout-controller/src/rollout_schedule.rs +++ /dev/null @@ -1,243 +0,0 @@ -use std::{collections::BTreeMap, time::Duration}; - -use anyhow::Ok; -use chrono::{Local, NaiveDate}; -use humantime::format_duration; -use ic_management_backend::registry::RegistryState; -use prometheus_http_query::Client; -use serde::Deserialize; -use slog::{debug, info, Logger}; - -pub async fn calculate_progress( - logger: &Logger, - index: Index, - prometheus_client: &Client, - registry_state: RegistryState, -) -> anyhow::Result> { - // Check if the plan is paused - if index.rollout.pause { - info!(logger, "Release is paused, no progress to be made."); - return Ok(vec![]); - } - - // Check if this day should be skipped - let today = Local::now().to_utc().date_naive(); - if index.rollout.skip_days.iter().any(|f| f.eq(&today)) { - info!(logger, "'{}' should be skipped as per rollout skip days", today); - return Ok(vec![]); - } - - debug!(logger, "Fetching elected versions"); - let elected_versions = registry_state.get_blessed_replica_versions().await?; - let current_release = match index.releases.first() { - Some(release) => release, - None => return Err(anyhow::anyhow!("Expected to have atleast one release")), - }; - - debug!(logger, "Checking if versions are elected"); - let mut current_release_feature_spec: BTreeMap> = BTreeMap::new(); - let mut current_version = String::from(""); - for version in ¤t_release.versions { - if !elected_versions.contains(&version.version) { - return Err(anyhow::anyhow!( - "Version '{}' is not blessed and is required for release '{}' with build name: {}", - version.version, - current_release.rc_name, - version.name - )); - } - - if version.name.eq(¤t_release.rc_name) { - current_version = version.version.to_string(); - continue; - } - - if !version.name.eq(¤t_release.rc_name) && version.subnets.is_empty() { - return Err(anyhow::anyhow!( - "Feature build '{}' doesn't have subnets specified", - version.name - )); - } - - current_release_feature_spec.insert(version.version.to_string(), version.subnets.clone()); - } - - let mut last_bake_status: BTreeMap = BTreeMap::new(); - let result = prometheus_client - .query( - r#" - time() - max(last_over_time( - (timestamp( - sum by(ic_active_version,ic_subnet) (ic_replica_info) - ))[21d:1m] - ) unless (sum by (ic_active_version, ic_subnet) (ic_replica_info))) by (ic_subnet) - "#, - ) - .get() - .await?; - - let last = match result.data().clone().into_vector().into_iter().last() { - Some(data) => data, - None => return Err(anyhow::anyhow!("There should be data regarding ic_replica_info")), - }; - - for vector in last.iter() { - let subnet = vector.metric().get("ic_subnet").expect("To have ic_subnet key"); - let last_update = vector.sample().value(); - last_bake_status.insert(subnet.to_string(), last_update); - } - - let subnet_update_proposals = registry_state.open_subnet_upgrade_proposals().await?; - - for (i, stage) in index.rollout.stages.iter().enumerate() { - info!(logger, "### Checking stage {}", i); - let mut actions = vec![]; - let mut stage_baked = true; - - if stage.update_unassigned_nodes { - debug!(logger, "Unassigned nodes stage"); - - let unassigned_nodes_version = registry_state.get_unassigned_nodes_replica_version().await?; - - if unassigned_nodes_version.eq(¤t_version) { - info!(logger, "Unassigned version already at version '{}'", current_version); - continue; - } - // Should update unassigned nodes - actions.push(format!("Update unassigned nodes to version: {}", current_version)); - return Ok(actions); - } - - debug!(logger, "Regular nodes stage"); - for subnet_short in &stage.subnets { - let desired_version = match current_release_feature_spec - .iter() - .find(|(_, subnets)| subnets.contains(subnet_short)) - { - Some((name, _)) => name, - None => ¤t_version, - }; - debug!( - logger, - "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version - ); - - let subnet = match registry_state - .subnets() - .into_iter() - .find(|(key, _)| key.to_string().starts_with(subnet_short)) - { - Some((_, subnet)) => subnet, - None => { - return Err(anyhow::anyhow!( - "Couldn't find subnet that starts with '{}'", - subnet_short - )) - } - }; - - if subnet.replica_version.eq(desired_version) { - info!(logger, "Subnet {} is at desired version", subnet_short); - // Check bake time - let bake = last_bake_status - .get(&subnet.principal.to_string()) - .expect("Should have key"); - if bake.gt(&stage.bake_time.as_secs_f64()) { - info!(logger, "Subnet {} is baked", subnet_short); - continue; - } else { - let remaining = Duration::from_secs_f64(stage.bake_time.as_secs_f64() - bake); - let formatted = format_duration(remaining); - info!( - logger, - "Waiting for subnet {} to bake, pending {}", subnet_short, formatted - ); - stage_baked |= false; - } - } - info!(logger, "Subnet {} is not at desired version", subnet_short); - // Check if there is an open proposal => if yes, wait; if false, place and exit - - if let Some(proposal) = subnet_update_proposals.iter().find(|proposal| { - proposal.payload.subnet_id == subnet.principal - && proposal.payload.replica_version_id.eq(desired_version) - && !proposal.info.executed - }) { - info!( - logger, - "For subnet '{}' found open proposal 'https://dashboard.internetcomputer.org/proposal/{}'", - subnet_short, - proposal.info.id - ); - continue; - } - - actions.push(format!( - "Should update subnet '{}' to version '{}'", - subnet_short, desired_version - )) - } - - if !stage_baked { - return Ok(vec![]); - } - - if !actions.is_empty() { - return Ok(actions); - } - - info!(logger, "### Stage {} completed", i) - } - - // Iterate over stages and compare desired state from file and the state it is live - // Take into consideration: - // 1. If the subnet is on the desired version, proceed - // 2. If the subnet isn't on the desired version: - // 2.1. Check if there is an open proposal for upgrading, if there isn't place one - // 2.2. If the proposal is executed check when it was upgraded and query prometheus - // to see if there were any alerts and if the bake time has passed. If the bake - // time didn't pass don't do anything. If there were alerts don't do anything. - - Ok(vec![]) -} - -#[derive(Deserialize)] -pub struct Index { - pub rollout: Rollout, - pub releases: Vec, -} - -#[derive(Deserialize)] -pub struct Rollout { - #[serde(default)] - pub pause: bool, - pub skip_days: Vec, - pub stages: Vec, -} - -#[derive(Deserialize, Default)] -#[serde(default)] - -pub struct Stage { - pub subnets: Vec, - #[serde(with = "humantime_serde")] - pub bake_time: Duration, - pub wait_for_next_week: bool, - update_unassigned_nodes: bool, -} - -#[derive(Deserialize)] -pub struct Release { - pub rc_name: String, - pub versions: Vec, -} - -#[derive(Deserialize)] -pub struct Version { - pub version: String, - pub name: String, - #[serde(default)] - pub release_notes_read: bool, - #[serde(default)] - pub subnets: Vec, -} From 7522e40206183300a501ea63e8563c34054a1635 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Tue, 12 Mar 2024 13:20:00 +0100 Subject: [PATCH 12/26] refactored --- rs/rollout-controller/src/calculation/mod.rs | 19 +- .../src/calculation/stage_checks.rs | 302 +++++++++++------- 2 files changed, 203 insertions(+), 118 deletions(-) diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index b7773e812..89dec1406 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -3,13 +3,14 @@ use std::{collections::BTreeMap, time::Duration}; use crate::calculation::should_proceed::should_proceed; use chrono::{Local, NaiveDate}; use ic_management_backend::registry::RegistryState; +use ic_management_types::Subnet; use prometheus_http_query::Client; use serde::Deserialize; use slog::{info, Logger}; use self::{ release_actions::{create_current_release_feature_spec, find_latest_release}, - stage_checks::check_stages, + stage_checks::{check_stages, SubnetAction}, }; mod release_actions; @@ -57,12 +58,12 @@ pub struct Version { pub subnets: Vec, } -pub async fn calculate_progress( - logger: &Logger, +pub async fn calculate_progress<'a>( + logger: &'a Logger, index: Index, - prometheus_client: &Client, + prometheus_client: &'a Client, registry_state: RegistryState, -) -> anyhow::Result> { +) -> anyhow::Result> { if !should_proceed(&index, Local::now().to_utc().date_naive()) { info!(logger, "Rollout controller paused or should skip this day."); return Ok(vec![]); @@ -105,13 +106,13 @@ pub async fn calculate_progress( let actions = check_stages( ¤t_version, - current_feature_spec, - last_bake_status, - subnet_update_proposals, + ¤t_feature_spec, + &last_bake_status, + &subnet_update_proposals, &index.rollout.stages, Some(&logger), &unassigned_nodes_version, - ®istry_state.subnets().into_values().collect(), + ®istry_state.subnets().into_values().collect::>(), )?; Ok(actions) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 7febf3fbb..fc260aa43 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -7,139 +7,223 @@ use slog::{debug, info, Logger}; use super::Stage; -pub fn check_stages( - current_version: &String, - current_release_feature_spec: BTreeMap>, - last_bake_status: BTreeMap, - subnet_update_proposals: Vec, - stages: &Vec, - logger: Option<&Logger>, - unassigned_version: &String, - subnets: &Vec, -) -> anyhow::Result> { +#[derive(Debug)] +pub enum SubnetAction { + Noop { + subnet_short: String, + }, + Baking { + subnet_short: String, + remaining: Duration, + }, + PendingProposal { + subnet_short: String, + proposal_id: u64, + }, + PlaceProposal { + is_unassigned: bool, + subnet_principal: String, + version: String, + }, +} + +pub fn check_stages<'a>( + current_version: &'a String, + current_release_feature_spec: &'a BTreeMap>, + last_bake_status: &'a BTreeMap, + subnet_update_proposals: &'a [SubnetUpdateProposal], + stages: &'a [Stage], + logger: Option<&'a Logger>, + unassigned_version: &'a String, + subnets: &'a [Subnet], +) -> anyhow::Result> { for (i, stage) in stages.iter().enumerate() { if let Some(logger) = logger { - info!(logger, "### Checking stage {}", i); + info!(logger, "Checking stage {}", i) } - let mut actions = vec![]; - let mut stage_baked = true; + let stage_actions = check_stage( + current_version, + current_release_feature_spec, + last_bake_status, + subnet_update_proposals, + stage, + logger, + unassigned_version, + subnets, + )?; - if stage.update_unassigned_nodes { - if let Some(logger) = logger { - debug!(logger, "Unassigned nodes stage"); + if !stage_actions.iter().all(|a| { + if let SubnetAction::Noop { subnet_short: _ } = a { + return true; } + return false; + }) { + return Ok(stage_actions); + } - let unassigned_nodes_version = unassigned_version; - - if unassigned_nodes_version.eq(current_version) { - if let Some(logger) = logger { - info!(logger, "Unassigned version already at version '{}'", current_version); - } - continue; - } - // Should update unassigned nodes - actions.push(format!("Update unassigned nodes to version: {}", current_version)); - return Ok(actions); + if let Some(logger) = logger { + info!(logger, "Stage {} is completed", i) } + } + if let Some(logger) = logger { + info!(logger, "The current rollout '{}' is completed.", current_version); + } + + Ok(vec![]) +} + +fn check_stage<'a>( + current_version: &'a String, + current_release_feature_spec: &'a BTreeMap>, + last_bake_status: &'a BTreeMap, + subnet_update_proposals: &'a [SubnetUpdateProposal], + stage: &'a Stage, + logger: Option<&'a Logger>, + unassigned_version: &'a String, + subnets: &'a [Subnet], +) -> anyhow::Result> { + let mut stage_actions = vec![]; + if stage.update_unassigned_nodes { + // Update unassigned nodes if let Some(logger) = logger { - debug!(logger, "Regular nodes stage"); + debug!(logger, "Unassigned nodes stage"); } - for subnet_short in &stage.subnets { - let desired_version = match current_release_feature_spec - .iter() - .find(|(_, subnets)| subnets.contains(subnet_short)) - { - Some((name, _)) => name, - None => current_version, - }; - if let Some(logger) = logger { - debug!( - logger, - "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version - ); - } + if !unassigned_version.eq(current_version) { + stage_actions.push(SubnetAction::PlaceProposal { + is_unassigned: true, + subnet_principal: "".to_string(), + version: current_version.clone(), + }); + return Ok(stage_actions); + } + } - let subnet = match subnets - .into_iter() - .find(|key| key.principal.to_string().starts_with(subnet_short)) - { - Some(subnet) => subnet, - None => { - return Err(anyhow::anyhow!( - "Couldn't find subnet that starts with '{}'", - subnet_short - )) - } - }; + for subnet_short in &stage.subnets { + // Get desired version + let desired_version = + get_desired_version_for_subnet(subnet_short, current_release_feature_spec, current_version); - if subnet.replica_version.eq(desired_version) { - if let Some(logger) = logger { - info!(logger, "Subnet {} is at desired version", subnet_short); - } - // Check bake time - let bake = last_bake_status - .get(&subnet.principal.to_string()) - .expect("Should have key"); - if bake.gt(&stage.bake_time.as_secs_f64()) { - if let Some(logger) = logger { - info!(logger, "Subnet {} is baked", subnet_short); - } - continue; - } else { - let remaining = Duration::from_secs_f64(stage.bake_time.as_secs_f64() - bake); - let formatted = format_duration(remaining); - if let Some(logger) = logger { - info!( - logger, - "Waiting for subnet {} to bake, pending {}", subnet_short, formatted - ); - } - stage_baked |= false; - } + // Find subnet to by the subnet_short + let subnet = find_subnet_by_short(subnets, subnet_short)?; + + if let Some(logger) = logger { + debug!( + logger, + "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version + ); + } + + // If subnet is on desired version, check bake time + if subnet.replica_version.eq(desired_version) { + let remaining = + get_remaining_bake_time_for_subnet(last_bake_status, subnet, stage.bake_time.as_secs_f64())?; + let remaining_duration = Duration::from_secs_f64(remaining); + let formatted = format_duration(remaining_duration); + + if remaining != 0.0 { + stage_actions.push(SubnetAction::Baking { + subnet_short: subnet_short.clone(), + remaining: remaining_duration, + }) } + if let Some(logger) = logger { - info!(logger, "Subnet {} is not at desired version", subnet_short); - } - // Check if there is an open proposal => if yes, wait; if false, place and exit - - if let Some(proposal) = subnet_update_proposals.iter().find(|proposal| { - proposal.payload.subnet_id == subnet.principal - && proposal.payload.replica_version_id.eq(desired_version) - && !proposal.info.executed - }) { - if let Some(logger) = logger { - info!( + if remaining == 0.0 { + debug!(logger, "Subnet {} baked", subnet_short) + } else { + debug!( logger, - "For subnet '{}' found open proposal 'https://dashboard.internetcomputer.org/proposal/{}'", - subnet_short, - proposal.info.id - ); + "Waiting for subnet {} to bake, remaining {}", subnet_short, formatted + ) } - continue; } - actions.push(format!( - "Should update subnet '{}' to version '{}'", - subnet_short, desired_version - )) + stage_actions.push(SubnetAction::Noop { + subnet_short: subnet_short.clone(), + }) } - if !stage_baked { - return Ok(vec![]); + // If subnet is not on desired version, check if there is an open proposal + if let Some(proposal) = get_open_proposal_for_subnet(subnet_update_proposals, subnet, desired_version) { + if let Some(logger) = logger { + info!( + logger, + "For subnet '{}' found open proposal with id '{}'", subnet_short, proposal.info.id + ) + } + stage_actions.push(SubnetAction::PendingProposal { + subnet_short: subnet_short.clone(), + proposal_id: proposal.info.id, + }) } - if !actions.is_empty() { - return Ok(actions); + // If subnet is not on desired version and there is no open proposal submit it + stage_actions.push(SubnetAction::PlaceProposal { + is_unassigned: false, + subnet_principal: subnet.principal.to_string(), + version: current_version.clone(), + }) + } + + Ok(stage_actions) +} + +fn get_desired_version_for_subnet<'a>( + subnet_short: &'a String, + current_release_feature_spec: &'a BTreeMap>, + current_version: &'a String, +) -> &'a String { + return match current_release_feature_spec + .iter() + .find(|(_, subnets)| subnets.contains(subnet_short)) + { + Some((name, _)) => name, + None => current_version, + }; +} + +fn find_subnet_by_short<'a>(subnets: &'a [Subnet], subnet_short: &'a String) -> anyhow::Result<&'a Subnet> { + return match subnets + .iter() + .find(|s| s.principal.to_string().starts_with(subnet_short)) + { + Some(subnet) => Ok(subnet), + None => Err(anyhow::anyhow!("No subnet with short id '{}'", subnet_short)), + }; +} + +fn get_remaining_bake_time_for_subnet( + last_bake_status: &BTreeMap, + subnet: &Subnet, + stage_bake_time: f64, +) -> anyhow::Result { + let bake = match last_bake_status.get(&subnet.principal.to_string()) { + Some(bake) => bake, + None => { + return Err(anyhow::anyhow!( + "Subnet with principal '{}' not found", + subnet.principal.to_string() + )) } + }; - if let Some(logger) = logger { - info!(logger, "### Stage {} completed", i) + return match bake.gt(&stage_bake_time) { + true => Ok(0.0), + false => { + let remaining = Duration::from_secs_f64(stage_bake_time - bake); + return Ok(remaining.as_secs_f64()); } - } - if let Some(logger) = logger { - info!(logger, "The current rollout '{}' is completed.", current_version); - } - Ok(vec![]) + }; +} + +fn get_open_proposal_for_subnet<'a>( + subnet_update_proposals: &'a [SubnetUpdateProposal], + subnet: &'a Subnet, + desired_version: &'a String, +) -> Option<&'a SubnetUpdateProposal> { + subnet_update_proposals.iter().find(|p| { + p.payload.subnet_id == subnet.principal && p.payload.replica_version_id.eq(desired_version) && !p.info.executed + }) } From 614fca9748a46e7bfea9d048355460112bf582f0 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 11:09:09 +0100 Subject: [PATCH 13/26] added tests --- Cargo.Bazel.lock | 14 +- Cargo.lock | 3 + rs/rollout-controller/Cargo.toml | 3 + .../src/calculation/stage_checks.rs | 146 +++++++++++++++++- 4 files changed, 162 insertions(+), 4 deletions(-) diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index c971f13bd..a56589154 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "0a3544653a97a98cdf0a217ae9d9dc03617c23c7740ecb6690766998f600132a", + "checksum": "99d3ca2a43da6e64e7bdad694a356c67e45ea008484fb76de23a5c64174144ee", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -37188,6 +37188,10 @@ "id": "anyhow 1.0.80", "target": "anyhow" }, + { + "id": "candid 0.9.11", + "target": "candid" + }, { "id": "chrono 0.4.35", "target": "chrono" @@ -37212,6 +37216,10 @@ "id": "humantime-serde 1.1.1", "target": "humantime_serde" }, + { + "id": "ic-base-types 0.9.0", + "target": "ic_base_types" + }, { "id": "ic-registry-keys 0.9.0", "target": "ic_registry_keys" @@ -37228,6 +37236,10 @@ "id": "regex 1.10.3", "target": "regex" }, + { + "id": "registry-canister 0.9.0", + "target": "registry_canister" + }, { "id": "reqwest 0.11.25", "target": "reqwest" diff --git a/Cargo.lock b/Cargo.lock index b15b8bc51..29ece9afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7397,18 +7397,21 @@ name = "rollout-controller" version = "0.3.0" dependencies = [ "anyhow", + "candid", "chrono", "clap 4.4.18", "crossbeam", "crossbeam-channel", "humantime", "humantime-serde", + "ic-base-types", "ic-management-backend", "ic-management-types", "ic-registry-keys", "ic-registry-local-registry", "prometheus-http-query", "regex", + "registry-canister", "reqwest", "serde", "serde_json", diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index aea758d3c..b3477ff15 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -33,4 +33,7 @@ prometheus-http-query = { workspace = true } humantime-serde = "1.1.1" reqwest = { workspace = true } regex = { workspace = true } +registry-canister = { workspace = true } +candid = { workspace = true } +ic-base-types = { workspace = true } diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index fc260aa43..95ef65512 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -106,7 +106,7 @@ fn check_stage<'a>( get_desired_version_for_subnet(subnet_short, current_release_feature_spec, current_version); // Find subnet to by the subnet_short - let subnet = find_subnet_by_short(subnets, subnet_short)?; + let subnet = find_subnet_by_short_id(subnets, subnet_short)?; if let Some(logger) = logger { debug!( @@ -184,7 +184,7 @@ fn get_desired_version_for_subnet<'a>( }; } -fn find_subnet_by_short<'a>(subnets: &'a [Subnet], subnet_short: &'a String) -> anyhow::Result<&'a Subnet> { +fn find_subnet_by_short_id<'a>(subnets: &'a [Subnet], subnet_short: &'a String) -> anyhow::Result<&'a Subnet> { return match subnets .iter() .find(|s| s.principal.to_string().starts_with(subnet_short)) @@ -221,9 +221,149 @@ fn get_remaining_bake_time_for_subnet( fn get_open_proposal_for_subnet<'a>( subnet_update_proposals: &'a [SubnetUpdateProposal], subnet: &'a Subnet, - desired_version: &'a String, + desired_version: &'a str, ) -> Option<&'a SubnetUpdateProposal> { subnet_update_proposals.iter().find(|p| { p.payload.subnet_id == subnet.principal && p.payload.replica_version_id.eq(desired_version) && !p.info.executed }) } + +#[cfg(test)] +mod get_open_proposal_for_subnet_tests { + use std::str::FromStr; + + use candid::Principal; + use ic_base_types::PrincipalId; + use ic_management_backend::proposal::ProposalInfoInternal; + use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; + + use super::*; + + fn craft_proposals<'a>( + subnet_with_execution_status: &'a [(&'a str, bool)], + version: &'a str, + ) -> impl Iterator + 'a { + subnet_with_execution_status + .iter() + .enumerate() + .map(|(i, (id, executed))| SubnetUpdateProposal { + payload: UpdateSubnetReplicaVersionPayload { + subnet_id: PrincipalId(Principal::from_str(id).expect("Can create principal")), + replica_version_id: version.to_string(), + }, + info: ProposalInfoInternal { + id: i as u64, + // These values are not important for the function + executed_timestamp_seconds: 1, + proposal_timestamp_seconds: 1, + executed: *executed, + }, + }) + } + + fn craft_open_proposals<'a>(subnet_ids: &'a [&'a str], version: &'a str) -> Vec { + craft_proposals( + &subnet_ids.iter().map(|id| (*id, false)).collect::>(), + version, + ) + .collect() + } + + fn craft_executed_proposals<'a>(subnet_ids: &'a [&'a str], version: &'a str) -> Vec { + craft_proposals( + &subnet_ids.iter().map(|id| (*id, true)).collect::>(), + version, + ) + .collect() + } + + #[test] + fn should_find_open_proposal_for_subnet() { + let proposals = craft_open_proposals( + &vec![ + "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", + ], + "version", + ); + + let subnet = Subnet { + principal: PrincipalId( + Principal::from_str("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae") + .expect("Can create principal"), + ), + ..Default::default() + }; + let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); + + assert!(proposal.is_some()) + } + + #[test] + fn should_not_find_open_proposal_all_are_executed() { + let proposals = craft_executed_proposals( + &vec![ + "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", + ], + "version", + ); + let subnet = Subnet { + principal: PrincipalId( + Principal::from_str("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae") + .expect("Can create principal"), + ), + ..Default::default() + }; + let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); + + assert!(proposal.is_none()) + } + + #[test] + fn should_not_find_open_proposal_all_are_executed_for_different_version() { + let proposals = craft_executed_proposals( + &vec![ + "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", + ], + "other-version", + ); + let subnet = Subnet { + principal: PrincipalId( + Principal::from_str("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae") + .expect("Can create principal"), + ), + ..Default::default() + }; + let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); + + assert!(proposal.is_none()) + } + + #[test] + fn should_not_find_open_proposal_all_are_executed_for_different_subnets() { + let proposals = craft_executed_proposals( + &vec![ + "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", + ], + "version", + ); + let subnet = Subnet { + principal: PrincipalId( + Principal::from_str("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae") + .expect("Can create principal"), + ), + ..Default::default() + }; + let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); + + assert!(proposal.is_none()) + } +} + +#[cfg(test)] +mod get_remaining_bake_time_for_subnet_tests { + use super::*; +} From 850ce4d45ef74113b2d6e27b2f77ad01a43eb178 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 12:10:16 +0100 Subject: [PATCH 14/26] added more tests --- Cargo.Bazel.lock | 236 +++++++++++++++++- Cargo.lock | 42 ++++ rs/rollout-controller/Cargo.toml | 2 + .../src/calculation/stage_checks.rs | 150 ++++++----- 4 files changed, 369 insertions(+), 61 deletions(-) diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index a56589154..a531a1392 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "99d3ca2a43da6e64e7bdad694a356c67e45ea008484fb76de23a5c64174144ee", + "checksum": "043dcfd1c559f8778e799b8b70dde7599cbb44e8047be3ce3719437a4bf15e90", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -14177,6 +14177,36 @@ }, "license": "MIT OR Apache-2.0" }, + "futures-timer 3.0.3": { + "name": "futures-timer", + "version": "3.0.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/futures-timer/3.0.3/download", + "sha256": "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + } + }, + "targets": [ + { + "Library": { + "crate_name": "futures_timer", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "futures_timer", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "3.0.3" + }, + "license": "MIT/Apache-2.0" + }, "futures-util 0.3.30": { "name": "futures-util", "version": "0.3.30", @@ -36469,6 +36499,42 @@ }, "license": null }, + "relative-path 1.9.2": { + "name": "relative-path", + "version": "1.9.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/relative-path/1.9.2/download", + "sha256": "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "relative_path", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "relative_path", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "edition": "2021", + "version": "1.9.2" + }, + "license": "MIT OR Apache-2.0" + }, "rend 0.4.2": { "name": "rend", "version": "0.4.2", @@ -37283,11 +37349,179 @@ ], "selects": {} }, + "deps_dev": { + "common": [ + { + "id": "rstest 0.18.2", + "target": "rstest" + } + ], + "selects": {} + }, "edition": "2021", "version": "0.3.0" }, "license": null }, + "rstest 0.18.2": { + "name": "rstest", + "version": "0.18.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/rstest/0.18.2/download", + "sha256": "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rstest", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rstest", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "async-timeout", + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "futures 0.3.30", + "target": "futures" + }, + { + "id": "futures-timer 3.0.3", + "target": "futures_timer" + } + ], + "selects": {} + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "rstest_macros 0.18.2", + "target": "rstest_macros" + } + ], + "selects": {} + }, + "version": "0.18.2" + }, + "license": "MIT OR Apache-2.0" + }, + "rstest_macros 0.18.2": { + "name": "rstest_macros", + "version": "0.18.2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/rstest_macros/0.18.2/download", + "sha256": "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "rstest_macros", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rstest_macros", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "async-timeout" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "glob 0.3.1", + "target": "glob" + }, + { + "id": "proc-macro2 1.0.78", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.35", + "target": "quote" + }, + { + "id": "regex 1.10.3", + "target": "regex" + }, + { + "id": "relative-path 1.9.2", + "target": "relative_path" + }, + { + "id": "rstest_macros 0.18.2", + "target": "build_script_build" + }, + { + "id": "syn 2.0.52", + "target": "syn" + }, + { + "id": "unicode-ident 1.0.12", + "target": "unicode_ident" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.18.2" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "rustc_version 0.4.0", + "target": "rustc_version" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, "rust_decimal 1.34.3": { "name": "rust_decimal", "version": "1.34.3", diff --git a/Cargo.lock b/Cargo.lock index 29ece9afe..00980e471 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2811,6 +2811,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.30" @@ -7259,6 +7265,12 @@ dependencies = [ "url", ] +[[package]] +name = "relative-path" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" + [[package]] name = "rend" version = "0.4.2" @@ -7413,6 +7425,7 @@ dependencies = [ "regex", "registry-canister", "reqwest", + "rstest", "serde", "serde_json", "serde_yaml 0.9.32", @@ -7425,6 +7438,35 @@ dependencies = [ "url", ] +[[package]] +name = "rstest" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.52", + "unicode-ident", +] + [[package]] name = "rust_decimal" version = "1.34.3" diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index b3477ff15..57713a92e 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -37,3 +37,5 @@ registry-canister = { workspace = true } candid = { workspace = true } ic-base-types = { workspace = true } +[dev-dependencies] +rstest = "0.18.2" \ No newline at end of file diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 95ef65512..b90f67a11 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -209,7 +209,7 @@ fn get_remaining_bake_time_for_subnet( } }; - return match bake.gt(&stage_bake_time) { + return match bake.ge(&stage_bake_time) { true => Ok(0.0), false => { let remaining = Duration::from_secs_f64(stage_bake_time - bake); @@ -238,8 +238,16 @@ mod get_open_proposal_for_subnet_tests { use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; use super::*; + use rstest::rstest; - fn craft_proposals<'a>( + pub(super) fn craft_subnet_from_id<'a>(subnet_id: &'a str) -> Subnet { + Subnet { + principal: PrincipalId(Principal::from_str(subnet_id).expect("Can create principal")), + ..Default::default() + } + } + + pub(super) fn craft_proposals<'a>( subnet_with_execution_status: &'a [(&'a str, bool)], version: &'a str, ) -> impl Iterator + 'a { @@ -261,7 +269,7 @@ mod get_open_proposal_for_subnet_tests { }) } - fn craft_open_proposals<'a>(subnet_ids: &'a [&'a str], version: &'a str) -> Vec { + pub(super) fn craft_open_proposals<'a>(subnet_ids: &'a [&'a str], version: &'a str) -> Vec { craft_proposals( &subnet_ids.iter().map(|id| (*id, false)).collect::>(), version, @@ -269,7 +277,10 @@ mod get_open_proposal_for_subnet_tests { .collect() } - fn craft_executed_proposals<'a>(subnet_ids: &'a [&'a str], version: &'a str) -> Vec { + pub(super) fn craft_executed_proposals<'a>( + subnet_ids: &'a [&'a str], + version: &'a str, + ) -> Vec { craft_proposals( &subnet_ids.iter().map(|id| (*id, true)).collect::>(), version, @@ -287,83 +298,102 @@ mod get_open_proposal_for_subnet_tests { "version", ); - let subnet = Subnet { - principal: PrincipalId( - Principal::from_str("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae") - .expect("Can create principal"), - ), - ..Default::default() - }; + let subnet = craft_subnet_from_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); assert!(proposal.is_some()) } - #[test] - fn should_not_find_open_proposal_all_are_executed() { + #[rstest] + #[case( + "version", + "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", + "version" + )] + #[case( + "other-version", + "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", + "version" + )] + #[case( + "version", + "5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae", + "version" + )] + fn should_not_find_open_proposal( + #[case] proposal_version: &str, + #[case] subnet_id: &str, + #[case] current_version: &str, + ) { let proposals = craft_executed_proposals( &vec![ "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", ], - "version", + proposal_version, ); - let subnet = Subnet { - principal: PrincipalId( - Principal::from_str("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae") - .expect("Can create principal"), - ), - ..Default::default() - }; - let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); + let subnet = craft_subnet_from_id(subnet_id); + let proposal = get_open_proposal_for_subnet(&proposals, &subnet, current_version); assert!(proposal.is_none()) } +} - #[test] - fn should_not_find_open_proposal_all_are_executed_for_different_version() { - let proposals = craft_executed_proposals( - &vec![ - "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", - "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", - ], - "other-version", - ); - let subnet = Subnet { - principal: PrincipalId( - Principal::from_str("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae") - .expect("Can create principal"), - ), - ..Default::default() - }; - let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); +#[cfg(test)] +mod get_remaining_bake_time_for_subnet_tests { + use super::*; + use rstest::rstest; - assert!(proposal.is_none()) + fn craft_bake_status_from_tuples(tuples: &[(&str, f64)]) -> BTreeMap { + tuples + .iter() + .map(|(id, bake_time)| (id.to_string(), *bake_time)) + .collect::>() } #[test] - fn should_not_find_open_proposal_all_are_executed_for_different_subnets() { - let proposals = craft_executed_proposals( - &vec![ - "snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae", - "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", - ], - "version", + fn should_return_error_subnet_not_found() { + let subnet = get_open_proposal_for_subnet_tests::craft_subnet_from_id( + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", ); - let subnet = Subnet { - principal: PrincipalId( - Principal::from_str("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae") - .expect("Can create principal"), - ), - ..Default::default() - }; - let proposal = get_open_proposal_for_subnet(&proposals, &subnet, "version"); - assert!(proposal.is_none()) + let bake_status = craft_bake_status_from_tuples(&[("random-subnet", 1.0)]); + + let maybe_remaining_bake_time = get_remaining_bake_time_for_subnet(&bake_status, &subnet, 100.0); + + assert!(maybe_remaining_bake_time.is_err()) } -} -#[cfg(test)] -mod get_remaining_bake_time_for_subnet_tests { - use super::*; + #[rstest] + #[case(100.0, 100.0, 0.0)] + #[case(150.0, 100.0, 0.0)] + #[case(100.0, 150.0, 50.0)] + // Should these be allowed? Technically we will never get + // something like this from prometheus and there should + // be validation for incoming configuration, but it is a + // possibility in our code. Maybe we could add validation + // checks that disallow of negative baking time? + #[case(-100.0, 150.0, 250.0)] + #[case(-100.0, -150.0, 0.0)] + #[case(-100.0, -50.0, 50.0)] + fn should_return_subnet_baking_time( + #[case] subnet_bake_status: f64, + #[case] stage_bake: f64, + #[case] remaining: f64, + ) { + let subnet = get_open_proposal_for_subnet_tests::craft_subnet_from_id( + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", + ); + + let bake_status = craft_bake_status_from_tuples(&[( + "pae4o-o6dxf-xki7q-ezclx-znyd6-fnk6w-vkv5z-5lfwh-xym2i-otrrw-fqe", + subnet_bake_status, + )]); + + let maybe_remaining_bake_time = get_remaining_bake_time_for_subnet(&bake_status, &subnet, stage_bake); + + assert!(maybe_remaining_bake_time.is_ok()); + let remaining_bake_time = maybe_remaining_bake_time.unwrap(); + assert_eq!(remaining_bake_time, remaining) + } } From 18205e7438930337be6c1a2d929f2cd93131d5d6 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 12:38:35 +0100 Subject: [PATCH 15/26] more tests --- .../src/calculation/stage_checks.rs | 81 +++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index b90f67a11..5481fb784 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -171,20 +171,20 @@ fn check_stage<'a>( } fn get_desired_version_for_subnet<'a>( - subnet_short: &'a String, + subnet_short: &'a str, current_release_feature_spec: &'a BTreeMap>, - current_version: &'a String, -) -> &'a String { + current_version: &'a str, +) -> &'a str { return match current_release_feature_spec .iter() - .find(|(_, subnets)| subnets.contains(subnet_short)) + .find(|(_, subnets)| subnets.contains(&subnet_short.to_string())) { Some((name, _)) => name, None => current_version, }; } -fn find_subnet_by_short_id<'a>(subnets: &'a [Subnet], subnet_short: &'a String) -> anyhow::Result<&'a Subnet> { +fn find_subnet_by_short_id<'a>(subnets: &'a [Subnet], subnet_short: &'a str) -> anyhow::Result<&'a Subnet> { return match subnets .iter() .find(|s| s.principal.to_string().starts_with(subnet_short)) @@ -397,3 +397,74 @@ mod get_remaining_bake_time_for_subnet_tests { assert_eq!(remaining_bake_time, remaining) } } + +#[cfg(test)] +mod find_subnet_by_short_id_tests { + use self::get_open_proposal_for_subnet_tests::craft_subnet_from_id; + + use super::*; + + #[test] + fn should_find_subnet() { + let subnet_1 = craft_subnet_from_id("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae"); + let subnet_2 = craft_subnet_from_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); + let subnets = &[subnet_1, subnet_2]; + + let maybe_subnet = find_subnet_by_short_id(subnets, "5kdm2"); + + assert!(maybe_subnet.is_ok()); + let subnet = maybe_subnet.unwrap(); + let subnet_principal = subnet.principal.to_string(); + assert!(subnet_principal.starts_with("5kdm2")); + assert!(subnet_principal.eq("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae")); + } + + #[test] + fn should_find_not_subnet() { + let subnet_1 = craft_subnet_from_id("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae"); + let subnet_2 = craft_subnet_from_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); + let subnets = &[subnet_1, subnet_2]; + + let maybe_subnet = find_subnet_by_short_id(subnets, "random-subnet"); + + assert!(maybe_subnet.is_err()); + } +} + +#[cfg(test)] +mod get_desired_version_for_subnet_test { + use super::*; + + pub(super) fn craft_feature_spec(tuples: &[(&str, &[&str])]) -> BTreeMap> { + tuples + .iter() + .map(|(commit, subnets)| (commit.to_string(), subnets.iter().map(|id| id.to_string()).collect())) + .collect::>>() + } + + #[test] + fn should_return_current_version() { + let current_release_feature_spec = + craft_feature_spec(&[("feature-a-commit", &["subnet-1", "subnet-2", "subnet-3"])]); + + let version = get_desired_version_for_subnet("subnet", ¤t_release_feature_spec, "current_version"); + assert_eq!(version, "current_version") + } + + #[test] + fn should_return_current_version_empty_feature_spec() { + let current_release_feature_spec = craft_feature_spec(&vec![]); + + let version = get_desired_version_for_subnet("subnet", ¤t_release_feature_spec, "current_version"); + assert_eq!(version, "current_version") + } + + #[test] + fn should_return_feature_version() { + let current_release_feature_spec = + craft_feature_spec(&[("feature-a-commit", &["subnet-1", "subnet-2", "subnet-3"])]); + + let version = get_desired_version_for_subnet("subnet-1", ¤t_release_feature_spec, "current_version"); + assert_eq!(version, "feature-a-commit") + } +} From e7bdac10c7a56db51e2a59f2ee5cd41bbdda6c1d Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 12:46:50 +0100 Subject: [PATCH 16/26] adding bazel target for tests --- rs/rollout-controller/BUILD.bazel | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/rs/rollout-controller/BUILD.bazel b/rs/rollout-controller/BUILD.bazel index 1e0e57130..0c0ecf6d3 100644 --- a/rs/rollout-controller/BUILD.bazel +++ b/rs/rollout-controller/BUILD.bazel @@ -1,6 +1,6 @@ load("@//rs:oci_images.bzl", "rust_binary_oci_image_rules") load("@crate_index_dre//:defs.bzl", "aliases", "all_crate_deps") -load("@rules_rust//rust:defs.bzl", "rust_binary") +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") # Define a custom rule to copy the .zip file as a data dependency DEPS = [ @@ -28,5 +28,17 @@ rust_binary_oci_image_rules( base_image = "@debian-slim", ) - - +rust_test( + name = "unit_test", + aliases = aliases( + normal_dev = True, + proc_macro_dev = True, + ), + crate = ":rollout-controller", + proc_macro_deps = all_crate_deps( + proc_macro_dev = True, + ), + deps = all_crate_deps( + normal_dev = True, + ) + DEPS, +) \ No newline at end of file From 214ae12e6030b7e80fea61d481dd19a516c99970 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 14:57:17 +0100 Subject: [PATCH 17/26] adding checks if week passed --- rs/rollout-controller/src/calculation/mod.rs | 4 +- .../src/calculation/release_actions.rs | 4 +- .../src/calculation/stage_checks.rs | 166 +++++++++++++++++- 3 files changed, 166 insertions(+), 8 deletions(-) diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index 89dec1406..691d6bdf3 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -69,7 +69,7 @@ pub async fn calculate_progress<'a>( return Ok(vec![]); } - let latest_release = find_latest_release(&index)?; + let (latest_release, start_date) = find_latest_release(&index)?; let elected_versions = registry_state.get_blessed_replica_versions().await?; let (current_version, current_feature_spec) = @@ -113,6 +113,8 @@ pub async fn calculate_progress<'a>( Some(&logger), &unassigned_nodes_version, ®istry_state.subnets().into_values().collect::>(), + start_date.date(), + Local::now().date_naive(), )?; Ok(actions) diff --git a/rs/rollout-controller/src/calculation/release_actions.rs b/rs/rollout-controller/src/calculation/release_actions.rs index 4202a7ac0..22ea0bb14 100644 --- a/rs/rollout-controller/src/calculation/release_actions.rs +++ b/rs/rollout-controller/src/calculation/release_actions.rs @@ -5,7 +5,7 @@ use regex::Regex; use super::{Index, Release}; -pub fn find_latest_release(index: &Index) -> anyhow::Result { +pub fn find_latest_release(index: &Index) -> anyhow::Result<(Release, NaiveDateTime)> { let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); let mut mapped: Vec<(Release, NaiveDateTime)> = index @@ -30,7 +30,7 @@ pub fn find_latest_release(index: &Index) -> anyhow::Result { mapped.reverse(); match mapped.first() { - Some((found, _)) => Ok(found.clone()), + Some((found, datetime)) => Ok((found.clone(), datetime.clone())), None => Err(anyhow::anyhow!("There aren't any releases that match the criteria")), } } diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 5481fb784..298b1bb07 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, time::Duration}; +use chrono::{Datelike, Days, NaiveDate, Weekday}; use humantime::format_duration; use ic_management_backend::proposal::SubnetUpdateProposal; use ic_management_types::Subnet; @@ -25,6 +26,9 @@ pub enum SubnetAction { subnet_principal: String, version: String, }, + WaitForNextWeek { + subnet_short: String, + }, } pub fn check_stages<'a>( @@ -36,11 +40,25 @@ pub fn check_stages<'a>( logger: Option<&'a Logger>, unassigned_version: &'a String, subnets: &'a [Subnet], + start_of_release: NaiveDate, + now: NaiveDate, ) -> anyhow::Result> { for (i, stage) in stages.iter().enumerate() { if let Some(logger) = logger { info!(logger, "Checking stage {}", i) } + + if stage.wait_for_next_week && !week_passed(start_of_release, now) { + let actions = stage + .subnets + .iter() + .map(|subnet| SubnetAction::WaitForNextWeek { + subnet_short: subnet.to_string(), + }) + .collect(); + return Ok(actions); + } + let stage_actions = check_stage( current_version, current_release_feature_spec, @@ -73,6 +91,22 @@ pub fn check_stages<'a>( Ok(vec![]) } +fn week_passed(release_start: NaiveDate, now: NaiveDate) -> bool { + let counter = release_start.clone(); + counter + .checked_add_days(Days::new(1)) + .expect("Should be able to add a day"); + while counter <= now { + if counter.weekday() == Weekday::Mon { + return true; + } + counter + .checked_add_days(Days::new(1)) + .expect("Should be able to add a day"); + } + false +} + fn check_stage<'a>( current_version: &'a String, current_release_feature_spec: &'a BTreeMap>, @@ -247,6 +281,19 @@ mod get_open_proposal_for_subnet_tests { } } + pub(super) fn craft_subnet_from_similar_id<'a>(subnet_id: &'a str) -> Subnet { + let original_principal = Principal::from_str(subnet_id).expect("Can create principal"); + let mut new_principal = original_principal.as_slice().to_vec(); + let len = new_principal.len(); + if let Some(byte) = new_principal.get_mut(len - 1) { + *byte += 1; + } + Subnet { + principal: PrincipalId(Principal::try_from_slice(&new_principal[..]).expect("Can create principal")), + ..Default::default() + } + } + pub(super) fn craft_proposals<'a>( subnet_with_execution_status: &'a [(&'a str, bool)], version: &'a str, @@ -400,7 +447,7 @@ mod get_remaining_bake_time_for_subnet_tests { #[cfg(test)] mod find_subnet_by_short_id_tests { - use self::get_open_proposal_for_subnet_tests::craft_subnet_from_id; + use self::get_open_proposal_for_subnet_tests::{craft_subnet_from_id, craft_subnet_from_similar_id}; use super::*; @@ -408,15 +455,16 @@ mod find_subnet_by_short_id_tests { fn should_find_subnet() { let subnet_1 = craft_subnet_from_id("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae"); let subnet_2 = craft_subnet_from_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); - let subnets = &[subnet_1, subnet_2]; + let subnet_3 = craft_subnet_from_similar_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); + let subnets = &[subnet_1, subnet_2, subnet_3]; - let maybe_subnet = find_subnet_by_short_id(subnets, "5kdm2"); + let maybe_subnet = find_subnet_by_short_id(subnets, "snjp4"); assert!(maybe_subnet.is_ok()); let subnet = maybe_subnet.unwrap(); let subnet_principal = subnet.principal.to_string(); - assert!(subnet_principal.starts_with("5kdm2")); - assert!(subnet_principal.eq("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae")); + assert!(subnet_principal.starts_with("snjp4")); + assert!(subnet_principal.eq("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae")); } #[test] @@ -468,3 +516,111 @@ mod get_desired_version_for_subnet_test { assert_eq!(version, "feature-a-commit") } } + +// E2E tests for decision making process for happy path without feature builds +#[cfg(test)] +mod check_stages_tests_no_feature_builds { + use crate::calculation::{Index, Release, Rollout, Version}; + + use super::*; + + /// Part one => No feature builds + /// `current_version` - can be defined + /// `current_release_feature_spec` - empty because we don't have feature builds for this part + /// `last_bake_status` - can be defined + /// `subnet_update_proposals` - can be defined + /// `stages` - must be defined + /// `logger` - can be defined, but won't be because these are only tests + /// `unassigned_version` - should be defined + /// `subnets` - should be defined + /// + /// For all use cases we will use the following setup + /// rollout: + /// pause: false // Tested in `should_proceed.rs` module + /// skip_days: [] // Tested in `should_proceed.rs` module + /// stages: + /// - subnets: [io67a] + /// bake_time: 8h + /// - subnets: [shefu, uzr34] + /// bake_time: 4h + /// - update_unassigned_nodes: true + /// - subnets: [pjljw] + /// wait_for_next_week: true + /// bake_time: 4h + /// releases: + /// - rc_name: rc--2024-02-21_23-01 + /// versions: + /// - version: 2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f + /// name: rc--2024-02-21_23-01 + /// release_notes_ready: + /// subnets: [] // empty because its a regular build + /// - rc_name: rc--2024-02-14_23-01 + /// versions: + /// - version: 85bd56a70e55b2cea75cae6405ae11243e5fdad8 + /// name: rc--2024-02-14_23-01 + /// release_notes_ready: + /// subnets: [] // empty because its a regular build + fn craft_index_state() -> Index { + Index { + rollout: Rollout { + pause: false, + skip_days: [], + stages: [ + Stage { + subnets: ["io67a".to_string()], + bake_time: humantime::parse_duration("8h").expect("Should be able to parse."), + ..Default::default() + }, + Stage { + subnets: ["shefu".to_string(), "uzr34".to_string()], + bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), + ..Default::default() + }, + Stage { + update_unassigned_nodes: true, + ..Default::default() + }, + Stage { + subnets: ["pjljw".to_string()], + bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), + wait_for_next_week: true, + ..Default::default() + }, + ], + }, + releases: [ + Release { + rc_name: "rc--2024-02-21_23-01".to_string(), + versions: [Version { + name: "rc--2024-02-21_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }], + }, + Release { + rc_name: "rc--2024-02-14_23-01".to_string(), + versions: [Version { + name: "rc--2024-02-14_23-01".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }], + }, + ], + } + } + + /// Use-Case 1: Beginning of a new rollout + /// + /// `current_version` - set to a commit that is being rolled out + /// `last_bake_status` - empty, because no subnets have the version + /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS + /// But it is for a different version (one from last week) + /// `unassigned_version` - one from previous week + /// `subnets` - can be seen in `craft_index_state` + #[test] + fn test_rollout_beginning() {} +} + +// E2E tests for decision making process for happy path with feature builds +#[cfg(test)] +mod check_stages_tests_feature_builds {} From 189a4be9890e922763da94425de993c48de9569e Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 15:14:40 +0100 Subject: [PATCH 18/26] adding tests for week passed method --- .../src/calculation/release_actions.rs | 10 ++++- .../src/calculation/stage_checks.rs | 43 ++++++++++++++----- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/rs/rollout-controller/src/calculation/release_actions.rs b/rs/rollout-controller/src/calculation/release_actions.rs index 22ea0bb14..4c2b89f91 100644 --- a/rs/rollout-controller/src/calculation/release_actions.rs +++ b/rs/rollout-controller/src/calculation/release_actions.rs @@ -140,6 +140,8 @@ impl Display for CreateCurrentReleaseFeatureSpecError { #[cfg(test)] mod find_latest_release_tests { + use chrono::NaiveDate; + use super::*; #[test] @@ -186,9 +188,13 @@ mod find_latest_release_tests { let latest = find_latest_release(&index); assert!(latest.is_ok()); - let latest = latest.unwrap(); + let (latest, date) = latest.unwrap(); - assert_eq!(latest.rc_name, String::from("rc--2024-03-10_23-01")) + assert_eq!(latest.rc_name, String::from("rc--2024-03-10_23-01")); + assert_eq!( + date.date(), + NaiveDate::parse_from_str("2024-03-10", "%Y-%m-%d").expect("Should parse date") + ) } #[test] diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 298b1bb07..91d78f243 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -92,15 +92,15 @@ pub fn check_stages<'a>( } fn week_passed(release_start: NaiveDate, now: NaiveDate) -> bool { - let counter = release_start.clone(); - counter + let mut counter = release_start.clone(); + counter = counter .checked_add_days(Days::new(1)) .expect("Should be able to add a day"); while counter <= now { if counter.weekday() == Weekday::Mon { return true; } - counter + counter = counter .checked_add_days(Days::new(1)) .expect("Should be able to add a day"); } @@ -262,6 +262,27 @@ fn get_open_proposal_for_subnet<'a>( }) } +#[cfg(test)] +mod week_passed_tests { + use super::*; + use chrono::NaiveDate; + use rstest::rstest; + + #[rstest] + #[case("2024-03-13", "2024-03-18", true)] + #[case("2024-03-13", "2024-03-19", true)] + #[case("2024-03-03", "2024-03-19", true)] + #[case("2024-03-13", "2024-03-13", false)] + #[case("2024-03-13", "2024-03-15", false)] + #[case("2024-03-13", "2024-03-17", false)] + fn should_complete(#[case] release_start: &str, #[case] now: &str, #[case] outcome: bool) { + let release_start = NaiveDate::parse_from_str(release_start, "%Y-%m-%d").expect("Should be able to parse date"); + let now = NaiveDate::parse_from_str(now, "%Y-%m-%d").expect("Should be able to parse date"); + + assert_eq!(week_passed(release_start, now), outcome) + } +} + #[cfg(test)] mod get_open_proposal_for_subnet_tests { use std::str::FromStr; @@ -564,15 +585,15 @@ mod check_stages_tests_no_feature_builds { Index { rollout: Rollout { pause: false, - skip_days: [], - stages: [ + skip_days: vec![], + stages: vec![ Stage { - subnets: ["io67a".to_string()], + subnets: vec!["io67a".to_string()], bake_time: humantime::parse_duration("8h").expect("Should be able to parse."), ..Default::default() }, Stage { - subnets: ["shefu".to_string(), "uzr34".to_string()], + subnets: vec!["shefu".to_string(), "uzr34".to_string()], bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), ..Default::default() }, @@ -581,17 +602,17 @@ mod check_stages_tests_no_feature_builds { ..Default::default() }, Stage { - subnets: ["pjljw".to_string()], + subnets: vec!["pjljw".to_string()], bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), wait_for_next_week: true, ..Default::default() }, ], }, - releases: [ + releases: vec![ Release { rc_name: "rc--2024-02-21_23-01".to_string(), - versions: [Version { + versions: vec![Version { name: "rc--2024-02-21_23-01".to_string(), version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), ..Default::default() @@ -599,7 +620,7 @@ mod check_stages_tests_no_feature_builds { }, Release { rc_name: "rc--2024-02-14_23-01".to_string(), - versions: [Version { + versions: vec![Version { name: "rc--2024-02-14_23-01".to_string(), version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), ..Default::default() From 372f47000e0a60605a11349988b14c0504b1b1fc Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 15:37:40 +0100 Subject: [PATCH 19/26] start adding e2e tests --- .../src/calculation/stage_checks.rs | 95 ++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 91d78f243..bd8cac81a 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -541,6 +541,11 @@ mod get_desired_version_for_subnet_test { // E2E tests for decision making process for happy path without feature builds #[cfg(test)] mod check_stages_tests_no_feature_builds { + use std::str::FromStr; + + use candid::Principal; + use ic_base_types::PrincipalId; + use crate::calculation::{Index, Release, Rollout, Version}; use super::*; @@ -554,6 +559,8 @@ mod check_stages_tests_no_feature_builds { /// `logger` - can be defined, but won't be because these are only tests /// `unassigned_version` - should be defined /// `subnets` - should be defined + /// `start_of_release` - should be defined + /// `now` - should be defined /// /// For all use cases we will use the following setup /// rollout: @@ -630,16 +637,98 @@ mod check_stages_tests_no_feature_builds { } } + fn craft_subnets() -> Vec { + vec![ + Subnet { + principal: PrincipalId( + Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") + .expect("Should be able to create a principal"), + ), + replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }, + Subnet { + principal: PrincipalId( + Principal::from_str("shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe") + .expect("Should be able to create a principal"), + ), + replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }, + Subnet { + principal: PrincipalId( + Principal::from_str("uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe") + .expect("Should be able to create a principal"), + ), + replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }, + Subnet { + principal: PrincipalId( + Principal::from_str("pjljw-kztyl-46ud4-ofrj6-nzkhm-3n4nt-wi3jt-ypmav-ijqkt-gjf66-uae") + .expect("Should be able to create a principal"), + ), + replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }, + ] + } + /// Use-Case 1: Beginning of a new rollout /// - /// `current_version` - set to a commit that is being rolled out + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - empty, because no subnets have the version /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS /// But it is for a different version (one from last week) - /// `unassigned_version` - one from previous week + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` #[test] - fn test_rollout_beginning() {} + fn test_rollout_beginning() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = BTreeMap::new(); + let subnet_update_proposals = Vec::new(); + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let subnets = &craft_subnets(); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + stages, + None, + &unassigned_version, + subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal: _, + version, + } => { + assert_eq!(is_unassigned, false); + assert_eq!(version, current_version); + } + // Fail the test + _ => assert!(false), + } + } + } } // E2E tests for decision making process for happy path with feature builds From d2b18428dcb35999647e7a162b672db49d978f24 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Wed, 13 Mar 2024 16:59:54 +0100 Subject: [PATCH 20/26] adding tests for one whole stage --- .../src/calculation/stage_checks.rs | 255 +++++++++++++++++- 1 file changed, 250 insertions(+), 5 deletions(-) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index bd8cac81a..f5e18f3b7 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -160,7 +160,8 @@ fn check_stage<'a>( stage_actions.push(SubnetAction::Baking { subnet_short: subnet_short.clone(), remaining: remaining_duration, - }) + }); + continue; } if let Some(logger) = logger { @@ -176,7 +177,8 @@ fn check_stage<'a>( stage_actions.push(SubnetAction::Noop { subnet_short: subnet_short.clone(), - }) + }); + continue; } // If subnet is not on desired version, check if there is an open proposal @@ -190,7 +192,8 @@ fn check_stage<'a>( stage_actions.push(SubnetAction::PendingProposal { subnet_short: subnet_short.clone(), proposal_id: proposal.info.id, - }) + }); + continue; } // If subnet is not on desired version and there is no open proposal submit it @@ -545,6 +548,8 @@ mod check_stages_tests_no_feature_builds { use candid::Principal; use ic_base_types::PrincipalId; + use ic_management_backend::proposal::ProposalInfoInternal; + use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; use crate::calculation::{Index, Release, Rollout, Version}; @@ -674,6 +679,14 @@ mod check_stages_tests_no_feature_builds { ] } + fn replace_versions(subnets: &mut Vec, tuples: &[(&str, &str)]) { + for (id, ver) in tuples { + if let Some(subnet) = subnets.iter_mut().find(|s| s.principal.to_string().contains(id)) { + subnet.replica_version = ver.to_string(); + } + } + } + /// Use-Case 1: Beginning of a new rollout /// /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` @@ -685,7 +698,7 @@ mod check_stages_tests_no_feature_builds { /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] - fn test_rollout_beginning() { + fn test_use_case_1() { let index = craft_index_state(); let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); let last_bake_status = BTreeMap::new(); @@ -718,17 +731,249 @@ mod check_stages_tests_no_feature_builds { match action { SubnetAction::PlaceProposal { is_unassigned, - subnet_principal: _, + subnet_principal, version, } => { assert_eq!(is_unassigned, false); assert_eq!(version, current_version); + assert!(subnet_principal.starts_with("io67a")) } // Fail the test _ => assert!(false), } } } + + /// Use case 2: First batch is submitted but the proposal wasn't executed + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - empty, because no subnets have the version + /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_2() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = BTreeMap::new(); + let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") + .expect("Should be possible to create principal"); + let subnet_update_proposals = vec![SubnetUpdateProposal { + info: ProposalInfoInternal { + executed: false, + executed_timestamp_seconds: 0, + proposal_timestamp_seconds: 0, + id: 1, + }, + payload: UpdateSubnetReplicaVersionPayload { + subnet_id: PrincipalId(subnet_principal.clone()), + replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + }, + }]; + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let subnets = &craft_subnets(); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + stages, + None, + &unassigned_version, + subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::PendingProposal { + subnet_short, + proposal_id, + } => { + assert_eq!(proposal_id, 1); + assert!(subnet_principal.to_string().starts_with(&subnet_short)) + } + // Just fail + _ => assert!(false), + } + } + } + + /// Use case 3: First batch is submitted the proposal was executed and the subnet is baking + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for the first subnet + /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_3() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("3h"), + )] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") + .expect("Should be possible to create principal"); + let subnet_update_proposals = vec![SubnetUpdateProposal { + info: ProposalInfoInternal { + executed: true, + executed_timestamp_seconds: 0, + proposal_timestamp_seconds: 0, + id: 1, + }, + payload: UpdateSubnetReplicaVersionPayload { + subnet_id: PrincipalId(subnet_principal.clone()), + replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + }, + }]; + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let mut subnets = craft_subnets(); + replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::Baking { + subnet_short, + remaining, + } => { + assert!(subnet_principal.to_string().starts_with(&subnet_short)); + assert!(remaining.eq(&humantime::parse_duration("5h").expect("Should parse duration"))) + } + // Just fail + _ => assert!(false), + } + } + } + + /// Use case 4: First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for the first subnet + /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_4() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + )] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") + .expect("Should be possible to create principal"); + let subnet_update_proposals = vec![SubnetUpdateProposal { + info: ProposalInfoInternal { + executed: true, + executed_timestamp_seconds: 0, + proposal_timestamp_seconds: 0, + id: 1, + }, + payload: UpdateSubnetReplicaVersionPayload { + subnet_id: PrincipalId(subnet_principal.clone()), + replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + }, + }]; + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let mut subnets = craft_subnets(); + replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 2); + let subnets = vec![ + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + ]; + for action in actions { + match action { + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal, + version, + } => { + assert_eq!(is_unassigned, false); + assert_eq!(version, current_version); + assert!(subnets.contains(&subnet_principal.as_str())) + } + // Just fail + _ => assert!(false), + } + } + } } // E2E tests for decision making process for happy path with feature builds From ce2ee49436470353183e72cb250cc5dcdd500d99 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 14 Mar 2024 13:05:53 +0100 Subject: [PATCH 21/26] adding checks if there is a placed update unassigned version proposal --- rs/ic-management-backend/src/proposal.rs | 19 +- rs/ic-management-backend/src/registry.rs | 8 +- rs/ic-management-types/src/lib.rs | 5 + rs/rollout-controller/src/calculation/mod.rs | 2 + .../src/calculation/stage_checks.rs | 251 +++++++++++++++++- 5 files changed, 276 insertions(+), 9 deletions(-) diff --git a/rs/ic-management-backend/src/proposal.rs b/rs/ic-management-backend/src/proposal.rs index 5db06a291..033a89d3a 100644 --- a/rs/ic-management-backend/src/proposal.rs +++ b/rs/ic-management-backend/src/proposal.rs @@ -7,9 +7,9 @@ use futures_util::future::try_join_all; use ic_agent::agent::http_transport::reqwest_transport::ReqwestHttpReplicaV2Transport; use ic_agent::Agent; use ic_management_types::UpdateElectedHostosVersionsProposal; +use ic_management_types::UpdateElectedReplicaVersionsProposal; use ic_management_types::UpdateNodesHostosVersionsProposal; use ic_management_types::{NnsFunctionProposal, TopologyChangePayload, TopologyChangeProposal}; -use ic_management_types::UpdateElectedReplicaVersionsProposal; use ic_nns_governance::pb::v1::{proposal::Action, ListProposalInfo, ListProposalInfoResponse, NnsFunction}; use ic_nns_governance::pb::v1::{ProposalInfo, ProposalStatus, Topic}; use itertools::Itertools; @@ -21,6 +21,7 @@ use registry_canister::mutations::do_update_elected_hostos_versions::UpdateElect use registry_canister::mutations::do_update_elected_replica_versions::UpdateElectedReplicaVersionsPayload; use registry_canister::mutations::do_update_nodes_hostos_version::UpdateNodesHostosVersionPayload; use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; +use registry_canister::mutations::do_update_unassigned_nodes_config::UpdateUnassignedNodesConfigPayload; use registry_canister::mutations::node_management::do_remove_nodes::RemoveNodesPayload; use serde::Serialize; @@ -74,6 +75,12 @@ pub struct SubnetUpdateProposal { pub payload: UpdateSubnetReplicaVersionPayload, } +#[derive(Clone, Serialize)] +pub struct UpdateUnassignedNodesProposal { + pub info: ProposalInfoInternal, + pub payload: UpdateUnassignedNodesConfigPayload, +} + impl ProposalAgent { pub fn new(url: String) -> Self { let agent = Agent::builder() @@ -200,6 +207,16 @@ impl ProposalAgent { .collect::>()) } + pub async fn list_update_unassigned_nodes_version_proposals(&self) -> Result> { + Ok(filter_map_nns_function_proposals(&self.list_proposals(vec![]).await?) + .into_iter() + .map(|(info, payload)| UpdateUnassignedNodesProposal { + info: info.into(), + payload, + }) + .collect::>()) + } + async fn list_proposals(&self, include_status: Vec) -> Result> { let mut proposals = vec![]; loop { diff --git a/rs/ic-management-backend/src/registry.rs b/rs/ic-management-backend/src/registry.rs index 25ed27582..cbbd4773d 100644 --- a/rs/ic-management-backend/src/registry.rs +++ b/rs/ic-management-backend/src/registry.rs @@ -1,7 +1,7 @@ use crate::config::get_nns_url_vec_from_target_network; use crate::factsdb; use crate::git_ic_repo::IcRepo; -use crate::proposal::{self, SubnetUpdateProposal}; +use crate::proposal::{self, SubnetUpdateProposal, UpdateUnassignedNodesProposal}; use crate::public_dashboard::query_ic_dashboard_list; use async_trait::async_trait; use decentralization::network::{AvailableNodesQuerier, SubnetQuerier, SubnetQueryBy}; @@ -724,6 +724,12 @@ impl RegistryState { proposal_agent.list_update_subnet_version_proposals().await } + pub async fn open_upgrade_unassigned_nodes_proposals(&self) -> Result> { + let proposal_agent = proposal::ProposalAgent::new(self.nns_url.clone()); + + proposal_agent.list_update_unassigned_nodes_version_proposals().await + } + async fn retireable_hostos_versions(&self) -> Result> { let active_releases = self.hostos_releases.get_active_branches(); let hostos_versions: BTreeSet = self.nodes.values().map(|s| s.hostos_version.clone()).collect(); diff --git a/rs/ic-management-types/src/lib.rs b/rs/ic-management-types/src/lib.rs index e47d3cb3d..3d50257bc 100644 --- a/rs/ic-management-types/src/lib.rs +++ b/rs/ic-management-types/src/lib.rs @@ -18,6 +18,7 @@ use registry_canister::mutations::do_update_elected_hostos_versions::UpdateElect use registry_canister::mutations::do_update_elected_replica_versions::UpdateElectedReplicaVersionsPayload; use registry_canister::mutations::do_update_nodes_hostos_version::UpdateNodesHostosVersionPayload; use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; +use registry_canister::mutations::do_update_unassigned_nodes_config::UpdateUnassignedNodesConfigPayload; use registry_canister::mutations::node_management::do_remove_nodes::RemoveNodesPayload; use serde::{Deserialize, Serialize}; use std::cmp::{Eq, Ord, PartialEq, PartialOrd}; @@ -42,6 +43,10 @@ pub trait NnsFunctionProposal: CandidType + serde::de::DeserializeOwned { } } +impl NnsFunctionProposal for UpdateUnassignedNodesConfigPayload { + const TYPE: NnsFunction = NnsFunction::UpdateUnassignedNodesConfig; +} + impl NnsFunctionProposal for AddNodesToSubnetPayload { const TYPE: NnsFunction = NnsFunction::AddNodeToSubnet; } diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index 691d6bdf3..be669412e 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -103,12 +103,14 @@ pub async fn calculate_progress<'a>( let subnet_update_proposals = registry_state.open_subnet_upgrade_proposals().await?; let unassigned_nodes_version = registry_state.get_unassigned_nodes_replica_version().await?; + let unassigned_nodes_proposals = registry_state.open_upgrade_unassigned_nodes_proposals().await?; let actions = check_stages( ¤t_version, ¤t_feature_spec, &last_bake_status, &subnet_update_proposals, + &unassigned_nodes_proposals, &index.rollout.stages, Some(&logger), &unassigned_nodes_version, diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index f5e18f3b7..77d262df8 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, time::Duration}; use chrono::{Datelike, Days, NaiveDate, Weekday}; use humantime::format_duration; -use ic_management_backend::proposal::SubnetUpdateProposal; +use ic_management_backend::proposal::{SubnetUpdateProposal, UpdateUnassignedNodesProposal}; use ic_management_types::Subnet; use slog::{debug, info, Logger}; @@ -36,6 +36,7 @@ pub fn check_stages<'a>( current_release_feature_spec: &'a BTreeMap>, last_bake_status: &'a BTreeMap, subnet_update_proposals: &'a [SubnetUpdateProposal], + unassigned_node_update_proposals: &'a [UpdateUnassignedNodesProposal], stages: &'a [Stage], logger: Option<&'a Logger>, unassigned_version: &'a String, @@ -64,6 +65,7 @@ pub fn check_stages<'a>( current_release_feature_spec, last_bake_status, subnet_update_proposals, + unassigned_node_update_proposals, stage, logger, unassigned_version, @@ -112,6 +114,7 @@ fn check_stage<'a>( current_release_feature_spec: &'a BTreeMap>, last_bake_status: &'a BTreeMap, subnet_update_proposals: &'a [SubnetUpdateProposal], + unassigned_node_update_proposals: &'a [UpdateUnassignedNodesProposal], stage: &'a Stage, logger: Option<&'a Logger>, unassigned_version: &'a String, @@ -125,13 +128,33 @@ fn check_stage<'a>( } if !unassigned_version.eq(current_version) { - stage_actions.push(SubnetAction::PlaceProposal { - is_unassigned: true, - subnet_principal: "".to_string(), - version: current_version.clone(), - }); + match unassigned_node_update_proposals.iter().find(|proposal| { + if !proposal.info.executed { + if let Some(version) = &proposal.payload.replica_version { + if version.eq(current_version) { + return true; + } + } + } + return false; + }) { + None => stage_actions.push(SubnetAction::PlaceProposal { + is_unassigned: true, + subnet_principal: "".to_string(), + version: current_version.clone(), + }), + Some(proposal) => stage_actions.push(SubnetAction::PendingProposal { + subnet_short: "unassigned-version".to_string(), + proposal_id: proposal.info.id, + }), + } return Ok(stage_actions); } + + stage_actions.push(SubnetAction::Noop { + subnet_short: "unassigned-nodes".to_string(), + }); + return Ok(stage_actions); } for subnet_short in &stage.subnets { @@ -547,9 +570,13 @@ mod check_stages_tests_no_feature_builds { use std::str::FromStr; use candid::Principal; + use check_stages_tests_no_feature_builds::get_open_proposal_for_subnet_tests::craft_executed_proposals; use ic_base_types::PrincipalId; use ic_management_backend::proposal::ProposalInfoInternal; - use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; + use registry_canister::mutations::{ + do_update_subnet_replica::UpdateSubnetReplicaVersionPayload, + do_update_unassigned_nodes_config::UpdateUnassignedNodesConfigPayload, + }; use crate::calculation::{Index, Release, Rollout, Version}; @@ -693,6 +720,7 @@ mod check_stages_tests_no_feature_builds { /// `last_bake_status` - empty, because no subnets have the version /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS /// But it is for a different version (one from last week) + /// `unassigned_nodes_proposals` - empty /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` /// `start_of_release` - some `2024-02-21` @@ -705,6 +733,7 @@ mod check_stages_tests_no_feature_builds { let subnet_update_proposals = Vec::new(); let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); @@ -715,6 +744,7 @@ mod check_stages_tests_no_feature_builds { ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, + &unassigned_nodes_proposals, stages, None, &unassigned_version, @@ -749,6 +779,7 @@ mod check_stages_tests_no_feature_builds { /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - empty, because no subnets have the version /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_nodes_proposals` - empty /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` /// `start_of_release` - some `2024-02-21` @@ -774,6 +805,7 @@ mod check_stages_tests_no_feature_builds { }]; let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); @@ -784,6 +816,7 @@ mod check_stages_tests_no_feature_builds { ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, + &unassigned_nodes_proposals, stages, None, &unassigned_version, @@ -816,6 +849,7 @@ mod check_stages_tests_no_feature_builds { /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for the first subnet /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_nodes_proposals` - empty /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` /// `start_of_release` - some `2024-02-21` @@ -852,6 +886,7 @@ mod check_stages_tests_no_feature_builds { }]; let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); let current_release_feature_spec = BTreeMap::new(); @@ -863,6 +898,7 @@ mod check_stages_tests_no_feature_builds { ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, + &unassigned_nodes_proposals, stages, None, &unassigned_version, @@ -895,6 +931,7 @@ mod check_stages_tests_no_feature_builds { /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for the first subnet /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_nodes_proposals` - empty /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` /// `start_of_release` - some `2024-02-21` @@ -931,6 +968,7 @@ mod check_stages_tests_no_feature_builds { }]; let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); let current_release_feature_spec = BTreeMap::new(); @@ -942,6 +980,7 @@ mod check_stages_tests_no_feature_builds { ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, + &unassigned_nodes_proposals, stages, None, &unassigned_version, @@ -974,6 +1013,204 @@ mod check_stages_tests_no_feature_builds { } } } + + /// Use case 5: Updating unassigned nodes + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for all subnets before unassigned nodes + /// `subnet_update_proposals` - contains proposals from previous two stages + /// `unassigned_nodes_proposals` - empty + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_5() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [ + ( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + ), + ( + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + humantime::parse_duration("5h"), + ), + ( + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + humantime::parse_duration("5h"), + ), + ] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_update_proposals = craft_executed_proposals( + &[ + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + ], + ¤t_version, + ); + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; + let mut subnets = craft_subnets(); + replace_versions( + &mut subnets, + &[ + ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ], + ); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposals, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal: _, + version, + } => { + assert!(is_unassigned); + assert_eq!(version, current_version); + } + // Just fail + _ => assert!(false), + } + } + } + + /// Use case 6: Proposal sent for updating unassigned nodes but it is not executed + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for all subnets before unassigned nodes + /// `subnet_update_proposals` - contains proposals from previous two stages + /// `unassigned_nodes_proposals` - contains open proposal for unassigned nodes + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_6() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [ + ( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + ), + ( + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + humantime::parse_duration("5h"), + ), + ( + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + humantime::parse_duration("5h"), + ), + ] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_update_proposals = craft_executed_proposals( + &[ + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + ], + ¤t_version, + ); + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { + info: ProposalInfoInternal { + executed: false, + executed_timestamp_seconds: 0, + id: 5, + proposal_timestamp_seconds: 0, + }, + payload: UpdateUnassignedNodesConfigPayload { + ssh_readonly_access: None, + replica_version: Some(current_version.clone()), + }, + }]; + let mut subnets = craft_subnets(); + replace_versions( + &mut subnets, + &[ + ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ], + ); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposal, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::PendingProposal { + proposal_id, + subnet_short, + } => { + assert_eq!(proposal_id, 5); + assert_eq!(subnet_short, "unassigned-version"); + } + // Just fail + _ => assert!(false), + } + } + } } // E2E tests for decision making process for happy path with feature builds From 76521ef765e86187c43f7d6a02e63332a6556c8c Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Thu, 14 Mar 2024 13:28:21 +0100 Subject: [PATCH 22/26] added full scenario for a simple rollout --- .../src/calculation/stage_checks.rs | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 77d262df8..ac9b38d29 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -1211,6 +1211,308 @@ mod check_stages_tests_no_feature_builds { } } } + + /// Use case 7: Executed update unassigned nodes, waiting for next week + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for all subnets before unassigned nodes + /// `subnet_update_proposals` - contains proposals from previous two stages + /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes + /// `unassigned_version` - same as `current_version` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-24` + #[test] + fn test_use_case_7() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [ + ( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + ), + ( + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + humantime::parse_duration("5h"), + ), + ( + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + humantime::parse_duration("5h"), + ), + ] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_update_proposals = craft_executed_proposals( + &[ + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + ], + ¤t_version, + ); + let stages = &index.rollout.stages; + let unassigned_version = current_version.clone(); + let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { + info: ProposalInfoInternal { + executed: true, + executed_timestamp_seconds: 0, + id: 5, + proposal_timestamp_seconds: 0, + }, + payload: UpdateUnassignedNodesConfigPayload { + ssh_readonly_access: None, + replica_version: Some(current_version.clone()), + }, + }]; + let mut subnets = craft_subnets(); + replace_versions( + &mut subnets, + &[ + ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ], + ); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-24", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposal, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::WaitForNextWeek { subnet_short } => { + assert_eq!(subnet_short, "pjljw"); + } + // Just fail + _ => assert!(false), + } + } + } + + /// Use case 8: Next monday came, should place proposal for updating the last subnet + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for all subnets before unassigned nodes + /// `subnet_update_proposals` - contains proposals from previous two stages + /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes + /// `unassigned_version` - same as `current_version` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-26` + #[test] + fn test_use_case_8() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [ + ( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + ), + ( + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + humantime::parse_duration("5h"), + ), + ( + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + humantime::parse_duration("5h"), + ), + ] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_update_proposals = craft_executed_proposals( + &[ + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + ], + ¤t_version, + ); + let stages = &index.rollout.stages; + let unassigned_version = current_version.clone(); + let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { + info: ProposalInfoInternal { + executed: true, + executed_timestamp_seconds: 0, + id: 5, + proposal_timestamp_seconds: 0, + }, + payload: UpdateUnassignedNodesConfigPayload { + ssh_readonly_access: None, + replica_version: Some(current_version.clone()), + }, + }]; + let mut subnets = craft_subnets(); + replace_versions( + &mut subnets, + &[ + ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ], + ); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposal, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal, + version, + } => { + assert!(subnet_principal.starts_with("pjljw")); + assert_eq!(is_unassigned, false); + assert_eq!(version, current_version) + } + // Just fail + _ => assert!(false), + } + } + } + + /// Use case 9: Next monday came, proposal for last subnet executed and bake time passed. Rollout finished + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `last_bake_status` - contains the status for all subnets before unassigned nodes + /// `subnet_update_proposals` - contains proposals from previous two stages + /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes + /// `unassigned_version` - same as `current_version` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-26` + #[test] + fn test_use_case_9() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [ + ( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + ), + ( + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + humantime::parse_duration("5h"), + ), + ( + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + humantime::parse_duration("5h"), + ), + ( + "pjljw-kztyl-46ud4-ofrj6-nzkhm-3n4nt-wi3jt-ypmav-ijqkt-gjf66-uae", + humantime::parse_duration("5h"), + ), + ] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_update_proposals = craft_executed_proposals( + &[ + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + "pjljw-kztyl-46ud4-ofrj6-nzkhm-3n4nt-wi3jt-ypmav-ijqkt-gjf66-uae", + ], + ¤t_version, + ); + let stages = &index.rollout.stages; + let unassigned_version = current_version.clone(); + let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { + info: ProposalInfoInternal { + executed: true, + executed_timestamp_seconds: 0, + id: 5, + proposal_timestamp_seconds: 0, + }, + payload: UpdateUnassignedNodesConfigPayload { + ssh_readonly_access: None, + replica_version: Some(current_version.clone()), + }, + }]; + let mut subnets = craft_subnets(); + replace_versions( + &mut subnets, + &[ + ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ("pjljw", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), + ], + ); + let current_release_feature_spec = BTreeMap::new(); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposal, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + assert_eq!(actions.len(), 0); + } } // E2E tests for decision making process for happy path with feature builds From eaf7af96a45248a678d9df091f4aa50b917e96ff Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Fri, 15 Mar 2024 13:03:08 +0100 Subject: [PATCH 23/26] added tests for feature builds --- .../src/calculation/stage_checks.rs | 292 +++++++++++++++++- 1 file changed, 288 insertions(+), 4 deletions(-) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index ac9b38d29..67fa89689 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -223,7 +223,7 @@ fn check_stage<'a>( stage_actions.push(SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: subnet.principal.to_string(), - version: current_version.clone(), + version: desired_version.to_string(), }) } @@ -669,7 +669,7 @@ mod check_stages_tests_no_feature_builds { } } - fn craft_subnets() -> Vec { + pub(super) fn craft_subnets() -> Vec { vec![ Subnet { principal: PrincipalId( @@ -706,7 +706,7 @@ mod check_stages_tests_no_feature_builds { ] } - fn replace_versions(subnets: &mut Vec, tuples: &[(&str, &str)]) { + pub(super) fn replace_versions(subnets: &mut Vec, tuples: &[(&str, &str)]) { for (id, ver) in tuples { if let Some(subnet) = subnets.iter_mut().find(|s| s.principal.to_string().contains(id)) { subnet.replica_version = ver.to_string(); @@ -1517,4 +1517,288 @@ mod check_stages_tests_no_feature_builds { // E2E tests for decision making process for happy path with feature builds #[cfg(test)] -mod check_stages_tests_feature_builds {} +mod check_stages_tests_feature_builds { + use std::str::FromStr; + + use candid::Principal; + use check_stages_tests_feature_builds::check_stages_tests_no_feature_builds::{craft_subnets, replace_versions}; + use ic_base_types::PrincipalId; + use ic_management_backend::proposal::ProposalInfoInternal; + use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; + + use crate::calculation::{Index, Release, Rollout, Version}; + + use super::*; + + /// Part two => Feature builds + /// `current_version` - has to be defined + /// `current_release_feature_spec` - has to be defined with mapped versions to subnets + /// `last_bake_status` - can be defined depending on the use case + /// `subnet_update_proposals` - can be defined depending on the use case + /// `unassigned_nodes_update_proposals` - can be defined depending on the use case + /// `stages` - has to be defined + /// `logger` - can be defined, but won't be because these are only tests + /// `unassigned_version` - has to be defined + /// `subnets` - has to be defined + /// `start_of_release` - has to be defined + /// `now` - has to be defined + /// + /// For all use cases we will use the following setup + /// rollout: + /// pause: false // Tested in `should_proceed.rs` module + /// skip_days: [] // Tested in `should_proceed.rs` module + /// stages: + /// - subnets: [io67a] + /// bake_time: 8h + /// - subnets: [shefu, uzr34] + /// bake_time: 4h + /// - update_unassigned_nodes: true + /// - subnets: [pjljw] + /// wait_for_next_week: true + /// bake_time: 4h + /// releases: + /// - rc_name: rc--2024-02-21_23-01 + /// versions: + /// - version: 2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f + /// name: rc--2024-02-21_23-01 + /// release_notes_ready: + /// subnets: [] + /// - version: 76521ef765e86187c43f7d6a02e63332a6556c8c + /// name: rc--2024-02-21_23-01-feat + /// release_notes_ready: + /// subnets: + /// - shefu + /// - io67a + /// - rc_name: rc--2024-02-14_23-01 + /// versions: + /// - version: 85bd56a70e55b2cea75cae6405ae11243e5fdad8 + /// name: rc--2024-02-14_23-01 + /// release_notes_ready: + /// subnets: [] // empty because its a regular build + fn craft_index_state() -> Index { + Index { + rollout: Rollout { + pause: false, + skip_days: vec![], + stages: vec![ + Stage { + subnets: vec!["io67a".to_string()], + bake_time: humantime::parse_duration("8h").expect("Should be able to parse."), + ..Default::default() + }, + Stage { + subnets: vec!["shefu".to_string(), "uzr34".to_string()], + bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), + ..Default::default() + }, + Stage { + update_unassigned_nodes: true, + ..Default::default() + }, + Stage { + subnets: vec!["pjljw".to_string()], + bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), + wait_for_next_week: true, + ..Default::default() + }, + ], + }, + releases: vec![ + Release { + rc_name: "rc--2024-02-21_23-01".to_string(), + versions: vec![ + Version { + name: "rc--2024-02-21_23-01".to_string(), + version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), + ..Default::default() + }, + Version { + name: "rc--2024-02-21_23-01-feat".to_string(), + version: "76521ef765e86187c43f7d6a02e63332a6556c8c".to_string(), + subnets: ["io67a", "shefu"].iter().map(|f| f.to_string()).collect(), + ..Default::default() + }, + ], + }, + Release { + rc_name: "rc--2024-02-14_23-01".to_string(), + versions: vec![Version { + name: "rc--2024-02-14_23-01".to_string(), + version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), + ..Default::default() + }], + }, + ], + } + } + + /// Use-Case 1: Beginning of a new rollout + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f + /// `current_release_feature_spec` - contains the spec for version `76521ef765e86187c43f7d6a02e63332a6556c8c` + /// `last_bake_status` - empty, because no subnets have the version + /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS + /// But it is for a different version (one from last week) + /// `unassigned_nodes_proposals` - empty + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_1() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = BTreeMap::new(); + let subnet_update_proposals = Vec::new(); + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; + let subnets = &craft_subnets(); + let feature = index + .releases + .get(0) + .expect("Should be at least one") + .versions + .get(1) + .expect("Should be set to be the second version being rolled out"); + let mut current_release_feature_spec = BTreeMap::new(); + current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposals, + stages, + None, + &unassigned_version, + subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + + assert_eq!(actions.len(), 1); + for action in actions { + match action { + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal, + version, + } => { + assert_eq!(is_unassigned, false); + assert_eq!(version, feature.version); + assert!(subnet_principal.starts_with("io67a")) + } + // Fail the test + _ => assert!(false), + } + } + } + + /// Use case 2: First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage + /// + /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` + /// `current_release_feature_spec` - contains the spec for version `76521ef765e86187c43f7d6a02e63332a6556c8c` + /// `last_bake_status` - contains the status for the first subnet + /// `subnet_update_proposals` - contains proposals from the first stage + /// `unassigned_nodes_proposals` - empty + /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` + /// `subnets` - can be seen in `craft_index_state` + /// `start_of_release` - some `2024-02-21` + /// `now` - same `2024-02-21` + #[test] + fn test_use_case_2() { + let index = craft_index_state(); + let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); + let last_bake_status = [( + "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", + humantime::parse_duration("9h"), + )] + .iter() + .map(|(id, duration)| { + ( + id.to_string(), + duration.clone().expect("Should parse duration").as_secs_f64(), + ) + }) + .collect(); + let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") + .expect("Should be possible to create principal"); + let subnet_update_proposals = vec![SubnetUpdateProposal { + info: ProposalInfoInternal { + executed: true, + executed_timestamp_seconds: 0, + proposal_timestamp_seconds: 0, + id: 1, + }, + payload: UpdateSubnetReplicaVersionPayload { + subnet_id: PrincipalId(subnet_principal.clone()), + replica_version_id: "76521ef765e86187c43f7d6a02e63332a6556c8c".to_string(), + }, + }]; + let stages = &index.rollout.stages; + let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); + let unassigned_nodes_proposals = vec![]; + let mut subnets = craft_subnets(); + replace_versions(&mut subnets, &[("io67a", "76521ef765e86187c43f7d6a02e63332a6556c8c")]); + let feature = index + .releases + .get(0) + .expect("Should be at least one") + .versions + .get(1) + .expect("Should be set to be the second version being rolled out"); + let mut current_release_feature_spec = BTreeMap::new(); + current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); + let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); + + let maybe_actions = check_stages( + ¤t_version, + ¤t_release_feature_spec, + &last_bake_status, + &subnet_update_proposals, + &unassigned_nodes_proposals, + stages, + None, + &unassigned_version, + &subnets, + start_of_release, + now, + ); + + assert!(maybe_actions.is_ok()); + let actions = maybe_actions.unwrap(); + println!("{:#?}", actions); + assert_eq!(actions.len(), 2); + let subnets = vec![ + "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", + "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", + ]; + for action in actions { + match action { + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal, + version, + } => { + assert_eq!(is_unassigned, false); + if subnet_principal.starts_with("shefu") { + assert_eq!(version, feature.version); + } else { + assert_eq!(version, current_version); + } + assert!(subnets.contains(&subnet_principal.as_str())) + } + // Just fail + _ => assert!(false), + } + } + } +} From fb4ac91d39103305c97102f4e2b8c13a73417528 Mon Sep 17 00:00:00 2001 From: Luka Skugor Date: Fri, 15 Mar 2024 15:59:48 +0100 Subject: [PATCH 24/26] desired rollout release version --- Cargo.lock | 2 + Cargo.toml | 1 + rs/rollout-controller/Cargo.toml | 6 +- rs/rollout-controller/src/calculation/mod.rs | 4 +- .../src/calculation/stage_checks.rs | 179 +++++++++++++++++- 5 files changed, 187 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00980e471..b3d37ac50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7421,6 +7421,8 @@ dependencies = [ "ic-management-types", "ic-registry-keys", "ic-registry-local-registry", + "itertools 0.12.1", + "pretty_assertions", "prometheus-http-query", "regex", "registry-canister", diff --git a/Cargo.toml b/Cargo.toml index f140c9355..63e360728 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,6 +133,7 @@ log = "0.4.20" lru = "0.12.1" phantom_newtype = { git = "https://github.com/dfinity/ic.git", rev = "4b3b2ce76c4bde0c1c60fb80b0915931003b7eca" } pkcs11 = "0.5.0" +pretty_assertions = "1.4.0" pretty_env_logger = "0.5.0" prometheus-http-query = "0.8.2" prometheus = { version = "0.13.3", features = ["process"] } diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index 57713a92e..cfcd557c3 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -27,7 +27,7 @@ serde_yaml = "0.9.32" ic-registry-keys = { workspace = true } ic-registry-local-registry = { workspace = true } ic-management-backend = { workspace = true } -chrono = { version = "0.4", features = [ "serde" ] } +chrono = { version = "0.4", features = ["serde"] } url = { workspace = true } prometheus-http-query = { workspace = true } humantime-serde = "1.1.1" @@ -36,6 +36,8 @@ regex = { workspace = true } registry-canister = { workspace = true } candid = { workspace = true } ic-base-types = { workspace = true } +pretty_assertions = { workspace = true } +itertools = { workspace = true } [dev-dependencies] -rstest = "0.18.2" \ No newline at end of file +rstest = "0.18.2" diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index be669412e..a3e4a7cfd 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -42,13 +42,13 @@ pub struct Stage { update_unassigned_nodes: bool, } -#[derive(Deserialize, Clone, Default)] +#[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash)] pub struct Release { pub rc_name: String, pub versions: Vec, } -#[derive(Deserialize, Clone, Default)] +#[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash)] pub struct Version { pub version: String, pub name: String, diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 67fa89689..e2858ab33 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -2,8 +2,10 @@ use std::{collections::BTreeMap, time::Duration}; use chrono::{Datelike, Days, NaiveDate, Weekday}; use humantime::format_duration; +use ic_base_types::PrincipalId; use ic_management_backend::proposal::{SubnetUpdateProposal, UpdateUnassignedNodesProposal}; -use ic_management_types::Subnet; +use ic_management_types::{Release, Subnet}; +use itertools::Itertools; use slog::{debug, info, Logger}; use super::Stage; @@ -244,6 +246,52 @@ fn get_desired_version_for_subnet<'a>( }; } +fn desired_rollout_release_version( + subnets: Vec, + releases: Vec, +) -> BTreeMap { + let subnets_releases = subnets + .iter() + .map(|s| { + releases + .iter() + .find(|r| r.versions.iter().any(|v| v.version == s.replica_version)) + .expect("version should exist in releases") + }) + .unique() + .collect::>(); + // assumes `releases` are already sorted, but we can sort it if needed + if subnets_releases.len() > 2 { + panic!("more than two releases active") + } + let mut newest_release = releases + .iter() + .find(|r| subnets_releases.contains(r)) + .expect("should find some release"); + + if subnets_releases.len() == 1 { + newest_release = &releases[releases + .iter() + .position(|r| r == newest_release) + .expect("release should exist") + .saturating_sub(1)]; + } + // need to add some checks here to verify + subnets + .iter() + .map(|s| { + ( + s.principal, + newest_release + .versions + .iter() + .find_or_first(|v| v.subnets.iter().any(|vs| s.principal.to_string().starts_with(vs))) + .expect("versions should not be empty so it should return the first element if it doesn't match anything").clone(), + ) + }) + .collect() +} + fn find_subnet_by_short_id<'a>(subnets: &'a [Subnet], subnet_short: &'a str) -> anyhow::Result<&'a Subnet> { return match subnets .iter() @@ -564,6 +612,135 @@ mod get_desired_version_for_subnet_test { } } +#[cfg(test)] +mod test { + + use ic_base_types::PrincipalId; + use ic_management_types::SubnetMetadata; + use pretty_assertions::assert_eq; + + use crate::calculation::{Release, Version}; + + use super::*; + + #[test] + fn desired_version_test_cases() { + struct TestCase { + name: &'static str, + subnets: Vec, + releases: Vec, + want: BTreeMap, + } + + fn subnet(id: u64, version: &str) -> Subnet { + Subnet { + principal: PrincipalId::new_subnet_test_id(id), + replica_version: version.to_string(), + metadata: SubnetMetadata { + name: format!("{id}"), + ..Default::default() + }, + ..Default::default() + } + } + + fn release(name: &str, versions: Vec<(&str, Vec)>) -> Release { + Release { + rc_name: name.to_string(), + versions: versions + .iter() + .map(|(v, subnets)| Version { + version: v.to_string(), + subnets: subnets + .iter() + .map(|id| PrincipalId::new_subnet_test_id(*id).to_string()) + .collect(), + ..Default::default() + }) + .collect(), + } + } + + for tc in vec![ + TestCase { + name: "all versions on the newest version already", + subnets: vec![subnet(1, "A.default")], + releases: vec![release("A", vec![("A.default", vec![])])], + want: vec![(1, "A.default")] + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(), + }, + TestCase { + name: "upgrade one subnet", + subnets: vec![subnet(1, "B.default"), subnet(2, "A.default")], + releases: vec![ + release("B", vec![("B.default", vec![])]), + release("A", vec![("A.default", vec![])]), + ], + want: vec![(1, "B.default"), (2, "B.default")] + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(), + }, + TestCase { + name: "extra new and old releases are ignored", + subnets: vec![subnet(1, "C.default"), subnet(2, "B.default")], + releases: vec![ + release("D", vec![("D.default", vec![])]), + release("C", vec![("C.default", vec![])]), + release("B", vec![("B.default", vec![])]), + release("A", vec![("A.default", vec![])]), + ], + want: vec![(1, "C.default"), (2, "C.default")] + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(), + }, + TestCase { + name: "all subnets on same release, should proceed to upgrade everything to newer release", + subnets: vec![subnet(1, "B.default"), subnet(2, "B.default")], + releases: vec![ + release("D", vec![("D.default", vec![])]), + release("C", vec![("C.default", vec![]), ("C.feature", vec![2])]), + release("B", vec![("B.default", vec![])]), + release("A", vec![("A.default", vec![])]), + ], + want: vec![(1, "C.default"), (2, "C.feature")] + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(), + }, + TestCase { + name: "feature", + subnets: vec![subnet(1, "B.default"), subnet(2, "A.default"), subnet(3, "A.default")], + releases: vec![ + release("B", vec![("B.default", vec![]), ("B.feature", vec![2])]), + release("A", vec![("A.default", vec![])]), + ], + want: vec![(1, "B.default"), (2, "B.feature"), (3, "B.default")] + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(), + }, + ] { + let release = desired_rollout_release_version(tc.subnets, tc.releases) + .into_iter() + .map(|(k, v)| (k, v.version)) + .collect::>(); + assert_eq!( + tc.want + .into_iter() + .map(|(k, v)| (PrincipalId::new_subnet_test_id(k), v)) + .collect::>(), + release, + "test case '{}' failed", + tc.name, + ) + } + } +} + // E2E tests for decision making process for happy path without feature builds #[cfg(test)] mod check_stages_tests_no_feature_builds { From bb46f785a8ee9493ab1ee70a2184d8a443953fbe Mon Sep 17 00:00:00 2001 From: Luka Skugor Date: Fri, 15 Mar 2024 17:36:45 +0100 Subject: [PATCH 25/26] adapt tests for new logic --- rs/rollout-controller/src/calculation/mod.rs | 33 ++-- .../src/calculation/stage_checks.rs | 146 ++++++++---------- 2 files changed, 84 insertions(+), 95 deletions(-) diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index a3e4a7cfd..a75413e00 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -1,15 +1,16 @@ use std::{collections::BTreeMap, time::Duration}; use crate::calculation::should_proceed::should_proceed; -use chrono::{Local, NaiveDate}; +use chrono::{Local, NaiveDate, NaiveDateTime}; use ic_management_backend::registry::RegistryState; use ic_management_types::Subnet; use prometheus_http_query::Client; +use regex::Regex; use serde::Deserialize; use slog::{info, Logger}; use self::{ - release_actions::{create_current_release_feature_spec, find_latest_release}, + release_actions::create_current_release_feature_spec, stage_checks::{check_stages, SubnetAction}, }; @@ -48,6 +49,23 @@ pub struct Release { pub versions: Vec, } +impl Release { + pub fn date(&self) -> NaiveDateTime { + let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); + + NaiveDateTime::parse_from_str( + regex + .captures(&self.rc_name) + .expect("should have format with date") + .name("datetime") + .expect("should match group datetime") + .as_str(), + "%Y-%m-%d_%H-%M", + ) + .expect("should be valid date") + } +} + #[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash)] pub struct Version { pub version: String, @@ -69,13 +87,9 @@ pub async fn calculate_progress<'a>( return Ok(vec![]); } - let (latest_release, start_date) = find_latest_release(&index)?; + // TODO: this hsould be used somewhere else to check if proposal can be placed let elected_versions = registry_state.get_blessed_replica_versions().await?; - let (current_version, current_feature_spec) = - create_current_release_feature_spec(&latest_release, elected_versions) - .map_err(|e| anyhow::anyhow!(e.to_string()))?; - let mut last_bake_status: BTreeMap = BTreeMap::new(); let result = prometheus_client .query( @@ -106,16 +120,13 @@ pub async fn calculate_progress<'a>( let unassigned_nodes_proposals = registry_state.open_upgrade_unassigned_nodes_proposals().await?; let actions = check_stages( - ¤t_version, - ¤t_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - &index.rollout.stages, + index, Some(&logger), &unassigned_nodes_version, ®istry_state.subnets().into_values().collect::>(), - start_date.date(), Local::now().date_naive(), )?; diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index e2858ab33..1fee0554f 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -8,7 +8,7 @@ use ic_management_types::{Release, Subnet}; use itertools::Itertools; use slog::{debug, info, Logger}; -use super::Stage; +use super::{Index, Stage}; #[derive(Debug)] pub enum SubnetAction { @@ -34,24 +34,23 @@ pub enum SubnetAction { } pub fn check_stages<'a>( - current_version: &'a String, - current_release_feature_spec: &'a BTreeMap>, last_bake_status: &'a BTreeMap, subnet_update_proposals: &'a [SubnetUpdateProposal], unassigned_node_update_proposals: &'a [UpdateUnassignedNodesProposal], - stages: &'a [Stage], + index: Index, logger: Option<&'a Logger>, unassigned_version: &'a String, subnets: &'a [Subnet], - start_of_release: NaiveDate, now: NaiveDate, ) -> anyhow::Result> { - for (i, stage) in stages.iter().enumerate() { + let desired_versions = desired_rollout_release_version(subnets.to_vec(), index.releases); + for (i, stage) in index.rollout.stages.iter().enumerate() { if let Some(logger) = logger { info!(logger, "Checking stage {}", i) } - if stage.wait_for_next_week && !week_passed(start_of_release, now) { + let start_of_release = desired_versions.release.date(); + if stage.wait_for_next_week && !week_passed(start_of_release.date(), now) { let actions = stage .subnets .iter() @@ -63,8 +62,6 @@ pub fn check_stages<'a>( } let stage_actions = check_stage( - current_version, - current_release_feature_spec, last_bake_status, subnet_update_proposals, unassigned_node_update_proposals, @@ -72,6 +69,7 @@ pub fn check_stages<'a>( logger, unassigned_version, subnets, + desired_versions.clone(), )?; if !stage_actions.iter().all(|a| { @@ -89,7 +87,10 @@ pub fn check_stages<'a>( } if let Some(logger) = logger { - info!(logger, "The current rollout '{}' is completed.", current_version); + info!( + logger, + "The current rollout '{}' is completed.", desired_versions.release.rc_name + ); } Ok(vec![]) @@ -112,8 +113,6 @@ fn week_passed(release_start: NaiveDate, now: NaiveDate) -> bool { } fn check_stage<'a>( - current_version: &'a String, - current_release_feature_spec: &'a BTreeMap>, last_bake_status: &'a BTreeMap, subnet_update_proposals: &'a [SubnetUpdateProposal], unassigned_node_update_proposals: &'a [UpdateUnassignedNodesProposal], @@ -121,6 +120,7 @@ fn check_stage<'a>( logger: Option<&'a Logger>, unassigned_version: &'a String, subnets: &'a [Subnet], + desired_versions: DesiredReleaseVersion, ) -> anyhow::Result> { let mut stage_actions = vec![]; if stage.update_unassigned_nodes { @@ -129,11 +129,11 @@ fn check_stage<'a>( debug!(logger, "Unassigned nodes stage"); } - if !unassigned_version.eq(current_version) { + if *unassigned_version != desired_versions.unassigned_nodes.version { match unassigned_node_update_proposals.iter().find(|proposal| { if !proposal.info.executed { if let Some(version) = &proposal.payload.replica_version { - if version.eq(current_version) { + if *version == desired_versions.unassigned_nodes.version { return true; } } @@ -143,7 +143,7 @@ fn check_stage<'a>( None => stage_actions.push(SubnetAction::PlaceProposal { is_unassigned: true, subnet_principal: "".to_string(), - version: current_version.clone(), + version: desired_versions.unassigned_nodes.version, }), Some(proposal) => stage_actions.push(SubnetAction::PendingProposal { subnet_short: "unassigned-version".to_string(), @@ -161,21 +161,27 @@ fn check_stage<'a>( for subnet_short in &stage.subnets { // Get desired version - let desired_version = - get_desired_version_for_subnet(subnet_short, current_release_feature_spec, current_version); + let (subnet_principal, desired_version) = desired_versions + .subnets + .iter() + .find(|(s, v)| s.to_string().starts_with(subnet_short)) + .expect("should find the subnet"); // Find subnet to by the subnet_short - let subnet = find_subnet_by_short_id(subnets, subnet_short)?; + let subnet = subnets + .iter() + .find(|s| *subnet_principal == s.principal) + .expect("subnet should exist"); if let Some(logger) = logger { debug!( logger, - "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version + "Checking if subnet {} is on desired version '{}'", subnet_short, desired_version.version ); } // If subnet is on desired version, check bake time - if subnet.replica_version.eq(desired_version) { + if *subnet.replica_version == desired_version.version { let remaining = get_remaining_bake_time_for_subnet(last_bake_status, subnet, stage.bake_time.as_secs_f64())?; let remaining_duration = Duration::from_secs_f64(remaining); @@ -207,7 +213,8 @@ fn check_stage<'a>( } // If subnet is not on desired version, check if there is an open proposal - if let Some(proposal) = get_open_proposal_for_subnet(subnet_update_proposals, subnet, desired_version) { + if let Some(proposal) = get_open_proposal_for_subnet(subnet_update_proposals, subnet, &desired_version.version) + { if let Some(logger) = logger { info!( logger, @@ -225,7 +232,7 @@ fn check_stage<'a>( stage_actions.push(SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: subnet.principal.to_string(), - version: desired_version.to_string(), + version: desired_version.version.clone(), }) } @@ -246,10 +253,17 @@ fn get_desired_version_for_subnet<'a>( }; } +#[derive(Clone)] +struct DesiredReleaseVersion { + subnets: BTreeMap, + unassigned_nodes: crate::calculation::Version, + release: crate::calculation::Release, +} + fn desired_rollout_release_version( subnets: Vec, releases: Vec, -) -> BTreeMap { +) -> DesiredReleaseVersion { let subnets_releases = subnets .iter() .map(|s| { @@ -276,8 +290,9 @@ fn desired_rollout_release_version( .expect("release should exist") .saturating_sub(1)]; } - // need to add some checks here to verify - subnets + DesiredReleaseVersion { + release: newest_release.clone(), + subnets: subnets .iter() .map(|s| { ( @@ -289,7 +304,9 @@ fn desired_rollout_release_version( .expect("versions should not be empty so it should return the first element if it doesn't match anything").clone(), ) }) - .collect() + .collect(), + unassigned_nodes: newest_release.versions[0].clone(), + } } fn find_subnet_by_short_id<'a>(subnets: &'a [Subnet], subnet_short: &'a str) -> anyhow::Result<&'a Subnet> { @@ -724,16 +741,17 @@ mod test { .collect(), }, ] { - let release = desired_rollout_release_version(tc.subnets, tc.releases) - .into_iter() - .map(|(k, v)| (k, v.version)) - .collect::>(); + let desired_release = desired_rollout_release_version(tc.subnets, tc.releases); assert_eq!( tc.want .into_iter() .map(|(k, v)| (PrincipalId::new_subnet_test_id(k), v)) .collect::>(), - release, + desired_release + .subnets + .into_iter() + .map(|(k, v)| (k, v.version)) + .collect::>(), "test case '{}' failed", tc.name, ) @@ -912,21 +930,17 @@ mod check_stages_tests_no_feature_builds { let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index, None, &unassigned_version, subnets, - start_of_release, now, ); @@ -984,21 +998,17 @@ mod check_stages_tests_no_feature_builds { let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index, None, &unassigned_version, subnets, - start_of_release, now, ); @@ -1066,21 +1076,17 @@ mod check_stages_tests_no_feature_builds { let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1148,21 +1154,17 @@ mod check_stages_tests_no_feature_builds { let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1247,21 +1249,17 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1352,21 +1350,17 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposal, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1456,21 +1450,17 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-24", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposal, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1556,21 +1546,17 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposal, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1668,21 +1654,17 @@ mod check_stages_tests_no_feature_builds { ("pjljw", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let current_release_feature_spec = BTreeMap::new(); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposal, - stages, + index, None, &unassigned_version, &subnets, - start_of_release, now, ); @@ -1838,22 +1820,20 @@ mod check_stages_tests_feature_builds { .versions .get(1) .expect("Should be set to be the second version being rolled out"); + // TODO: replace in index let mut current_release_feature_spec = BTreeMap::new(); current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index.clone(), None, &unassigned_version, subnets, - start_of_release, now, ); @@ -1931,22 +1911,20 @@ mod check_stages_tests_feature_builds { .versions .get(1) .expect("Should be set to be the second version being rolled out"); + // TODO: replace in index let mut current_release_feature_spec = BTreeMap::new(); current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( - ¤t_version, - ¤t_release_feature_spec, &last_bake_status, &subnet_update_proposals, &unassigned_nodes_proposals, - stages, + index.clone(), None, &unassigned_version, &subnets, - start_of_release, now, ); From 8478d83575f6078ccde23f3073321a95b4aa96e6 Mon Sep 17 00:00:00 2001 From: nikolamilosa Date: Mon, 18 Mar 2024 10:36:10 +0100 Subject: [PATCH 26/26] removing unused code --- rs/rollout-controller/src/calculation/mod.rs | 9 +- .../src/calculation/release_actions.rs | 458 ------------------ .../src/calculation/stage_checks.rs | 185 +------ 3 files changed, 6 insertions(+), 646 deletions(-) delete mode 100644 rs/rollout-controller/src/calculation/release_actions.rs diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index a75413e00..99dd6a2e7 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -9,12 +9,8 @@ use regex::Regex; use serde::Deserialize; use slog::{info, Logger}; -use self::{ - release_actions::create_current_release_feature_spec, - stage_checks::{check_stages, SubnetAction}, -}; +use self::stage_checks::{check_stages, SubnetAction}; -mod release_actions; mod should_proceed; mod stage_checks; @@ -87,9 +83,6 @@ pub async fn calculate_progress<'a>( return Ok(vec![]); } - // TODO: this hsould be used somewhere else to check if proposal can be placed - let elected_versions = registry_state.get_blessed_replica_versions().await?; - let mut last_bake_status: BTreeMap = BTreeMap::new(); let result = prometheus_client .query( diff --git a/rs/rollout-controller/src/calculation/release_actions.rs b/rs/rollout-controller/src/calculation/release_actions.rs deleted file mode 100644 index 4c2b89f91..000000000 --- a/rs/rollout-controller/src/calculation/release_actions.rs +++ /dev/null @@ -1,458 +0,0 @@ -use std::{collections::BTreeMap, fmt::Display}; - -use chrono::NaiveDateTime; -use regex::Regex; - -use super::{Index, Release}; - -pub fn find_latest_release(index: &Index) -> anyhow::Result<(Release, NaiveDateTime)> { - let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); - - let mut mapped: Vec<(Release, NaiveDateTime)> = index - .releases - .iter() - .cloned() - .filter_map(|release| { - let captures = match regex.captures(&release.rc_name) { - Some(captures) => captures, - None => return None, - }; - let datetime_str = captures.name("datetime").unwrap().as_str(); - let datetime = match NaiveDateTime::parse_from_str(datetime_str, "%Y-%m-%d_%H-%M") { - Ok(val) => val, - Err(_) => return None, - }; - Some((release, datetime)) - }) - .collect(); - - mapped.sort_by_key(|(_, datetime)| *datetime); - mapped.reverse(); - - match mapped.first() { - Some((found, datetime)) => Ok((found.clone(), datetime.clone())), - None => Err(anyhow::anyhow!("There aren't any releases that match the criteria")), - } -} - -pub fn create_current_release_feature_spec( - current_release: &Release, - blessed_versions: Vec, -) -> Result<(String, BTreeMap>), CreateCurrentReleaseFeatureSpecError> { - let mut current_release_feature_spec: BTreeMap> = BTreeMap::new(); - let mut current_release_version = "".to_string(); - - for version in ¤t_release.versions { - if !blessed_versions.contains(&version.version) { - return Err(CreateCurrentReleaseFeatureSpecError::VersionNotBlessed { - rc: current_release.rc_name.to_string(), - version_name: version.name.to_string(), - version: version.version.to_string(), - }); - } - - if version.name.eq(¤t_release.rc_name) { - if current_release_version.is_empty() { - current_release_version = version.version.to_string(); - continue; - } - - // Version override attempt. Shouldn't be possible - return Err(CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { - rc: current_release.rc_name.to_string(), - version_name: version.name.to_string(), - }); - } - - if !version.name.eq(¤t_release.rc_name) && version.subnets.is_empty() { - return Err(CreateCurrentReleaseFeatureSpecError::FeatureBuildNoSubnets { - rc: current_release.rc_name.to_string(), - version_name: version.name.to_string(), - }); - } - - if current_release_feature_spec.contains_key(&version.version) { - return Err(CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { - rc: current_release.rc_name.to_string(), - version_name: version.name.to_string(), - }); - } - - current_release_feature_spec.insert(version.version.to_string(), version.subnets.clone()); - } - - if current_release_version.is_empty() { - return Err(CreateCurrentReleaseFeatureSpecError::CurrentVersionNotFound { - rc: current_release.rc_name.to_string(), - }); - } - - Ok((current_release_version, current_release_feature_spec)) -} - -#[derive(PartialEq, Debug)] -pub enum CreateCurrentReleaseFeatureSpecError { - CurrentVersionNotFound { - rc: String, - }, - FeatureBuildNoSubnets { - rc: String, - version_name: String, - }, - VersionNotBlessed { - rc: String, - version_name: String, - version: String, - }, - VersionSpecifiedTwice { - rc: String, - version_name: String, - }, -} - -impl Display for CreateCurrentReleaseFeatureSpecError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - CreateCurrentReleaseFeatureSpecError::CurrentVersionNotFound { rc } => f.write_fmt(format_args!( - "Regular release version not found for release named: {}", - rc - )), - CreateCurrentReleaseFeatureSpecError::FeatureBuildNoSubnets { rc, version_name } => { - f.write_fmt(format_args!( - "Feature build '{}' that is part of rc '{}' doesn't have subnets specified", - version_name, rc - )) - } - CreateCurrentReleaseFeatureSpecError::VersionNotBlessed { - rc, - version, - version_name, - } => f.write_fmt(format_args!( - "Version '{}', named '{}' that is part of rc '{}' is not blessed", - version, version_name, rc - )), - CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { rc, version_name } => f.write_fmt( - format_args!("Version '{}' is defined twice within rc named '{}'", version_name, rc), - ), - } - } -} - -#[cfg(test)] -mod find_latest_release_tests { - use chrono::NaiveDate; - - use super::*; - - #[test] - fn should_not_find_release_none_match_regex() { - let index = Index { - releases: vec![ - Release { - rc_name: String::from("bad-name"), - versions: Default::default(), - }, - Release { - rc_name: String::from("rc--kind-of-ok_no-no"), - versions: Default::default(), - }, - ], - ..Default::default() - }; - - let latest = find_latest_release(&index); - - assert!(latest.is_err()); - } - - #[test] - fn should_return_latest_correct_release() { - let index = Index { - releases: vec![ - Release { - rc_name: String::from("rc--kind-of-ok_no-no"), - ..Default::default() - }, - Release { - rc_name: String::from("rc--2024-03-09_23-01"), - ..Default::default() - }, - Release { - rc_name: String::from("rc--2024-03-10_23-01"), - ..Default::default() - }, - ], - ..Default::default() - }; - - let latest = find_latest_release(&index); - - assert!(latest.is_ok()); - let (latest, date) = latest.unwrap(); - - assert_eq!(latest.rc_name, String::from("rc--2024-03-10_23-01")); - assert_eq!( - date.date(), - NaiveDate::parse_from_str("2024-03-10", "%Y-%m-%d").expect("Should parse date") - ) - } - - #[test] - fn should_not_return_release_empty_index() { - let index = Index { ..Default::default() }; - - let latest = find_latest_release(&index); - - assert!(latest.is_err()) - } -} - -#[cfg(test)] -mod create_current_release_feature_spec_tests { - use crate::calculation::Version; - - use super::*; - - #[test] - fn should_create_map() { - let blessed_versions = vec![ - "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", - "85bd56a70e55b2cea75cae6405ae11243e5fdad8", - ] - .iter() - .map(|v| v.to_string()) - .collect::>(); - let current_release = Release { - rc_name: "rc--2024-03-10_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-03-10_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01-p2p".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), - ..Default::default() - }, - ], - }; - - let response = create_current_release_feature_spec(¤t_release, blessed_versions); - - assert!(response.is_ok()); - let (current_version, feat_map) = response.unwrap(); - assert_eq!(current_version, "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string()); - assert_eq!(feat_map.len(), 1); - - let (version, subnets) = feat_map.first_key_value().unwrap(); - assert_eq!(version, "85bd56a70e55b2cea75cae6405ae11243e5fdad8"); - assert_eq!(subnets.len(), 1); - let subnet = subnets.first().unwrap(); - assert_eq!(subnet, "shefu"); - } - - #[test] - fn shouldnt_create_map_regular_version_not_blessed() { - let blessed_versions = vec!["85bd56a70e55b2cea75cae6405ae11243e5fdad8"] - .iter() - .map(|v| v.to_string()) - .collect::>(); - - let current_release = Release { - rc_name: "rc--2024-03-10_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-03-10_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01-p2p".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), - ..Default::default() - }, - ], - }; - - let response = create_current_release_feature_spec(¤t_release, blessed_versions); - - assert!(response.is_err()); - let error = response.err().unwrap(); - assert_eq!( - error, - CreateCurrentReleaseFeatureSpecError::VersionNotBlessed { - rc: "rc--2024-03-10_23-01".to_string(), - version_name: "rc--2024-03-10_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string() - } - ) - } - - #[test] - fn shouldnt_create_map_version_override() { - let blessed_versions = vec![ - "85bd56a70e55b2cea75cae6405ae11243e5fdad8", - "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", - ] - .iter() - .map(|v| v.to_string()) - .collect::>(); - - let current_release = Release { - rc_name: "rc--2024-03-10_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-03-10_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), - ..Default::default() - }, - ], - }; - - let response = create_current_release_feature_spec(¤t_release, blessed_versions); - - assert!(response.is_err()); - let error = response.err().unwrap(); - assert_eq!( - error, - CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { - rc: "rc--2024-03-10_23-01".to_string(), - version_name: "rc--2024-03-10_23-01".to_string() - } - ) - } - - #[test] - fn shouldnt_create_map_version_override_for_feature_builds() { - let blessed_versions = vec![ - "85bd56a70e55b2cea75cae6405ae11243e5fdad8", - "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", - ] - .iter() - .map(|v| v.to_string()) - .collect::>(); - - let current_release = Release { - rc_name: "rc--2024-03-10_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-03-10_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01-p2p".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01-p2p2".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - subnets: vec!["shefu"].iter().map(|s| s.to_string()).collect(), - ..Default::default() - }, - ], - }; - - let response = create_current_release_feature_spec(¤t_release, blessed_versions); - - assert!(response.is_err()); - let error = response.err().unwrap(); - assert_eq!( - error, - CreateCurrentReleaseFeatureSpecError::VersionSpecifiedTwice { - rc: "rc--2024-03-10_23-01".to_string(), - version_name: "rc--2024-03-10_23-01-p2p2".to_string() - } - ) - } - - #[test] - fn shouldnt_create_map_version_no_subnets_for_feature() { - let blessed_versions = vec![ - "85bd56a70e55b2cea75cae6405ae11243e5fdad8", - "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", - ] - .iter() - .map(|v| v.to_string()) - .collect::>(); - - let current_release = Release { - rc_name: "rc--2024-03-10_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-03-10_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01-p2p".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }, - ], - }; - - let response = create_current_release_feature_spec(¤t_release, blessed_versions); - - assert!(response.is_err()); - let error = response.err().unwrap(); - assert_eq!( - error, - CreateCurrentReleaseFeatureSpecError::FeatureBuildNoSubnets { - rc: "rc--2024-03-10_23-01".to_string(), - version_name: "rc--2024-03-10_23-01-p2p".to_string() - } - ) - } - - #[test] - fn shouldnt_create_map_version_no_regular_build() { - let blessed_versions = vec![ - "85bd56a70e55b2cea75cae6405ae11243e5fdad8", - "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f", - ] - .iter() - .map(|v| v.to_string()) - .collect::>(); - - let current_release = Release { - rc_name: "rc--2024-03-10_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-03-10_23-01-notregular".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - subnets: vec!["shefu".to_string()], - ..Default::default() - }, - Version { - name: "rc--2024-03-10_23-01-p2p".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - subnets: vec!["qdvhd".to_string()], - ..Default::default() - }, - ], - }; - - let response = create_current_release_feature_spec(¤t_release, blessed_versions); - - assert!(response.is_err()); - let error = response.err().unwrap(); - assert_eq!( - error, - CreateCurrentReleaseFeatureSpecError::CurrentVersionNotFound { - rc: "rc--2024-03-10_23-01".to_string(), - } - ) - } -} diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 1fee0554f..2a918f069 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -4,7 +4,7 @@ use chrono::{Datelike, Days, NaiveDate, Weekday}; use humantime::format_duration; use ic_base_types::PrincipalId; use ic_management_backend::proposal::{SubnetUpdateProposal, UpdateUnassignedNodesProposal}; -use ic_management_types::{Release, Subnet}; +use ic_management_types::Subnet; use itertools::Itertools; use slog::{debug, info, Logger}; @@ -164,7 +164,7 @@ fn check_stage<'a>( let (subnet_principal, desired_version) = desired_versions .subnets .iter() - .find(|(s, v)| s.to_string().starts_with(subnet_short)) + .find(|(s, _)| s.to_string().starts_with(subnet_short)) .expect("should find the subnet"); // Find subnet to by the subnet_short @@ -239,20 +239,6 @@ fn check_stage<'a>( Ok(stage_actions) } -fn get_desired_version_for_subnet<'a>( - subnet_short: &'a str, - current_release_feature_spec: &'a BTreeMap>, - current_version: &'a str, -) -> &'a str { - return match current_release_feature_spec - .iter() - .find(|(_, subnets)| subnets.contains(&subnet_short.to_string())) - { - Some((name, _)) => name, - None => current_version, - }; -} - #[derive(Clone)] struct DesiredReleaseVersion { subnets: BTreeMap, @@ -309,16 +295,6 @@ fn desired_rollout_release_version( } } -fn find_subnet_by_short_id<'a>(subnets: &'a [Subnet], subnet_short: &'a str) -> anyhow::Result<&'a Subnet> { - return match subnets - .iter() - .find(|s| s.principal.to_string().starts_with(subnet_short)) - { - Some(subnet) => Ok(subnet), - None => Err(anyhow::anyhow!("No subnet with short id '{}'", subnet_short)), - }; -} - fn get_remaining_bake_time_for_subnet( last_bake_status: &BTreeMap, subnet: &Subnet, @@ -393,19 +369,6 @@ mod get_open_proposal_for_subnet_tests { } } - pub(super) fn craft_subnet_from_similar_id<'a>(subnet_id: &'a str) -> Subnet { - let original_principal = Principal::from_str(subnet_id).expect("Can create principal"); - let mut new_principal = original_principal.as_slice().to_vec(); - let len = new_principal.len(); - if let Some(byte) = new_principal.get_mut(len - 1) { - *byte += 1; - } - Subnet { - principal: PrincipalId(Principal::try_from_slice(&new_principal[..]).expect("Can create principal")), - ..Default::default() - } - } - pub(super) fn craft_proposals<'a>( subnet_with_execution_status: &'a [(&'a str, bool)], version: &'a str, @@ -557,78 +520,6 @@ mod get_remaining_bake_time_for_subnet_tests { } } -#[cfg(test)] -mod find_subnet_by_short_id_tests { - use self::get_open_proposal_for_subnet_tests::{craft_subnet_from_id, craft_subnet_from_similar_id}; - - use super::*; - - #[test] - fn should_find_subnet() { - let subnet_1 = craft_subnet_from_id("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae"); - let subnet_2 = craft_subnet_from_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); - let subnet_3 = craft_subnet_from_similar_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); - let subnets = &[subnet_1, subnet_2, subnet_3]; - - let maybe_subnet = find_subnet_by_short_id(subnets, "snjp4"); - - assert!(maybe_subnet.is_ok()); - let subnet = maybe_subnet.unwrap(); - let subnet_principal = subnet.principal.to_string(); - assert!(subnet_principal.starts_with("snjp4")); - assert!(subnet_principal.eq("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae")); - } - - #[test] - fn should_find_not_subnet() { - let subnet_1 = craft_subnet_from_id("5kdm2-62fc6-fwnja-hutkz-ycsnm-4z33i-woh43-4cenu-ev7mi-gii6t-4ae"); - let subnet_2 = craft_subnet_from_id("snjp4-xlbw4-mnbog-ddwy6-6ckfd-2w5a2-eipqo-7l436-pxqkh-l6fuv-vae"); - let subnets = &[subnet_1, subnet_2]; - - let maybe_subnet = find_subnet_by_short_id(subnets, "random-subnet"); - - assert!(maybe_subnet.is_err()); - } -} - -#[cfg(test)] -mod get_desired_version_for_subnet_test { - use super::*; - - pub(super) fn craft_feature_spec(tuples: &[(&str, &[&str])]) -> BTreeMap> { - tuples - .iter() - .map(|(commit, subnets)| (commit.to_string(), subnets.iter().map(|id| id.to_string()).collect())) - .collect::>>() - } - - #[test] - fn should_return_current_version() { - let current_release_feature_spec = - craft_feature_spec(&[("feature-a-commit", &["subnet-1", "subnet-2", "subnet-3"])]); - - let version = get_desired_version_for_subnet("subnet", ¤t_release_feature_spec, "current_version"); - assert_eq!(version, "current_version") - } - - #[test] - fn should_return_current_version_empty_feature_spec() { - let current_release_feature_spec = craft_feature_spec(&vec![]); - - let version = get_desired_version_for_subnet("subnet", ¤t_release_feature_spec, "current_version"); - assert_eq!(version, "current_version") - } - - #[test] - fn should_return_feature_version() { - let current_release_feature_spec = - craft_feature_spec(&[("feature-a-commit", &["subnet-1", "subnet-2", "subnet-3"])]); - - let version = get_desired_version_for_subnet("subnet-1", ¤t_release_feature_spec, "current_version"); - assert_eq!(version, "feature-a-commit") - } -} - #[cfg(test)] mod test { @@ -778,15 +669,13 @@ mod check_stages_tests_no_feature_builds { use super::*; /// Part one => No feature builds - /// `current_version` - can be defined - /// `current_release_feature_spec` - empty because we don't have feature builds for this part /// `last_bake_status` - can be defined /// `subnet_update_proposals` - can be defined - /// `stages` - must be defined + /// `unassigned_node_update_proposals` - can be defined + /// `index` - must be defined /// `logger` - can be defined, but won't be because these are only tests /// `unassigned_version` - should be defined /// `subnets` - should be defined - /// `start_of_release` - should be defined /// `now` - should be defined /// /// For all use cases we will use the following setup @@ -911,14 +800,11 @@ mod check_stages_tests_no_feature_builds { /// Use-Case 1: Beginning of a new rollout /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - empty, because no subnets have the version /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS /// But it is for a different version (one from last week) /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_1() { @@ -926,11 +812,9 @@ mod check_stages_tests_no_feature_builds { let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); let last_bake_status = BTreeMap::new(); let subnet_update_proposals = Vec::new(); - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -967,18 +851,14 @@ mod check_stages_tests_no_feature_builds { /// Use case 2: First batch is submitted but the proposal wasn't executed /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - empty, because no subnets have the version /// `subnet_update_proposals` - contains proposals from the first stage /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_2() { let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); let last_bake_status = BTreeMap::new(); let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") .expect("Should be possible to create principal"); @@ -994,11 +874,9 @@ mod check_stages_tests_no_feature_builds { replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), }, }]; - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1033,18 +911,14 @@ mod check_stages_tests_no_feature_builds { /// Use case 3: First batch is submitted the proposal was executed and the subnet is baking /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for the first subnet /// `subnet_update_proposals` - contains proposals from the first stage /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_3() { let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); let last_bake_status = [( "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", humantime::parse_duration("3h"), @@ -1071,12 +945,10 @@ mod check_stages_tests_no_feature_builds { replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), }, }]; - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1111,13 +983,10 @@ mod check_stages_tests_no_feature_builds { /// Use case 4: First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for the first subnet /// `subnet_update_proposals` - contains proposals from the first stage /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_4() { @@ -1149,12 +1018,10 @@ mod check_stages_tests_no_feature_builds { replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), }, }]; - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1195,13 +1062,10 @@ mod check_stages_tests_no_feature_builds { /// Use case 5: Updating unassigned nodes /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for all subnets before unassigned nodes /// `subnet_update_proposals` - contains proposals from previous two stages /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_5() { @@ -1237,7 +1101,6 @@ mod check_stages_tests_no_feature_builds { ], ¤t_version, ); - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); @@ -1249,7 +1112,6 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1285,13 +1147,10 @@ mod check_stages_tests_no_feature_builds { /// Use case 6: Proposal sent for updating unassigned nodes but it is not executed /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for all subnets before unassigned nodes /// `subnet_update_proposals` - contains proposals from previous two stages /// `unassigned_nodes_proposals` - contains open proposal for unassigned nodes - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_6() { @@ -1327,7 +1186,6 @@ mod check_stages_tests_no_feature_builds { ], ¤t_version, ); - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { info: ProposalInfoInternal { @@ -1350,7 +1208,6 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1385,13 +1242,10 @@ mod check_stages_tests_no_feature_builds { /// Use case 7: Executed update unassigned nodes, waiting for next week /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for all subnets before unassigned nodes /// `subnet_update_proposals` - contains proposals from previous two stages /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes - /// `unassigned_version` - same as `current_version` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-24` #[test] fn test_use_case_7() { @@ -1427,7 +1281,6 @@ mod check_stages_tests_no_feature_builds { ], ¤t_version, ); - let stages = &index.rollout.stages; let unassigned_version = current_version.clone(); let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { info: ProposalInfoInternal { @@ -1450,7 +1303,6 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-24", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1481,13 +1333,10 @@ mod check_stages_tests_no_feature_builds { /// Use case 8: Next monday came, should place proposal for updating the last subnet /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for all subnets before unassigned nodes /// `subnet_update_proposals` - contains proposals from previous two stages /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes - /// `unassigned_version` - same as `current_version` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-26` #[test] fn test_use_case_8() { @@ -1523,7 +1372,6 @@ mod check_stages_tests_no_feature_builds { ], ¤t_version, ); - let stages = &index.rollout.stages; let unassigned_version = current_version.clone(); let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { info: ProposalInfoInternal { @@ -1546,7 +1394,6 @@ mod check_stages_tests_no_feature_builds { ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1583,13 +1430,10 @@ mod check_stages_tests_no_feature_builds { /// Use case 9: Next monday came, proposal for last subnet executed and bake time passed. Rollout finished /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` /// `last_bake_status` - contains the status for all subnets before unassigned nodes /// `subnet_update_proposals` - contains proposals from previous two stages /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes - /// `unassigned_version` - same as `current_version` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-26` #[test] fn test_use_case_9() { @@ -1630,7 +1474,6 @@ mod check_stages_tests_no_feature_builds { ], ¤t_version, ); - let stages = &index.rollout.stages; let unassigned_version = current_version.clone(); let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { info: ProposalInfoInternal { @@ -1654,7 +1497,6 @@ mod check_stages_tests_no_feature_builds { ("pjljw", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), ], ); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1690,16 +1532,12 @@ mod check_stages_tests_feature_builds { use super::*; /// Part two => Feature builds - /// `current_version` - has to be defined - /// `current_release_feature_spec` - has to be defined with mapped versions to subnets /// `last_bake_status` - can be defined depending on the use case /// `subnet_update_proposals` - can be defined depending on the use case /// `unassigned_nodes_update_proposals` - can be defined depending on the use case - /// `stages` - has to be defined + /// `index` - has to be defined /// `logger` - can be defined, but won't be because these are only tests - /// `unassigned_version` - has to be defined /// `subnets` - has to be defined - /// `start_of_release` - has to be defined /// `now` - has to be defined /// /// For all use cases we will use the following setup @@ -1793,23 +1631,17 @@ mod check_stages_tests_feature_builds { /// Use-Case 1: Beginning of a new rollout /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f - /// `current_release_feature_spec` - contains the spec for version `76521ef765e86187c43f7d6a02e63332a6556c8c` /// `last_bake_status` - empty, because no subnets have the version /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS /// But it is for a different version (one from last week) /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_1() { let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); let last_bake_status = BTreeMap::new(); let subnet_update_proposals = Vec::new(); - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let subnets = &craft_subnets(); @@ -1823,7 +1655,6 @@ mod check_stages_tests_feature_builds { // TODO: replace in index let mut current_release_feature_spec = BTreeMap::new(); current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages( @@ -1860,14 +1691,10 @@ mod check_stages_tests_feature_builds { /// Use case 2: First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage /// - /// `current_version` - set to a commit that is being rolled out `2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f` - /// `current_release_feature_spec` - contains the spec for version `76521ef765e86187c43f7d6a02e63332a6556c8c` /// `last_bake_status` - contains the status for the first subnet /// `subnet_update_proposals` - contains proposals from the first stage /// `unassigned_nodes_proposals` - empty - /// `unassigned_version` - one from previous week `85bd56a70e55b2cea75cae6405ae11243e5fdad8` /// `subnets` - can be seen in `craft_index_state` - /// `start_of_release` - some `2024-02-21` /// `now` - same `2024-02-21` #[test] fn test_use_case_2() { @@ -1899,7 +1726,6 @@ mod check_stages_tests_feature_builds { replica_version_id: "76521ef765e86187c43f7d6a02e63332a6556c8c".to_string(), }, }]; - let stages = &index.rollout.stages; let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); let unassigned_nodes_proposals = vec![]; let mut subnets = craft_subnets(); @@ -1914,7 +1740,6 @@ mod check_stages_tests_feature_builds { // TODO: replace in index let mut current_release_feature_spec = BTreeMap::new(); current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); - let start_of_release = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); let maybe_actions = check_stages(