Add infrastructure to simulate the OpenHCL test failures

openhcl/underhill_core/src/dispatch/mod.rs

@@ -179,6 +179,7 @@ pub(crate) struct LoadedVm {
     pub shared_vis_pool: Option<PagePool>,
     pub private_pool: Option<PagePool>,
     pub nvme_keep_alive: bool,
+    pub test_configuration: Option<String>,
 }
 
 pub struct LoadedVmState<T> {
@@ -433,11 +434,32 @@ impl LoadedVm {
         deadline: std::time::Instant,
         capabilities_flags: SaveGuestVtl2StateFlags,
     ) -> anyhow::Result<bool> {
+        if let Some(config) = &self.test_configuration {
+            if config == "SERVICING_SAVE_STUCK" {
+                tracing::info!(
+                    "Test configuration SERVICING_SAVE_STUCK is set. Waiting indefinitely."
+                );
+                loop {
+                    std::thread::sleep(Duration::from_secs(1));
+                }
+            }
+        }
+
         let running = self.state_units.is_running();
         let success = match self
             .handle_servicing_inner(correlation_id, deadline, capabilities_flags)
             .await
-        {
+            .and_then(|state| {
+                if let Some(config) = &self.test_configuration {
+                    if config == "SERVICING_SAVE_FAIL" {
+                        tracing::info!(
+                            "Test configuration SERVICING_SAVE_FAIL is set. Failing the save."
+                        );
+                        return Err(anyhow::anyhow!("Simulated servicing save failure"));
+                    }
+                }
+                Ok(state)
+            }) {
             Ok(state) => {
                 self.get_client
                     .send_servicing_state(mesh::payload::encode(state))

openhcl/underhill_core/src/lib.rs

@@ -323,6 +323,7 @@ async fn launch_workers(
         gdbstub: opt.gdbstub,
         hide_isolation: opt.hide_isolation,
         nvme_keep_alive: opt.nvme_keep_alive,
+        test_configuration: opt.test_configuration,
     };
 
     let (mut remote_console_cfg, framebuffer_access) =

openhcl/underhill_core/src/options.rs

@@ -119,6 +119,11 @@ pub struct Options {
 
     /// (OPENHCL_NVME_KEEP_ALIVE=1) Enable nvme keep alive when servicing.
     pub nvme_keep_alive: bool,
+
+    /// (OPENHCL_TEST_CONFIG=\<string\>)
+    /// Test configurations are designed to replicate specific behaviors and
+    /// conditions in order to simulate various test scenarios.
+    pub test_configuration: Option<String>,
 }
 
 impl Options {
@@ -203,6 +208,8 @@ impl Options {
         let gdbstub = parse_legacy_env_bool("OPENHCL_GDBSTUB");
         let gdbstub_port = parse_legacy_env_number("OPENHCL_GDBSTUB_PORT")?.map(|x| x as u32);
         let nvme_keep_alive = parse_env_bool("OPENHCL_NVME_KEEP_ALIVE");
+        let test_configuration =
+            legacy_openhcl_env("OPENHCL_TEST_CONFIG").map(|x| x.to_string_lossy().into_owned());
 
         let mut args = std::env::args().chain(extra_args);
         // Skip our own filename.
@@ -257,6 +264,7 @@ impl Options {
             halt_on_guest_halt,
             no_sidecar_hotplug,
             nvme_keep_alive,
+            test_configuration,
         })
     }
 

openhcl/underhill_core/src/worker.rs

@@ -300,6 +300,9 @@ pub struct UnderhillEnvCfg {
     pub hide_isolation: bool,
     /// Enable nvme keep alive.
     pub nvme_keep_alive: bool,
+
+    /// test configuration
+    pub test_configuration: Option<String>,
 }
 
 /// Bundle of config + runtime objects for hooking into the underhill remote
@@ -494,6 +497,17 @@ impl UnderhillVmWorker {
                 "cannot have saved state from two different sources"
             );
 
+            if let Some(config) = &params.env_cfg.test_configuration {
+                if config == "SERVICING_RESTORE_STUCK" {
+                    tracing::info!(
+                        "Test configuration SERVICING_RESTORE_STUCK is set. Waiting indefinitely in restore"
+                    );
+                    loop {
+                        std::thread::sleep(Duration::from_secs(1));
+                    }
+                }
+            }
+
             tracing::info!("VTL2 restart, getting servicing state from the host");
 
             let saved_state_buf = get_client
@@ -3066,6 +3080,7 @@ async fn new_underhill_vm(
         shared_vis_pool: shared_vis_pages_pool,
         private_pool,
         nvme_keep_alive: env_cfg.nvme_keep_alive,
+        test_configuration: env_cfg.test_configuration,
     };
 
     Ok(loaded_vm)