Upstairs state machine refactoring (3/3)

.github/buildomat/jobs/test-ds.sh

@@ -39,6 +39,7 @@ for t in "$input/bins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 banner test_ds
 ptime -m bash "$input/scripts/test_ds.sh"

.github/buildomat/jobs/test-live-repair.sh

@@ -78,6 +78,7 @@ for t in "$input/bins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 echo "BINDIR is $BINDIR"
 echo "bindir contains:"

.github/buildomat/jobs/test-memory.sh

@@ -39,6 +39,7 @@ for t in "$input/rbins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 banner setup
 pfexec plimit -n 9123456 $$

.github/buildomat/jobs/test-region-create.sh

@@ -39,6 +39,7 @@ for t in "$input/rbins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 banner region
 pfexec plimit -n 9123456 $$

.github/buildomat/jobs/test-repair.sh

@@ -45,6 +45,7 @@ for t in "$input/bins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 echo "Setup self timeout"
 # Give this test two hours to finish

.github/buildomat/jobs/test-replay.sh

@@ -43,6 +43,7 @@ for t in "$input/bins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 banner setup
 
 echo "Setup self timeout"

.github/buildomat/jobs/test-up-encrypted.sh

@@ -41,6 +41,7 @@ for t in "$input/bins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 # Give this test one hour to finish
 jobpid=$$; (sleep $(( 60 * 60 )); banner fail-timeout; ps -ef; zfs list;kill $jobpid) &

.github/buildomat/jobs/test-up-unencrypted.sh

@@ -41,6 +41,7 @@ for t in "$input/bins/"*.gz; do
 done
 
 export BINDIR=/var/tmp/bins
+export RUST_BACKTRACE=1
 
 # Give this test two hours to finish
 jobpid=$$; (sleep $(( 120 * 60 )); banner fail-timeout; ps -ef; zfs list;kill $jobpid) &

cmon/src/main.rs

@@ -8,7 +8,9 @@ use strum::IntoEnumIterator;
 use strum_macros::EnumIter;
 use tokio::time::{sleep, Duration};
 
-use crucible::{Arg, ClientStopReason, DsState};
+use crucible::{
+    Arg, ClientStopReason, ConnectionMode, DsState, NegotiationState,
+};
 
 /// Connect to crucible control server
 #[derive(Parser, Debug)]
@@ -87,24 +89,46 @@ enum Action {
 // Translate a DsState into a three letter string for printing.
 fn short_state(dss: DsState) -> String {
     match dss {
-        DsState::New
-        | DsState::Stopping(ClientStopReason::NegotiationFailed(..)) => {
-            "NEW".to_string()
-        }
-        DsState::WaitActive => "WAC".to_string(),
-        DsState::WaitQuorum => "WAQ".to_string(),
-        DsState::Reconcile => "REC".to_string(),
+        DsState::Connecting {
+            state: NegotiationState::WaitActive,
+            ..
+        } => "WAC".to_string(),
+
+        DsState::Connecting {
+            state: NegotiationState::WaitQuorum,
+            ..
+        } => "WAQ".to_string(),
+        DsState::Connecting {
+            state: NegotiationState::Reconcile,
+            ..
+        } => "REC".to_string(),
         DsState::Active => "ACT".to_string(),
-        DsState::Faulted | DsState::Stopping(ClientStopReason::Fault(..)) => {
-            "FLT".to_string()
+        DsState::Connecting {
+            state: NegotiationState::LiveRepairReady,
+            ..
+        } => "LRR".to_string(),
+        DsState::Stopping(ClientStopReason::NegotiationFailed(..))
+        | DsState::Connecting {
+            mode: ConnectionMode::New,
+            ..
+        } => "NEW".to_string(),
+        DsState::Connecting {
+            mode: ConnectionMode::Faulted,
+            ..
         }
-        DsState::LiveRepairReady => "LRR".to_string(),
+        | DsState::Stopping(ClientStopReason::Fault(..)) => "FLT".to_string(),
         DsState::LiveRepair => "LR".to_string(),
-        DsState::Offline => "OFF".to_string(),
+        DsState::Connecting {
+            mode: ConnectionMode::Offline,
+            ..
+        } => "OFF".to_string(),
         DsState::Stopping(ClientStopReason::Deactivated) => "DAV".to_string(),
         DsState::Stopping(ClientStopReason::Disabled) => "DIS".to_string(),
-        DsState::Stopping(ClientStopReason::Replacing) => "RPC".to_string(),
-        DsState::Replaced => "RPD".to_string(),
+        DsState::Stopping(ClientStopReason::Replacing)
+        | DsState::Connecting {
+            mode: ConnectionMode::Replaced,
+            ..
+        } => "RPD".to_string(),
     }
 }
 

common/src/lib.rs

@@ -133,9 +133,6 @@ pub enum CrucibleError {
     #[error("Repair stream error {0}")]
     RepairStreamError(String),
 
-    #[error("Generation number is too low: {0}")]
-    GenerationNumberTooLow(String),
-
     #[error("Active with different generation number")]
     GenerationNumberInvalid,
 
@@ -165,6 +162,9 @@ pub enum CrucibleError {
 
     #[error("Incompatible RegionDefinition {0}")]
     RegionIncompatible(String),
+
+    #[error("Negotiation error: {0}")]
+    NegotiationError(NegotiationError),
 }
 
 impl From<std::io::Error> for CrucibleError {
@@ -197,6 +197,61 @@ impl<T> From<std::sync::mpsc::SendError<T>> for CrucibleError {
     }
 }
 
+impl From<NegotiationError> for CrucibleError {
+    fn from(value: NegotiationError) -> Self {
+        CrucibleError::NegotiationError(value)
+    }
+}
+
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(
+    thiserror::Error,
+    Debug,
+    PartialEq,
+    Clone,
+    Copy,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+)]
+pub enum NegotiationError {
+    #[error("Message received out of order")]
+    OutOfOrder,
+
+    #[error("Generation 0 is illegal")]
+    GenerationZeroIsIllegal,
+
+    #[error(
+        "Generation number is too low: requested {requested}, found {actual}"
+    )]
+    GenerationNumberTooLow { requested: u64, actual: u64 },
+
+    #[error("Incompatible message version: wanted {expected}, got {actual}")]
+    IncompatibleVersion { expected: u32, actual: u32 },
+
+    #[error(
+        "Incompatible encryption settings: wanted {expected}, got {actual}"
+    )]
+    EncryptionMismatch { expected: bool, actual: bool },
+
+    #[error(
+        "Incompatible read-only settings: wanted {expected}, got {actual}"
+    )]
+    ReadOnlyMismatch { expected: bool, actual: bool },
+
+    #[error("Incompatible upstairs ID: wanted {expected}, got {actual}")]
+    UpstairsIdMismatch {
+        expected: uuid::Uuid,
+        actual: uuid::Uuid,
+    },
+
+    #[error("Incompatible session ID: wanted {expected}, got {actual}")]
+    SessionIdMismatch {
+        expected: uuid::Uuid,
+        actual: uuid::Uuid,
+    },
+}
+
 #[macro_export]
 macro_rules! crucible_bail {
     ($i:ident) => { return Err(CrucibleError::$i) };
@@ -394,7 +449,6 @@ impl From<CrucibleError> for dropshot::HttpError {
             | CrucibleError::DecryptionError
             | CrucibleError::Disconnect
             | CrucibleError::EncryptionError(_)
-            | CrucibleError::GenerationNumberTooLow(_)
             | CrucibleError::GenerationNumberInvalid
             | CrucibleError::GenericError(_)
             | CrucibleError::HashMismatch
@@ -417,7 +471,8 @@ impl From<CrucibleError> for dropshot::HttpError {
             | CrucibleError::MissingContextSlot(..)
             | CrucibleError::BadMetadata(..)
             | CrucibleError::BadContextSlot(..)
-            | CrucibleError::MissingBlockContext => {
+            | CrucibleError::MissingBlockContext
+            | CrucibleError::NegotiationError(..) => {
                 dropshot::HttpError::for_internal_error(e.to_string())
             }
         }