apiserver: add version 2 for /tx and /metadata/settings-generator

- `/v2/tx`: We will also return the pending metadata along with pending
  settings(that we used to return in version 1). As the return struct is
changing, we are doing versioning of the API.

- `v2/metadata/settings-generators`: We will also return the
  settings-generators(that contains strength and are saved as JSON
object in datastore). As we just used to return arrays and string
earlier as response for this API, returning object may break the
existing usage. Hence we need to version this API.

- `/settings`(patch and patchkeypair): For both of these we will set
  strength metadata. The default strength used is strong.

- `/tx/commit` and `/tx/commit_and_apply`:  We will commit the pending
  metadata(that just accounts for strength metadata for now) as part of
commit. No changes has been done in apply.

sources/api/apiserver/Cargo.toml

@@ -27,6 +27,7 @@ num.workspace = true
 rand = { workspace = true, features = ["default"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
+serde_plain.workspace = true
 simplelog.workspace = true
 snafu.workspace = true
 thar-be-updates.workspace = true

sources/api/apiserver/src/server/controller.rs

@@ -3,17 +3,21 @@
 
 use bottlerocket_release::BottlerocketRelease;
 use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, HashSet};
 use std::io::Write;
 use std::process::{Command, Stdio};
 
 use crate::server::error::{self, Result};
 use actix_web::HttpResponse;
+use datastore::constraints_check::{ApprovedWrite, ConstraintCheckResult};
 use datastore::deserialization::{from_map, from_map_with_prefix};
 use datastore::serialization::to_pairs_with_prefix;
-use datastore::{deserialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value};
-use model::{ConfigurationFiles, Services, Settings};
+use datastore::{
+    deserialize_scalar, serialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value,
+};
+use model::{ConfigurationFiles, Services, Settings, Strength};
 use num::FromPrimitive;
 use std::os::unix::process::ExitStatusExt;
 use thar_be_updates::error::TbuErrorStatus;
@@ -44,6 +48,51 @@ where
         .map(|maybe_settings| maybe_settings.unwrap_or_default())
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(transparent)]
+pub(crate) struct SettingsMetadata {
+    pub(crate) inner: HashMap<String, HashMap<String, String>>,
+}
+
+impl From<HashMap<Key, HashMap<Key, String>>> for SettingsMetadata {
+    fn from(transaction_metadata: HashMap<Key, HashMap<Key, String>>) -> Self {
+        let mut metadata = HashMap::new();
+        for (key, value) in transaction_metadata {
+            let mut inner_map = HashMap::new();
+            for (inner_key, inner_value) in value {
+                inner_map.insert(inner_key.name().clone(), inner_value);
+            }
+            metadata.insert(key.name().clone(), inner_map);
+        }
+
+        SettingsMetadata { inner: metadata }
+    }
+}
+
+/// Gets the metadata for metadata_key_name in the given transaction
+/// Returns all metadata if metadata_key_name is None
+pub(crate) fn get_transaction_metadata<D, S>(
+    datastore: &D,
+    transaction: S,
+    metadata_key_name: Option<String>,
+) -> Result<SettingsMetadata>
+where
+    D: DataStore,
+    S: Into<String>,
+{
+    let pending = Committed::Pending {
+        tx: transaction.into(),
+    };
+
+    let metadata = datastore
+        .get_metadata_prefix("settings.", &pending, &metadata_key_name)
+        .with_context(|_| error::DataStoreSnafu {
+            op: format!("get_metadata_prefix '{}' for {:?}", "settings.", pending),
+        })?;
+
+    Ok(SettingsMetadata::from(metadata))
+}
+
 /// Deletes the transaction from the data store, removing any uncommitted settings under that
 /// transaction name.
 pub(crate) fn delete_transaction<D: DataStore>(
@@ -364,6 +413,7 @@ pub(crate) fn set_settings<D: DataStore>(
     datastore: &mut D,
     settings: &Settings,
     transaction: &str,
+    strength: Strength,
 ) -> Result<()> {
     trace!("Serializing Settings to write to data store");
     let settings_json = serde_json::to_value(settings).context(error::SettingsToJsonSnafu)?;
@@ -372,6 +422,32 @@ pub(crate) fn set_settings<D: DataStore>(
     let pending = Committed::Pending {
         tx: transaction.into(),
     };
+
+    info!("Writing Metadata to pending transaction in datastore.");
+
+    // Write the metadata to pending transaction as provided.
+    // We will validate this transaction while committing.
+    for key in pairs.keys() {
+        let metadata_key_strength =
+            Key::new(KeyType::Meta, "strength").context(error::NewKeySnafu {
+                key_type: "meta",
+                name: "strength",
+            })?;
+
+        let metadata_value = datastore::serialize_scalar::<_, ScalarError>(&strength.to_string())
+            .with_context(|_| error::SerializeSnafu {})?;
+
+        datastore
+            .set_metadata(&metadata_key_strength, key, metadata_value, &pending)
+            .context(error::DataStoreSnafu {
+                op: "Failure in setting metadata.",
+            })?;
+    }
+
+    info!(
+        "Writing Settings to pending transaction in datastore: {:?}",
+        pairs
+    );
     datastore
         .set_keys(&pairs, &pending)
         .context(error::DataStoreSnafu { op: "set_keys" })
@@ -398,7 +474,7 @@ pub(crate) fn get_metadata_for_data_keys<D: DataStore, S: AsRef<str>>(
             key_type: "data",
             name: *data_key_str,
         })?;
-        let value_str = match datastore.get_metadata(&md_key, &data_key) {
+        let value_str = match datastore.get_metadata(&md_key, &data_key, &Committed::Live) {
             Ok(Some(v)) => v,
             // TODO: confirm we want to skip requested keys if not populated, or error
             Ok(None) => continue,
@@ -428,7 +504,7 @@ pub(crate) fn get_metadata_for_all_data_keys<D: DataStore, S: AsRef<str>>(
 ) -> Result<HashMap<String, Value>> {
     trace!("Getting metadata '{}'", md_key_str.as_ref());
     let meta_map = datastore
-        .get_metadata_prefix("", &Some(md_key_str))
+        .get_metadata_prefix("", &Committed::Live, &Some(md_key_str))
         .context(error::DataStoreSnafu {
             op: "get_metadata_prefix",
         })?;
@@ -449,13 +525,129 @@ pub(crate) fn get_metadata_for_all_data_keys<D: DataStore, S: AsRef<str>>(
     Ok(result)
 }
 
+// Parses and validates the settings and metadata in pending transaction and
+// returns the constraint check result containing approved settings and metadata to
+// commit to live transaction.
+// We will pass this function as argument to commit transaction function.
+fn datastore_transaction_check<D>(
+    datastore: &mut D,
+    committed: &Committed,
+) -> Result<ConstraintCheckResult>
+where
+    D: DataStore,
+{
+    // Get settings to commit from pending transaction
+    let settings_to_commit = datastore
+        .get_prefix("settings.", committed)
+        .context(error::DataStoreSnafu { op: "get_prefix" })?;
+
+    // Get metadata from pending transaction
+    let mut transaction_metadata = datastore
+        .get_metadata_prefix("settings.", committed, &None as &Option<&str>)
+        .context(error::DataStoreSnafu {
+            op: "get_metadata_prefix",
+        })?;
+
+    let mut metadata_to_commit: Vec<(Key, Key, String)> = Vec::new();
+
+    // Parse and validate all the metadata enteries from pending transaction
+    for (key, value) in transaction_metadata.iter_mut() {
+        for (metadata_key, metadata_value) in value {
+            // For now we are only processing the strength metadata from pending
+            // transaction to live
+            if metadata_key.name() != "strength" {
+                warn!(
+                    "Metadata key is {}, Skipping the commit as we only allow Strength metadata.",
+                    metadata_key.name()
+                );
+                continue;
+            }
+
+            // strength in pending transaction
+            let pending_strength: Strength =
+                deserialize_scalar::<String, ScalarError>(&metadata_value.clone())
+                    .with_context(|_| error::DeserializeStrengthSnafu {})?
+                    .parse::<Strength>()
+                    .context(error::ParseStrengthSnafu)?;
+
+            // Get the setting strength in live
+            // get_metadata function returns Ok(None) in case strength does not exist
+            // We will consider this case as strength equals strong.
+            let committed_strength: Strength = datastore
+                .get_metadata(metadata_key, key, &Committed::Live)
+                .context(error::DataStoreSnafu { op: "get_metadata" })?
+                .map(|x| x.parse::<Strength>())
+                .transpose()
+                .context(error::ParseStrengthSnafu)?
+                .unwrap_or_default();
+
+            let may_be_value = datastore
+                .get_key(key, &Committed::Live)
+                .context(error::DataStoreSnafu { op: "get_key" })?;
+
+            trace!(
+                "datastore_transaction_check: key: {:?}, metadata_key: {:?}, metadata_value: {:?}",
+                key.name(),
+                metadata_key.name(),
+                metadata_value
+            );
+
+            match (pending_strength, committed_strength, may_be_value) {
+                (Strength::Weak, Strength::Strong, None)
+                | (Strength::Strong, Strength::Weak, Some(..))
+                | (Strength::Strong, Strength::Weak, None) => {
+                    let met_value = serialize_scalar::<_, ScalarError>(&pending_strength)
+                        .with_context(|_| error::SerializeSnafu {})?;
+
+                    metadata_to_commit.push((metadata_key.clone(), key.clone(), met_value));
+                }
+                (Strength::Weak, Strength::Strong, Some(..)) => {
+                    // Do not change from strong to weak if setting exists
+                    // as the default strength is strong for a setting.
+                    return Ok(ConstraintCheckResult::Reject(format!(
+                        "Cannot change setting {} strength from strong to weak",
+                        key.name()
+                    )));
+                }
+                (Strength::Weak, Strength::Weak, ..) => {
+                    trace!("The strength for setting {} is already weak", key.name());
+                    continue;
+                }
+                (Strength::Strong, Strength::Strong, ..) => {
+                    trace!("The strength for setting {} is already strong", key.name());
+                    continue;
+                }
+            };
+        }
+    }
+
+    let approved_write = ApprovedWrite {
+        settings: settings_to_commit,
+        metadata: metadata_to_commit,
+    };
+
+    Ok(ConstraintCheckResult::from(Some(approved_write)))
+}
+
+/// Wrapper for our transaction constraint check that converts errors into the expected type
+fn datastore_transaction_check_wrapper<D>(
+    datastore: &mut D,
+    committed: &Committed,
+) -> Result<ConstraintCheckResult, Box<dyn std::error::Error + Send + Sync + 'static>>
+where
+    D: DataStore,
+{
+    datastore_transaction_check::<D>(datastore, committed)
+        .map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
+}
+
 /// Makes live any pending settings in the datastore, returning the changed keys.
 pub(crate) fn commit_transaction<D>(datastore: &mut D, transaction: &str) -> Result<HashSet<Key>>
 where
     D: DataStore,
 {
     datastore
-        .commit_transaction(transaction)
+        .commit_transaction(transaction, &datastore_transaction_check_wrapper::<D>)
         .context(error::DataStoreSnafu { op: "commit" })
 }
 
@@ -786,7 +978,7 @@ mod test {
         let mut ds = MemoryDataStore::new();
         let tx = "test transaction";
         let pending = Committed::Pending { tx: tx.into() };
-        set_settings(&mut ds, &settings, tx).unwrap();
+        set_settings(&mut ds, &settings, tx, Strength::Strong).unwrap();
 
         // Retrieve directly
         let key = Key::new(KeyType::Data, "settings.motd").unwrap();
@@ -805,6 +997,7 @@ mod test {
                 &Key::new(KeyType::Meta, "my-meta").unwrap(),
                 &Key::new(KeyType::Data, data_key).unwrap(),
                 "\"json string\"",
+                &Committed::Live,
             )
             .unwrap();
         }
@@ -829,6 +1022,7 @@ mod test {
                 &Key::new(KeyType::Meta, "my-meta").unwrap(),
                 &Key::new(KeyType::Data, data_key).unwrap(),
                 "\"json string\"",
+                &Committed::Live,
             )
             .unwrap();
         }
@@ -863,7 +1057,7 @@ mod test {
         get_settings(&ds, &Committed::Live).unwrap_err();
 
         // Commit, pending -> live
-        commit_transaction(&mut ds, tx).unwrap();
+        commit_transaction::<datastore::memory::MemoryDataStore>(&mut ds, tx).unwrap();
 
         // No more pending settings
         get_settings(&ds, &pending).unwrap_err();

sources/api/apiserver/src/server/error.rs

@@ -92,6 +92,9 @@ pub enum Error {
         source: datastore::deserialization::Error,
     },
 
+    #[snafu(display("Unable to deserialize data: {}", source))]
+    DeserializeStrength { source: serde_json::Error },
+
     #[snafu(display("Unable to serialize data: {}", source))]
     Serialize { source: serde_json::Error },
 
@@ -240,9 +243,31 @@ pub enum Error {
         stdout: Vec<u8>,
         source: serde_json::Error,
     },
+
+    #[snafu(display("Error deserializing response value to SettingsGenerator: {}", source))]
+    DeserializeSettingsGenerator { source: serde_json::Error },
+
+    #[snafu(display(
+        "Provided strength is not one of weak or strong. The given strength is: {}. {}",
+        strength,
+        source
+    ))]
+    InvalidStrength {
+        strength: String,
+        source: serde_plain::Error,
+    },
+
+    #[snafu(display(
+        "Trying to change the strength from strong to weak for key: {}, Operation restricted",
+        key
+    ))]
+    DisallowStrongToWeakStrength { key: String },
+
+    #[snafu(display("Unable to parse the given strength. Error: "))]
+    ParseStrength { source: serde_plain::Error },
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
+pub type Result<T, E = Error> = std::result::Result<T, E>;
 
 impl From<Error> for actix_web::HttpResponse {
     fn from(e: Error) -> Self {