Support PEM secret type (#53)

* Work in progress to add PEM support.

* Use pem-rs fork with serde support.

* Use git dependency for pem-rs fork.

* Use updated pem crate fork.

* Update readme.

* Update README design docs.

* Improving design docs in README.

* Update readme.

README.md

@@ -15,6 +15,44 @@ This repository contains the core library code and several command line interfac
 
 For webassembly bindings see the [browser][] repository.
 
+## Design
+
+A vault is a collection of encrypted secrets. Vaults can be represented as both an append-only log file for synchronization and a compact binary file for archiving and portability. Bi-directional conversion between the append-only log and compact binary file is straightforward; the library provides methods to *reduce* the append-only log to a vault and *split* a vault into it's header and a collection of events that can be appended to a log.
+
+Synchronization between nodes is done using the append-only log file (which we refer to as a write-ahead log or WAL); a Merkle tree is computed for each log file using the hash of the data for each record. By comparing Merkle proofs we can easily determine which tree is ahead or whether the trees have diverged; much in the same way that [git][] synchronizes source code.
+
+Secrets are *always encrypted on the client* using a random nonce and one of the supported algorithms, either XChaCha20Poly1305 or AES-GCM 256. The default algorithm is XChaCha20Poly1305 for it's extended 24 byte nonce and because it does not require AES-specific CPU instructions to be implemented safely.
+
+### Changes
+
+When a node wants to make changes to another node it sends a commit proof of it's current HEAD node and a patch file containing the events to apply to the remote node.
+
+If the remote node has the same HEAD commit then the patch can be applied safely and a success response is returned to the node that made the request; when the calling node gets a success response it applies the patch to it's local copy of the append-only log.
+
+If the remote node *contains* the HEAD commit then it will send a CONFLICT response and a proof that it contains the calling node's HEAD. The calling node can then synchronize by pulling changes from the remote node and try to apply the patch again.
+
+If a calling node gets a CONFLICT response and no match proof then it a *hard conflict* will need to be resolved, see [Conflicts](#conflicts).
+
+### Networking
+
+For the networking layer we plan to support three different modes of operation:
+
+* [x] `SPOT`: Single Point of Truth using a standard client/server architecture.
+* [ ] `PEER`: Synchronization of nodes on a trusted LAN using mDNS for discovery.
+* [ ] `VPN`: Synchronization of nodes over a WAN using the [wireguard][] VPN.
+
+### Conflicts
+
+Conflicts are categorised into *soft conflicts* which can be automatically resolved via synchronization and *hard conflicts* which may require user approval to be resolved.
+
+#### Soft Conflict
+
+Soft conflicts occur when a node tries to make changes to another node but cannot as their commit trees are out of sync but have not diverged. These sorts of conflicts can be resolved by pushing local changes or pulling remote changes to synchronize; if the synchronization is successful the calling node can try again.
+
+#### Hard Conflict
+
+The system is eventually consistent except in the case of two events; when a WAL is compacted to prune history or when the encryption password for a vault is changed. Either of these events will completely rewrite the append-only log and therefore the vault commit trees will have diverged. If all nodes are connected when these events occur then it is possible to synchronize automatically but if a node is offline (or an error occurs) then we have a conflict that must be resolved; we call this a *hard conflict*.
+
 ## Setup
 
 Tasks are run using `cargo make`, install it with:
@@ -89,6 +127,8 @@ To create a release build with the bundled GUI assets run:
 cargo make server-release
 ```
 
+[git]: https://git-scm.com/
+[wireguard]: https://www.wireguard.com/
 [lcov]: https://github.com/linux-test-project/lcov
 [grcov]: https://github.com/mozilla/grcov
 [mkcert]: https://github.com/FiloSottile/mkcert

workspace/core/Cargo.toml

@@ -28,6 +28,13 @@ bitflags = "1"
 ouroboros = "0.15"
 tokio = { version = "1", default-features = false, features = ["fs", "io-util"] }
 
+[dependencies.pem]
+version = "1.0.3-alpha.0"
+git = "https://github.com/tmpfs/pem-rs"
+#branch = "serde-feature"
+rev = "6381ef4f29d74182461c17b82ea9a8e6370cf050"
+features = ["serde"]
+
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 file-guard = "0.1"
 

workspace/core/src/commit_tree/mod.rs

@@ -358,6 +358,7 @@ impl CommitTree {
     }
 }
 
+/*
 /// Refrerence to the identifier and commit for a row.
 #[derive(Debug)]
 pub struct RowInfo {
@@ -394,6 +395,7 @@ impl RowInfo {
         &self.id
     }
 }
+*/
 
 #[cfg(test)]
 mod test {

workspace/core/src/error.rs

@@ -190,4 +190,8 @@ pub enum Error {
     /// Error generated formatting time.
     #[error(transparent)]
     TimeFormat(#[from] time::error::Format),
+
+    /// Error generated parsing PEM files.
+    #[error(transparent)]
+    Pem(#[from] pem::PemError),
 }

workspace/core/src/secret.rs

@@ -4,7 +4,8 @@ use serde_binary::{
     Result as BinaryResult, Serializer,
 };
 
-use serde::{Deserialize, Serialize};
+use pem::Pem;
+use serde::{Deserialize, Serialize, ser::SerializeStruct, de::{self, Visitor, EnumAccess, VariantAccess}};
 use std::{collections::HashMap, fmt, str::FromStr};
 use url::Url;
 use uuid::Uuid;
@@ -146,8 +147,8 @@ impl Decode for SecretMeta {
 }
 
 /// Encapsulates a secret.
-#[derive(Debug, Eq, PartialEq, Serialize, Deserialize, Clone)]
-#[serde(untagged)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum Secret {
     /// A UTF-8 encoded note.
     Note(String),
@@ -173,6 +174,8 @@ pub enum Secret {
     },
     /// Collection of credentials as key/value pairs.
     List(HashMap<String, String>),
+    /// PEM encoded binary data.
+    Pem(Vec<Pem>),
 }
 
 impl Secret {
@@ -194,10 +197,54 @@ impl Secret {
             Secret::File { .. } => kind::FILE,
             Secret::Account { .. } => kind::ACCOUNT,
             Secret::List(_) => kind::LIST,
+            Secret::Pem(_) => kind::PEM,
         }
     }
 }
 
+impl PartialEq for Secret {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::Note(a), Self::Note(b)) => a == b,
+            (
+                Self::Account {
+                    account: account_a,
+                    url: url_a,
+                    password: password_a,
+                },
+                Self::Account {
+                    account: account_b,
+                    url: url_b,
+                    password: password_b,
+                },
+            ) => {
+                account_a == account_b
+                    && url_a == url_b
+                    && password_a == password_b
+            }
+            (
+                Self::File {
+                    name: name_a,
+                    mime: mime_a,
+                    buffer: buffer_a,
+                },
+                Self::File {
+                    name: name_b,
+                    mime: mime_b,
+                    buffer: buffer_b,
+                },
+            ) => name_a == name_b && mime_a == mime_b && buffer_a == buffer_b,
+            (Self::List(a), Self::List(b)) => a == b,
+            (Self::Pem(a), Self::Pem(b)) => a
+                .iter()
+                .zip(b.iter())
+                .all(|(a, b)| a.tag == b.tag && a.contents == b.contents),
+            _ => false,
+        }
+    }
+}
+impl Eq for Secret {}
+
 impl Default for Secret {
     fn default() -> Self {
         Self::Note(String::new())
@@ -218,6 +265,8 @@ pub mod kind {
     pub const LIST: u8 = 0x03;
     /// Binary blob, may be file content.
     pub const FILE: u8 = 0x04;
+    /// List of PEM encoded binary blobs.
+    pub const PEM: u8 = 0x05;
 }
 
 impl Encode for Secret {
@@ -227,6 +276,7 @@ impl Encode for Secret {
             Self::File { .. } => kind::FILE,
             Self::Account { .. } => kind::ACCOUNT,
             Self::List { .. } => kind::LIST,
+            Self::Pem(_) => kind::PEM,
         };
         ser.writer.write_u8(kind)?;
 
@@ -259,6 +309,10 @@ impl Encode for Secret {
                     ser.writer.write_string(v)?;
                 }
             }
+            Self::Pem(pems) => {
+                let value = pem::encode_many(pems);
+                ser.writer.write_string(value)?;
+            }
         }
 
         Ok(())
@@ -312,6 +366,10 @@ impl Decode for Secret {
 
                 *self = Self::List(list);
             }
+            kind::PEM => {
+                let value = de.reader.read_string()?;
+                *self = Self::Pem(pem::parse_many(&value).map_err(Box::from)?);
+            }
             _ => {
                 return Err(BinaryError::Boxed(Box::from(
                     Error::UnknownSecretKind(kind),
@@ -321,3 +379,113 @@ impl Decode for Secret {
         Ok(())
     }
 }
+
+/*
+impl serde::Serialize for Secret {
+    fn serialize<S>(
+        &self,
+        serializer: S,
+    ) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        match self {
+            Secret::Note(value) => serializer
+                .serialize_newtype_variant("Secret", 0, "note", value),
+            Secret::File { name, mime, buffer } => {
+                let mut s = serializer.serialize_struct("file", 3)?;
+                s.serialize_field("name", name)?;
+                s.serialize_field("mime", mime)?;
+                s.serialize_field("buffer", buffer)?;
+                s.end()
+            }
+            Secret::Account { account, url, password } => {
+                let mut s = serializer.serialize_struct("account", 3)?;
+                s.serialize_field("account", account)?;
+                s.serialize_field("url", url)?;
+                s.serialize_field("password", password)?;
+                s.end()
+            }
+            Secret::List(value) => serializer
+                .serialize_newtype_variant("Secret", 3, "list", value),
+            Secret::Pem(pems) => {
+                let value = pem::encode_many(pems);
+                serializer
+                    .serialize_newtype_variant("Secret", 4, "pem", &value)
+            },
+        }
+    }
+}
+
+struct SecretVisitor;
+
+impl<'de> Visitor<'de> for SecretVisitor {
+    type Value = Secret;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        Ok(())
+    }
+
+    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error> where
+        A: EnumAccess<'de> {
+
+        let (key, access) = data.variant::<String>()?;
+
+        println!("key {}", &key[..]);
+        //println!("{:#?}", value);
+
+        match &key[..] {
+            "note" => {
+                //self.visit_string()
+                //todo!()
+                //access.newtype_variant()
+            }
+            "file" => {
+                todo!()
+            }
+            "account" => {
+                todo!()
+            }
+            "list" => {
+                //access.newtype_variant()
+                todo!()
+            }
+            "pem" => {
+                //access.newtype_variant()
+                todo!()
+            }
+            _ => Err(de::Error::custom("unknown secret type tag"))
+        }
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for Secret {
+    fn deserialize<D>(
+        deserializer: D,
+    ) -> std::result::Result<Secret, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        deserializer.deserialize_enum(
+            "Secret",
+            &["note", "file", "account", "list", "pem"], SecretVisitor)
+    }
+}
+*/
+
+
+#[cfg(test)]
+mod test {
+    use anyhow::Result;
+    use super::*;
+
+    #[test]
+    fn secret_serde() -> Result<()> {
+        let secret = Secret::Note(String::from("foo"));
+        let value = serde_json::to_string_pretty(&secret)?;
+        let result: Secret = serde_json::from_str(&value)?;
+        assert_eq!(secret, result);
+        Ok(())
+    }
+}
+