[server] Serialize raft logs to binary #1395 (#1437)

* Update lib.rs * Update lib.rs * use bianry serialization for ClusterAction * Update cluster_test.rs
agnesoft · Jan 1, 2025 · 7226ff3 · 7226ff3
1 parent 55dca12
commit 7226ff3
Show file tree

Hide file tree

Showing 50 changed files with 288 additions and 378 deletions.
diff --git a/.github/workflows/agdb_server.yaml b/.github/workflows/agdb_server.yaml
@@ -36,7 +36,7 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@v1
       - uses: taiki-e/install-action@cargo-llvm-cov
       - run: rustup component add llvm-tools-preview
-      - run: cargo llvm-cov --package agdb_server --all-features --ignore-filename-regex "agdb(.|..)src|agdb_derive|agdb_api|api.rs" --fail-uncovered-functions 21 --show-missing-lines
+      - run: cargo llvm-cov --package agdb_server --all-features --ignore-filename-regex "agdb(.|..)src|agdb_derive|agdb_api|api.rs" --show-missing-lines
 
   agdb_server_test:
     runs-on: ubuntu-latest

diff --git a/agdb/src/db/db_id.rs b/agdb/src/db/db_id.rs
@@ -14,6 +14,7 @@ use crate::StorageData;
 #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Ord, PartialOrd)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub struct DbId(pub i64);
 
 impl StableHash for DbId {

diff --git a/agdb/src/db/db_key_order.rs b/agdb/src/db/db_key_order.rs
@@ -4,6 +4,7 @@ use crate::DbValue;
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum DbKeyOrder {
     /// Ascending order (from smallest)
     Asc(DbValue),

diff --git a/agdb/src/db/db_key_value.rs b/agdb/src/db/db_key_value.rs
@@ -14,6 +14,7 @@ use crate::StorageData;
 #[derive(Clone, Debug, Default, Eq, PartialEq, PartialOrd, Ord)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub struct DbKeyValue {
     /// Key of the property
     pub key: DbValue,

diff --git a/agdb/src/db/db_value.rs b/agdb/src/db/db_value.rs
@@ -23,6 +23,7 @@ use std::fmt::Result as DisplayResult;
 #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum DbValue {
     /// Byte array, sometimes referred to as blob
     Bytes(Vec<u8>),

diff --git a/agdb/src/query.rs b/agdb/src/query.rs
@@ -56,6 +56,7 @@ use crate::{
 #[cfg(any(feature = "serde", feature = "openapi"))]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 #[expect(clippy::large_enum_variant)]
 pub enum QueryType {
@@ -209,10 +210,12 @@ impl From<SelectValuesQuery> for QueryType {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::AgdbSerialize;
+    use crate::DbKeyOrder;
     use crate::QueryBuilder;
 
     #[test]
-    fn derived_from_debug_and_partial_eq() {
+    fn derives() {
         let queries: Vec<QueryType> = vec![
             QueryBuilder::insert().nodes().count(2).query().into(),
             QueryBuilder::insert()
@@ -246,8 +249,33 @@ mod tests {
             QueryBuilder::remove().index("key").query().into(),
             QueryBuilder::remove().values("key").ids(1).query().into(),
             QueryBuilder::remove().ids("node1").query().into(),
+            QueryBuilder::search()
+                .depth_first()
+                .to(1)
+                .order_by(DbKeyOrder::Asc("id".into()))
+                .offset(10)
+                .limit(10)
+                .where_()
+                .node()
+                .or()
+                .edge()
+                .and()
+                .not_beyond()
+                .keys("hidden")
+                .and()
+                .where_()
+                .distance(crate::CountComparison::Equal(2))
+                .or()
+                .key("key")
+                .value(crate::Comparison::NotEqual("value".into()))
+                .query()
+                .into(),
         ];
         let _ = format!("{:?}", queries);
         assert_eq!(queries, queries);
+
+        let serialized = AgdbSerialize::serialize(&queries);
+        let deserialized: Vec<QueryType> = AgdbSerialize::deserialize(&serialized).unwrap();
+        assert_eq!(queries, deserialized);
     }
 }
diff --git a/agdb/src/query/insert_aliases_query.rs b/agdb/src/query/insert_aliases_query.rs
@@ -15,6 +15,7 @@ use crate::StorageData;
 /// The result will contain number of aliases inserted/updated but no elements.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct InsertAliasesQuery {
     /// Ids to be aliased

diff --git a/agdb/src/query/insert_edges_query.rs b/agdb/src/query/insert_edges_query.rs
@@ -28,6 +28,7 @@ use crate::StorageData;
 /// with their ids, origin and destination, but no properties.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct InsertEdgesQuery {
     /// Origins

diff --git a/agdb/src/query/insert_index_query.rs b/agdb/src/query/insert_index_query.rs
@@ -9,6 +9,7 @@ use crate::StorageData;
 /// a given key.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct InsertIndexQuery(pub DbValue);
 

diff --git a/agdb/src/query/insert_nodes_query.rs b/agdb/src/query/insert_nodes_query.rs
@@ -27,6 +27,7 @@ use crate::StorageData;
 /// with their ids but no properties.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct InsertNodesQuery {
     /// Number of nodes to be inserted.

diff --git a/agdb/src/query/insert_values_query.rs b/agdb/src/query/insert_values_query.rs
@@ -24,6 +24,7 @@ use crate::StorageData;
 /// NOTE: The result is NOT number of affected elements but individual properties.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct InsertValuesQuery {
     /// Ids whose properties should be updated

diff --git a/agdb/src/query/query_condition.rs b/agdb/src/query/query_condition.rs
@@ -6,6 +6,7 @@ use crate::QueryId;
 #[derive(Clone, Copy, Debug, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum QueryConditionLogic {
     /// Logical AND (&&)
     And,
@@ -18,6 +19,7 @@ pub enum QueryConditionLogic {
 #[derive(Clone, Copy, Debug, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum QueryConditionModifier {
     /// No modifier
     None,
@@ -38,6 +40,7 @@ pub enum QueryConditionModifier {
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum QueryConditionData {
     /// Distance from the search origin. Takes count comparison
     /// (e.g. Equal, GreaterThan).
@@ -90,6 +93,7 @@ pub enum QueryConditionData {
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub struct QueryCondition {
     /// Logic operator (e.g. And, Or)
     pub logic: QueryConditionLogic,
@@ -108,6 +112,7 @@ pub struct QueryCondition {
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum CountComparison {
     /// property == this
     Equal(u64),
@@ -139,6 +144,7 @@ pub enum CountComparison {
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum Comparison {
     /// property == this
     Equal(DbValue),

diff --git a/agdb/src/query/query_id.rs b/agdb/src/query/query_id.rs
@@ -6,6 +6,7 @@ use crate::DbId;
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum QueryId {
     /// Numerical id as [`DbId`]
     Id(DbId),

diff --git a/agdb/src/query/query_ids.rs b/agdb/src/query/query_ids.rs
@@ -11,6 +11,7 @@ use crate::SearchQuery;
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub enum QueryIds {
     /// List of [`QueryId`]s
     Ids(Vec<QueryId>),

diff --git a/agdb/src/query/query_values.rs b/agdb/src/query/query_values.rs
@@ -5,6 +5,7 @@ use crate::DbUserValue;
 /// and multiple (`Multi`) values in database queries.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub enum QueryValues {
     /// Single list of properties (key-value pairs)

diff --git a/agdb/src/query/remove_aliases_query.rs b/agdb/src/query/remove_aliases_query.rs
@@ -12,6 +12,7 @@ use crate::StorageData;
 /// many aliases have been actually removed.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct RemoveAliasesQuery(pub Vec<String>);
 

diff --git a/agdb/src/query/remove_index_query.rs b/agdb/src/query/remove_index_query.rs
@@ -9,6 +9,7 @@ use crate::StorageData;
 /// a given key.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct RemoveIndexQuery(pub DbValue);
 

diff --git a/agdb/src/query/remove_query.rs b/agdb/src/query/remove_query.rs
@@ -16,6 +16,7 @@ use crate::StorageData;
 /// also removed along with their properties.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct RemoveQuery(pub QueryIds);
 

diff --git a/agdb/src/query/remove_values_query.rs b/agdb/src/query/remove_values_query.rs
@@ -15,6 +15,7 @@ use crate::StorageData;
 /// do not exist on any of the elements).
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct RemoveValuesQuery(pub SelectValuesQuery);
 

diff --git a/agdb/src/query/search_query.rs b/agdb/src/query/search_query.rs
@@ -16,6 +16,7 @@ use std::cmp::Ordering;
 /// Search algorithm to be used
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum SearchQueryAlgorithm {
     /// Examines each distance level from the search origin in full
@@ -42,6 +43,7 @@ pub enum SearchQueryAlgorithm {
 #[derive(Debug, Clone, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 pub struct SearchQuery {
     /// Search algorithm to be used. Will be bypassed for path
     /// searches that unconditionally use A*.

diff --git a/agdb/src/query/select_aliases_query.rs b/agdb/src/query/select_aliases_query.rs
@@ -17,6 +17,7 @@ use crate::StorageData;
 /// the value `String`.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectAliasesQuery(pub QueryIds);
 

diff --git a/agdb/src/query/select_all_aliases_query.rs b/agdb/src/query/select_all_aliases_query.rs
@@ -12,6 +12,7 @@ use crate::StorageData;
 /// the value `String`.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectAllAliasesQuery {}
 

diff --git a/agdb/src/query/select_edge_count_query.rs b/agdb/src/query/select_edge_count_query.rs
@@ -22,6 +22,7 @@ use crate::StorageData;
 /// might be greater than number of unique db elements.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectEdgeCountQuery {
     /// Ids of the nodes to select edge count for.

diff --git a/agdb/src/query/select_indexes_query.rs b/agdb/src/query/select_indexes_query.rs
@@ -14,6 +14,7 @@ use crate::StorageData;
 /// index.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectIndexesQuery {}
 

diff --git a/agdb/src/query/select_key_count_query.rs b/agdb/src/query/select_key_count_query.rs
@@ -16,6 +16,7 @@ use crate::StorageData;
 /// a value `u64`.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectKeyCountQuery(pub QueryIds);
 

diff --git a/agdb/src/query/select_keys_query.rs b/agdb/src/query/select_keys_query.rs
@@ -15,6 +15,7 @@ use crate::StorageData;
 /// of elements with all properties except all values will be empty.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectKeysQuery(pub QueryIds);
 

diff --git a/agdb/src/query/select_node_count.rs b/agdb/src/query/select_node_count.rs
@@ -13,6 +13,7 @@ use crate::StorageData;
 /// a value `u64` represneting number of nodes in teh database.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectNodeCountQuery {}
 

diff --git a/agdb/src/query/select_values_query.rs b/agdb/src/query/select_values_query.rs
@@ -17,6 +17,7 @@ use crate::StorageData;
 /// list of elements with the requested properties.
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
+#[cfg_attr(feature = "derive", derive(agdb::AgdbDeSerialize))]
 #[derive(Clone, Debug, PartialEq)]
 pub struct SelectValuesQuery {
     pub keys: Vec<DbValue>,

diff --git a/agdb/src/utilities/serialize.rs b/agdb/src/utilities/serialize.rs
@@ -116,6 +116,23 @@ impl Serialize for String {
     }
 }
 
+impl Serialize for bool {
+    fn serialize(&self) -> Vec<u8> {
+        vec![*self as u8]
+    }
+
+    fn deserialize(bytes: &[u8]) -> Result<Self, DbError> {
+        bytes
+            .first()
+            .ok_or(DbError::from("bool deserialization error: out of bounds"))
+            .map(|&b| b != 0)
+    }
+
+    fn serialized_size(&self) -> u64 {
+        1
+    }
+}
+
 impl<T: Serialize> Serialize for Vec<T> {
     fn serialize(&self) -> Vec<u8> {
         let mut bytes = Vec::with_capacity(self.serialized_size() as usize);