feat: Add a new disk-based WAL implementation for standalone deployment

integration_tests/Makefile

@@ -54,7 +54,7 @@ build-meta:
 	./build_meta.sh
 
 build-horaedb:
-	cd .. && cargo build --bin horaedb-server --features wal-table-kv,wal-message-queue,wal-rocksdb
+	cd .. && cargo build --bin horaedb-server --features wal-table-kv,wal-message-queue,wal-rocksdb,wal-local-storage
 
 build-test:
 	cargo build

src/analytic_engine/Cargo.toml

@@ -35,6 +35,7 @@ test = ["tempfile"]
 wal-table-kv = ["wal/wal-table-kv"]
 wal-message-queue = ["wal/wal-message-queue"]
 wal-rocksdb = ["wal/wal-rocksdb"]
+wal-local-storage = ["wal/wal-local-storage"]
 
 [dependencies]
 # In alphabetical order

src/horaedb/Cargo.toml

@@ -31,10 +31,11 @@ workspace = true
 workspace = true
 
 [features]
-default = ["wal-rocksdb", "wal-table-kv", "wal-message-queue"]
+default = ["wal-rocksdb", "wal-table-kv", "wal-message-queue", "wal-local-storage"]
 wal-table-kv = ["wal/wal-table-kv", "analytic_engine/wal-table-kv"]
 wal-message-queue = ["wal/wal-message-queue", "analytic_engine/wal-message-queue"]
 wal-rocksdb = ["wal/wal-rocksdb", "analytic_engine/wal-rocksdb"]
+wal-local-storage = ["wal/wal-local-storage", "analytic_engine/wal-local-storage"]
 
 [dependencies]
 analytic_engine = { workspace = true }

src/horaedb/src/setup.rs

@@ -188,6 +188,22 @@ pub fn run_server(config: Config, log_runtime: RuntimeLevel) {
                     panic!("Message Queue WAL not bundled!");
                 }
             }
+            StorageConfig::Local(_) => {
+                #[cfg(feature = "wal-local-storage")]
+                {
+                    use wal::local_storage_impl::wal_manager::LocalStorageWalsOpener;
+                    run_server_with_runtimes::<LocalStorageWalsOpener>(
+                        config,
+                        engine_runtimes,
+                        log_runtime,
+                    )
+                    .await;
+                }
+                #[cfg(not(feature = "wal-local-storage"))]
+                {
+                    panic!("Local Storage WAL not bundled!");
+                }
+            }
         }
     });
 }

src/wal/Cargo.toml

@@ -40,23 +40,26 @@ optional = true
 wal-message-queue = ["dep:message_queue"]
 wal-table-kv = ["dep:table_kv"]
 wal-rocksdb = ["dep:rocksdb"]
+wal-local-storage = ["memmap2", "crc32fast"]
 
 [[test]]
 name = "read_write"
-required-features = ["wal-message-queue", "wal-table-kv", "wal-rocksdb"]
+required-features = ["wal-message-queue", "wal-table-kv", "wal-rocksdb", "wal-local-storage"]
 
 [dependencies]
 async-trait = { workspace = true }
 bytes_ext = { workspace = true }
 chrono = { workspace = true }
 codec = { workspace = true }
 common_types = { workspace = true }
+crc32fast = { version = "1.4.2", optional = true }
 futures = { workspace = true, features = ["async-await"], optional = true }
 generic_error = { workspace = true }
 horaedbproto = { workspace = true }
 lazy_static = { workspace = true }
 logger = { workspace = true }
 macros = { workspace = true }
+memmap2 = { version = "0.9.4", optional = true }
 message_queue = { workspace = true, optional = true }
 prometheus = { workspace = true }
 prost = { workspace = true }

src/wal/src/config.rs

@@ -35,6 +35,12 @@ pub type KafkaStorageConfig = crate::message_queue_impl::config::KafkaStorageCon
 #[derive(Debug, Default, Clone, Deserialize, Serialize)]
 pub struct KafkaStorageConfig;
 
+#[cfg(feature = "wal-local-storage")]
+pub type LocalStorageConfig = crate::local_storage_impl::config::LocalStorageConfig;
+#[cfg(not(feature = "wal-local-storage"))]
+#[derive(Debug, Default, Clone, Deserialize, Serialize)]
+pub struct LocalStorageConfig;
+
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct Config {
     // The flatten attribute inlines keys from a field into the parent struct.
@@ -63,4 +69,5 @@ pub enum StorageConfig {
     RocksDB(Box<RocksDBStorageConfig>),
     Obkv(Box<ObkvStorageConfig>),
     Kafka(Box<KafkaStorageConfig>),
+    Local(Box<LocalStorageConfig>),
 }

src/wal/src/lib.rs

@@ -22,6 +22,8 @@
 pub mod config;
 mod dummy;
 pub mod kv_encoder;
+#[cfg(feature = "wal-local-storage")]
+pub mod local_storage_impl;
 pub mod log_batch;
 pub mod manager;
 #[cfg(feature = "wal-message-queue")]

src/wal/src/local_storage_impl/config.rs

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LocalStorageConfig {
+    pub path: String,
+    pub max_segment_size: u64,
+    pub cache_size: usize,
+}
+
+impl Default for LocalStorageConfig {
+    fn default() -> Self {
+        Self {
+            path: "/tmp/horaedb".to_string(),
+            max_segment_size: 64 * 1024 * 1024, // 64MB
+            cache_size: 3,
+        }
+    }
+}

src/wal/src/local_storage_impl/mod.rs

@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod config;
+mod record_encoding;
+mod segment;
+pub mod wal_manager;