Merge pull request #19 from n0-computer/feat-iroh-store

Cargo.toml

@@ -6,5 +6,6 @@ members = [
   "iroh-gateway",
   "iroh-metrics",
   "iroh-p2p",
+  "iroh-store",
   "stores/*"
 ]

iroh-store/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "iroh-store"
+version = "0.1.0"
+edition = "2021"
+authors = ["Friedel Ziegelmayer <[email protected]>"]
+license = "Apache-2.0/MIT"
+repository = "https://github.com/n0-computer/iroh"
+description = "Implementation of the storage part of iroh"
+
+[dependencies]
+rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb", branch = "master" }
+eyre = "0.6.6"
+tokio = { version = "1.18.0", features = ["rt"] }
+cid = "0.8.4"
+rkyv = { version = "0.7.37", features = ["validation"] }
+bytecheck = "0.6.7"
+
+[dev-dependencies]
+criterion = { version = "0.3.5", features = ["async_tokio"] }
+tempfile = "3.3.0"
+tokio = { version = "1.18.0", features = ["rt", "macros", "rt-multi-thread"] }
+
+[features]
+default = []
+
+
+[[bench]]
+name = "store"
+harness = false

iroh-store/README.md

@@ -0,0 +1,27 @@
+# iroh-store
+
+> Storage for iroh.
+
+
+## How to run
+
+```sh
+# From the root of the workspace
+> cargo run --release -p iroh-store
+```
+
+## License
+
+<sup>
+Licensed under either of <a href="LICENSE-APACHE">Apache License, Version
+2.0</a> or <a href="LICENSE-MIT">MIT license</a> at your option.
+</sup>
+
+<br/>
+
+<sub>
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in this crate by you, as defined in the Apache-2.0 license, shall
+be dual licensed as above, without any additional terms or conditions.
+</sub>
+

iroh-store/benches/store.rs

@@ -0,0 +1,84 @@
+use std::time::Instant;
+
+use cid::multihash::{Code, MultihashDigest};
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use iroh_store::{Config, Store};
+use tokio::runtime::Runtime;
+
+const RAW: u64 = 0x55;
+
+pub fn put_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("store_put");
+    for value_size in [32, 128, 512, 1024].iter() {
+        let value = vec![8u8; *value_size];
+        let hash = Code::Sha2_256.digest(&value);
+        let key = cid::Cid::new_v1(RAW, hash);
+
+        group.throughput(criterion::Throughput::Bytes(*value_size as u64));
+        group.bench_with_input(
+            BenchmarkId::new("value_size", *value_size as u64),
+            &(key, value),
+            |b, (key, value)| {
+                let executor = Runtime::new().unwrap();
+                let dir = tempfile::tempdir().unwrap();
+                let config = Config {
+                    path: dir.path().into(),
+                };
+                let store = executor.block_on(async { Store::create(config).await.unwrap() });
+                let store_ref = &store;
+                b.to_async(&executor).iter(|| async move {
+                    store_ref.put(*key, black_box(value), []).await.unwrap()
+                });
+            },
+        );
+    }
+    group.finish();
+}
+
+pub fn get_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("store_get");
+    for value_size in [32, 128, 512, 1024].iter() {
+        group.throughput(criterion::Throughput::Bytes(*value_size as u64));
+        group.bench_with_input(
+            BenchmarkId::new("value_size", *value_size as u64),
+            &(),
+            |b, _| {
+                let executor = Runtime::new().unwrap();
+                let dir = tempfile::tempdir().unwrap();
+                let config = Config {
+                    path: dir.path().into(),
+                };
+                let store = executor.block_on(async { Store::create(config).await.unwrap() });
+                let store_ref = &store;
+                let keys = executor.block_on(async {
+                    let mut keys = Vec::new();
+                    for i in 0..1000 {
+                        let value = vec![i as u8; *value_size];
+                        let hash = Code::Sha2_256.digest(&value);
+                        let key = cid::Cid::new_v1(RAW, hash);
+                        keys.push(key);
+                        store_ref.put(key, &value, []).await.unwrap();
+                    }
+                    keys
+                });
+
+                let keys_ref = &keys[..];
+                b.to_async(&executor).iter_custom(|iters| async move {
+                    let l = keys_ref.len();
+
+                    let start = Instant::now();
+                    for i in 0..iters {
+                        let key = &keys_ref[(i as usize) % l];
+                        let res = store_ref.get(key).await.unwrap().unwrap();
+                        black_box(res);
+                    }
+                    start.elapsed()
+                });
+            },
+        );
+    }
+    group.finish();
+}
+
+criterion_group!(benches, put_benchmark, get_benchmark);
+criterion_main!(benches);

iroh-store/src/cf.rs

@@ -0,0 +1,37 @@
+use bytecheck::CheckBytes;
+use rkyv::{with::AsBox, Archive, Deserialize, Serialize};
+
+/// Column family to store actual data.
+/// - Maps id (u64) to bytes
+pub const CF_BLOBS_V0: &str = "blobs-v0";
+/// Column family that stores metdata about a given blob.
+/// - indexed by id (u64)
+pub const CF_METADATA_V0: &str = "metadata-v0";
+/// Column familty that stores the graph for a blob
+/// - indexed by id (u64)
+pub const CF_GRAPH_V0: &str = "graph-v0";
+/// Column family that stores the mapping multihash to id
+pub const CF_ID_V0: &str = "id-v0";
+
+// This wrapper type serializes the contained value out-of-line so that newer
+// versions can be viewed as the older version.
+#[derive(Debug, Archive, Deserialize, Serialize)]
+#[repr(transparent)]
+#[archive_attr(repr(transparent), derive(CheckBytes))]
+pub struct Versioned<T>(#[with(AsBox)] pub T);
+
+#[derive(Debug, Archive, Deserialize, Serialize)]
+#[repr(C)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct MetadataV0 {
+    pub codec: u64,
+    pub multihash: Vec<u8>,
+}
+
+#[derive(Debug, Archive, Deserialize, Serialize)]
+#[repr(C)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct GraphV0 {
+    /// The codec of the original CID.
+    pub children: Vec<u64>,
+}

iroh-store/src/config.rs

@@ -0,0 +1,8 @@
+use std::path::PathBuf;
+
+/// The configuration for the store.
+#[derive(Debug, Clone)]
+pub struct Config {
+    /// The location of the content database.
+    pub path: PathBuf,
+}

iroh-store/src/lib.rs

@@ -0,0 +1,6 @@
+mod cf;
+mod config;
+mod store;
+
+pub use crate::config::Config;
+pub use crate::store::Store;

iroh-store/src/main.rs

@@ -0,0 +1 @@
+fn main() {}