Skip to content

Commit

Permalink
Merge branch 'develop' into fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@mappum
mappum authored Nov 16, 2024
2 parents 770782d + 84261c2 commit 3538379
Show file tree
Hide file tree
Showing 9 changed files with 252 additions and 58 deletions.
2 changes: 2 additions & 0 deletions src/error.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ pub enum Error {
UnexpectedNode(String),
#[error("Unknown Error")]
Unknown,
#[error("Version Error: {0}")]
Version(String),
}

pub type Result<T> = std::result::Result<T, Error>;
49 changes: 25 additions & 24 deletions src/merk/chunks.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,70 +311,71 @@ mod tests {
assert_eq!(
chunk,
vec![
3, 8, 0, 0, 0, 0, 0, 0, 0, 18, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123,
3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 18, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 3, 8, 0, 0, 0, 0, 0, 0, 0, 19, 0, 60, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 19, 0, 60, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 16, 3, 8, 0, 0, 0, 0, 0, 0, 0, 20, 0, 60, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 20, 0,
60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 3, 8, 0, 0, 0, 0, 0, 0, 0,
21, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 3, 0, 8, 0, 0, 0,
0, 0, 0, 0, 21, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 8, 0,
0, 0, 0, 0, 0, 0, 22, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16,
3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 22, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 3, 8, 0, 0, 0, 0, 0, 0, 0, 23, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 23, 0, 60, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 16, 3, 8, 0, 0, 0, 0, 0, 0, 0, 24, 0, 60, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 24, 0,
60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 17, 3, 0, 8, 0, 0,
0, 0, 0, 0, 0, 25, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 17, 17, 3, 8, 0, 0, 0, 0, 0, 0, 0, 25, 0,
60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 8, 0, 0, 0, 0,
0, 0, 0, 26, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 26, 0, 60, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 3,
8, 0, 0, 0, 0, 0, 0, 0, 27, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 27, 0, 60, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 16, 3, 8, 0, 0, 0, 0, 0, 0, 0, 28, 0, 60, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0,
28, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 17, 3, 8, 0, 0, 0, 0, 0, 0, 0, 29, 0, 60, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 3, 0, 8,
0, 0, 0, 0, 0, 0, 0, 29, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 8, 0, 0, 0, 0, 0, 0,
0, 30, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 30, 0, 60, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 3, 8, 0, 0,
0, 0, 0, 0, 0, 31, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 31, 0, 60, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 16, 3, 8, 0, 0, 0, 0, 0, 0, 0, 32, 0, 60, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 16, 3, 0, 8, 0, 0, 0, 0, 0,
0, 0, 32, 0, 60, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123,
123, 123, 123, 123, 123, 17, 17, 17
123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 17,
17
]
);
}
Expand Down
114 changes: 113 additions & 1 deletion src/merk/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@ use crate::tree::{Batch, Commit, Fetch, GetResult, Hash, Op, RefWalker, Tree, Wa
pub use self::snapshot::Snapshot;

const ROOT_KEY_KEY: &[u8] = b"root";
const FORMAT_VERSION_KEY: &[u8] = b"format";
const AUX_CF_NAME: &str = "aux";
const INTERNAL_CF_NAME: &str = "internal";

const FORMAT_VERSION: u64 = 1;

fn column_families() -> Vec<ColumnFamilyDescriptor> {
vec![
// TODO: clone opts or take args
Expand Down Expand Up @@ -57,6 +60,14 @@ impl Merk {
false,
)?;

let format_version = load_format_version(&db)?;
if format_version != FORMAT_VERSION {
return Err(Error::Version(format!(
"Format version mismatch: expected {}, found {}",
FORMAT_VERSION, format_version,
)));
}

Ok(Merk {
tree: RwLock::new(load_root(&db)?),
db,
Expand All @@ -72,7 +83,25 @@ impl Merk {
{
let mut path_buf = PathBuf::new();
path_buf.push(path);
let db = rocksdb::DB::open_cf_descriptors(&db_opts, &path_buf, column_families())?;

let mut db = rocksdb::DB::open_cf_descriptors(&db_opts, &path_buf, column_families())?;
let format_version = load_format_version(&db)?;
let has_root = load_root(&db)?.is_some();

if has_root {
if format_version == 0 {
log::info!("Migrating store from version 0 to {}...", FORMAT_VERSION);

drop(db);
Merk::migrate_from_v0(&path_buf)?;
db = rocksdb::DB::open_cf_descriptors(&db_opts, &path_buf, column_families())?;
} else if format_version != FORMAT_VERSION {
return Err(Error::Version(format!(
"Unknown format version: expected <= {}, found {}",
FORMAT_VERSION, format_version,
)));
}
}

Ok(Merk {
tree: RwLock::new(load_root(&db)?),
Expand All @@ -81,6 +110,16 @@ impl Merk {
})
}

pub fn open_and_get_aux<P>(path: P, key: &[u8]) -> Result<Option<Vec<u8>>>
where
P: AsRef<Path>,
{
let db_opts = Merk::default_db_opts();
let db = rocksdb::DB::open_cf_descriptors(&db_opts, path, column_families())?;
let aux_cf = db.cf_handle(AUX_CF_NAME).unwrap();
Ok(db.get_cf(aux_cf, key)?)
}

pub fn default_db_opts() -> rocksdb::Options {
let mut opts = rocksdb::Options::default();
opts.create_if_missing(true);
Expand Down Expand Up @@ -265,6 +304,59 @@ impl Merk {
Self::open(path)
}

pub fn migrate_from_v0<P: AsRef<Path>>(path: P) -> Result<()> {
use rocksdb::IteratorMode;

let path = path.as_ref().to_path_buf();
let db =
rocksdb::DB::open_cf_descriptors(&Merk::default_db_opts(), &path, column_families())?;

let create_path = |suffix| {
let mut tmp_path = path.clone();
let tmp_file_name =
format!("{}-{}", path.file_name().unwrap().to_str().unwrap(), suffix);
tmp_path.set_file_name(tmp_file_name);
tmp_path
};

let tmp_path = create_path("migrate1");
let tmp = Merk::open(&tmp_path)?;
tmp.destroy()?;

// TODO: split up batch
let batch: Vec<_> = db
.iterator(IteratorMode::Start)
.map(|entry| -> Result<_> {
dbg!();
let (key, node_bytes) = entry.unwrap(); // TODO
dbg!(&key, node_bytes.len());
let node = Tree::decode_v0(&mut &node_bytes[..])?;
dbg!();
Ok((key.to_vec(), Op::Put(node.value().to_vec())))
})
.collect::<Result<_>>()?;

let aux_cf = db.cf_handle(AUX_CF_NAME).unwrap();
let aux: Vec<_> = db
.iterator_cf(aux_cf, IteratorMode::Start)
.map(|entry| {
let (key, value) = entry.unwrap(); // TODO
(key.to_vec(), Op::Put(value.to_vec()))
})
.collect();

let mut tmp = Self::open(&tmp_path)?;
tmp.apply(&batch, &aux)?;
drop(tmp);

let tmp_path2 = create_path("migrate2");
std::fs::rename(&path, &tmp_path2)?;
std::fs::rename(&tmp_path, &path)?;
std::fs::remove_dir_all(&tmp_path2)?;

Ok(())
}

/// Creates a Merkle proof for the list of queried keys. For each key in the
/// query, if the key is found in the store then the value will be proven to
/// be in the tree. For each key in the query that does not exist in the
Expand Down Expand Up @@ -328,6 +420,14 @@ impl Merk {
};
}

// update format version
// TODO: shouldn't need a write per commit
batch.put_cf(
internal_cf,
FORMAT_VERSION_KEY,
FORMAT_VERSION.to_be_bytes(),
);

// write to db
self.write(batch)?;

Expand Down Expand Up @@ -485,6 +585,18 @@ fn load_root(db: &DB) -> Result<Option<Tree>> {
.transpose()
}

fn load_format_version(db: &DB) -> Result<u64> {
let internal_cf = db.cf_handle(INTERNAL_CF_NAME).unwrap();
let maybe_version = db.get_pinned_cf(internal_cf, FORMAT_VERSION_KEY)?;
let Some(version) = maybe_version else {
return Ok(0);
};

let mut buf = [0; 8];
buf.copy_from_slice(&version);
Ok(u64::from_be_bytes(buf))
}

#[cfg(test)]
mod test {
use super::{Merk, MerkSource, RefWalker};
Expand Down
18 changes: 9 additions & 9 deletions src/proofs/encoding.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ impl Encode for Op {
dest.write_all(kv_hash)?;
}
Op::Push(Node::KV(key, value)) => {
debug_assert!(key.len() < 256);
debug_assert!(key.len() < 65536);
debug_assert!(value.len() < 65536);

dest.write_all(&[0x03, key.len() as u8])?;
dest.write_all(&[0x03])?;
(key.len() as u16).encode_into(dest)?;
dest.write_all(key)?;
(value.len() as u16).encode_into(dest)?;
dest.write_all(value)?;
Expand All @@ -36,7 +36,7 @@ impl Encode for Op {
Ok(match self {
Op::Push(Node::Hash(_)) => 1 + HASH_LENGTH,
Op::Push(Node::KVHash(_)) => 1 + HASH_LENGTH,
Op::Push(Node::KV(key, value)) => 4 + key.len() + value.len(),
Op::Push(Node::KV(key, value)) => 5 + key.len() + value.len(),
Op::Parent => 1,
Op::Child => 1,
})
Expand All @@ -59,7 +59,7 @@ impl Decode for Op {
Op::Push(Node::KVHash(hash))
}
0x03 => {
let key_len: u8 = Decode::decode(&mut input)?;
let key_len: u16 = Decode::decode(&mut input)?;
let mut key = vec![0; key_len as usize];
input.read_exact(key.as_mut_slice())?;

Expand Down Expand Up @@ -173,11 +173,11 @@ mod test {
#[test]
fn encode_push_kv() {
let op = Op::Push(Node::KV(vec![1, 2, 3], vec![4, 5, 6]));
assert_eq!(op.encoding_length(), 10);
assert_eq!(op.encoding_length(), 11);

let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
assert_eq!(bytes, vec![0x03, 3, 1, 2, 3, 0, 3, 4, 5, 6]);
assert_eq!(bytes, vec![0x03, 0, 3, 1, 2, 3, 0, 3, 4, 5, 6]);
}

#[test]
Expand All @@ -203,7 +203,7 @@ mod test {
#[test]
#[should_panic]
fn encode_push_kv_long_key() {
let op = Op::Push(Node::KV(vec![123; 300], vec![4, 5, 6]));
let op = Op::Push(Node::KV(vec![123; 70_000], vec![4, 5, 6]));
let mut bytes = vec![];
op.encode_into(&mut bytes).unwrap();
}
Expand All @@ -230,7 +230,7 @@ mod test {

#[test]
fn decode_push_kv() {
let bytes = [0x03, 3, 1, 2, 3, 0, 3, 4, 5, 6];
let bytes = [0x03, 0, 3, 1, 2, 3, 0, 3, 4, 5, 6];
let op = Op::decode(&bytes[..]).expect("decode failed");
assert_eq!(op, Op::Push(Node::KV(vec![1, 2, 3], vec![4, 5, 6])));
}
Expand Down
10 changes: 5 additions & 5 deletions src/proofs/query/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -933,11 +933,11 @@ mod test {
assert_eq!(
bytes,
vec![
3, 1, 1, 0, 1, 1, 3, 1, 2, 0, 1, 2, 16, 3, 1, 3, 0, 1, 3, 3, 1, 4, 0, 1, 4, 16, 17,
2, 169, 4, 73, 65, 62, 49, 160, 159, 37, 166, 195, 249, 63, 31, 23, 11, 169, 0, 24,
104, 179, 211, 218, 38, 108, 129, 117, 232, 65, 101, 194, 157, 16, 1, 148, 241,
151, 144, 247, 220, 92, 79, 70, 252, 168, 222, 27, 218, 53, 156, 0, 136, 161, 107,
83, 78, 150, 246, 51, 230, 164, 248, 17, 30, 147, 91, 17
3, 0, 1, 1, 0, 1, 1, 3, 0, 1, 2, 0, 1, 2, 16, 3, 0, 1, 3, 0, 1, 3, 3, 0, 1, 4, 0,
1, 4, 16, 17, 2, 169, 4, 73, 65, 62, 49, 160, 159, 37, 166, 195, 249, 63, 31, 23,
11, 169, 0, 24, 104, 179, 211, 218, 38, 108, 129, 117, 232, 65, 101, 194, 157, 16,
1, 148, 241, 151, 144, 247, 220, 92, 79, 70, 252, 168, 222, 27, 218, 53, 156, 0,
136, 161, 107, 83, 78, 150, 246, 51, 230, 164, 248, 17, 30, 147, 91, 17
]
);

Expand Down
Loading

0 comments on commit 3538379

Please sign in to comment.