refactor (#266)

experiments/object-access/src/main.rs

@@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc, time::Instant};
 use anyhow::anyhow;
 use git_repository::{hash::ObjectId, odb, threading::OwnShared, Repository};
 
-use crate::odb::{dynamic, Cache, RefreshMode};
+use crate::odb::{Cache, RefreshMode};
 
 const GITOXIDE_STATIC_CACHE_SIZE: usize = 64;
 const GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES: usize = 60_000_000;
@@ -449,21 +449,21 @@ fn do_new_gitoxide_store_in_parallel<C>(
     objects_dir: &Path,
     new_cache: impl Fn() -> C + Send + Sync + 'static,
     mode: AccessMode,
-    store: Option<OwnShared<dynamic::Store>>,
-) -> anyhow::Result<(std::sync::Arc<git_repository::odb::dynamic::Store>, u64)>
+    store: Option<OwnShared<odb::Store>>,
+) -> anyhow::Result<(std::sync::Arc<odb::Store>, u64)>
 where
     C: odb::pack::cache::DecodeEntry + Send + 'static,
 {
     use git_repository::prelude::FindExt;
     let bytes = std::sync::atomic::AtomicU64::default();
     let slots = std::env::args()
         .nth(2)
-        .and_then(|num| num.parse().ok().map(git_repository::odb::dynamic::init::Slots::Given))
+        .and_then(|num| num.parse().ok().map(odb::store::init::Slots::Given))
         .unwrap_or_default();
 
     let store = match store {
         Some(store) => store,
-        None => OwnShared::new(dynamic::Store::at_opts(objects_dir, slots)?),
+        None => OwnShared::new(odb::Store::at_opts(objects_dir, slots)?),
     };
     let handle =
         Cache::from(store.to_handle(RefreshMode::AfterAllIndicesLoaded)).with_pack_cache(move || Box::new(new_cache()));
@@ -504,7 +504,7 @@ where
     use git_repository::prelude::FindExt;
     let bytes = std::sync::atomic::AtomicU64::default();
     #[allow(deprecated)]
-    let odb = Arc::new(git_repository::odb::linked::Store::at(objects_dir)?);
+    let odb = Arc::new(odb::linked::Store::at(objects_dir)?);
 
     git_repository::parallel::in_parallel(
         hashes.chunks(1000),

git-odb/src/lib.rs

@@ -12,16 +12,18 @@
 // TODO: actually remove the deprecated items and remove thos allow
 #![allow(deprecated)]
 
+use arc_swap::ArcSwap;
 use std::cell::RefCell;
 use std::path::PathBuf;
+use std::sync::atomic::AtomicUsize;
 use std::sync::Arc;
 
 use git_features::threading::OwnShared;
 use git_features::zlib::stream::deflate;
 pub use git_pack as pack;
 
-mod store;
-pub use store::{compound, dynamic, linked, loose, RefreshMode};
+mod store_impls;
+pub use store_impls::{compound, dynamic as store, linked, loose, RefreshMode};
 
 pub mod alternate;
 
@@ -43,7 +45,7 @@ pub struct Cache<S> {
 pub mod cache;
 
 ///
-/// It can optionally compress the content, similarly to what would happen when using a [`loose::Store`][crate::store::loose::Store].
+/// It can optionally compress the content, similarly to what would happen when using a [`loose::Store`][crate::loose::Store].
 ///
 pub struct Sink {
     compressor: Option<RefCell<deflate::Write<std::io::Sink>>>,
@@ -66,17 +68,48 @@ mod traits;
 pub use traits::{Find, FindExt, Write};
 
 /// A thread-local handle to access any object.
-pub type Handle = Cache<dynamic::Handle<OwnShared<dynamic::Store>>>;
+pub type Handle = Cache<store::Handle<OwnShared<Store>>>;
 /// A thread-local handle to access any object, but thread-safe and independent of the actual type of `OwnShared` or feature toggles in `git-features`.
-pub type HandleArc = Cache<dynamic::Handle<Arc<dynamic::Store>>>;
+pub type HandleArc = Cache<store::Handle<Arc<Store>>>;
 
-/// A thread-safe store for creation of handles.
-pub type Store = dynamic::Store;
+use store::types;
+
+#[allow(missing_docs)]
+pub struct Store {
+    /// The central write lock without which the slotmap index can't be changed.
+    write: parking_lot::Mutex<()>,
+
+    /// The source directory from which all content is loaded, and the central write lock for use when a directory refresh is needed.
+    pub(crate) path: PathBuf,
+
+    /// A list of indices keeping track of which slots are filled with data. These are usually, but not always, consecutive.
+    pub(crate) index: ArcSwap<types::SlotMapIndex>,
+
+    /// The below state acts like a slot-map with each slot is mutable when the write lock is held, but readable independently of it.
+    /// This allows multiple file to be loaded concurrently if there is multiple handles requesting to load packs or additional indices.
+    /// The map is static and cannot typically change.
+    /// It's read often and changed rarely.
+    pub(crate) files: Vec<types::MutableIndexAndPack>,
+
+    /// The amount of handles that would prevent us from unloading packs or indices
+    pub(crate) num_handles_stable: AtomicUsize,
+    /// The amount of handles that don't affect our ability to compact our internal data structures or unload packs or indices.
+    pub(crate) num_handles_unstable: AtomicUsize,
+
+    /// The amount of times we re-read the disk state to consolidate our in-memory representation.
+    pub(crate) num_disk_state_consolidation: AtomicUsize,
+}
+
+impl Store {
+    /// The root path at which we expect to find all objects and packs.
+    pub fn path(&self) -> &std::path::Path {
+        &self.path
+    }
+}
 
 /// Create a new cached odb handle with support for additional options.
-pub fn at_opts(objects_dir: impl Into<PathBuf>, slots: dynamic::init::Slots) -> std::io::Result<Handle> {
-    let handle =
-        OwnShared::new(dynamic::Store::at_opts(objects_dir, slots)?).to_handle(RefreshMode::AfterAllIndicesLoaded);
+pub fn at_opts(objects_dir: impl Into<PathBuf>, slots: store::init::Slots) -> std::io::Result<Handle> {
+    let handle = OwnShared::new(Store::at_opts(objects_dir, slots)?).to_handle(RefreshMode::AfterAllIndicesLoaded);
     Ok(Cache::from(handle))
 }
 

git-odb/src/store/compound/find.rs → git-odb/src/store_impls/compound/find.rs

@@ -1,6 +1,6 @@
 use crate::{
     pack,
-    store::{compound, loose},
+    store_impls::{compound, loose},
 };
 
 /// Returned by [`compound::Store::try_find()`]

git-odb/src/store/compound/init.rs → git-odb/src/store_impls/compound/init.rs

@@ -2,7 +2,7 @@ use std::path::PathBuf;
 
 use crate::{
     pack,
-    store::{compound, loose},
+    store_impls::{compound, loose},
 };
 
 /// Returned by [`compound::Store::at()`]
@@ -23,7 +23,7 @@ pub enum Error {
 impl compound::Store {
     /// Returns a compound database as initialized from the given git `objects_directory`, commonly `.git/objects`.
     ///
-    /// Only loose and packed objects will be considered. See the [linked Db][crate::store::linked::Store] for a database with
+    /// Only loose and packed objects will be considered. See the [linked Db][crate::linked::Store] for a database with
     /// support for _git alternates_, i.e. linking to other repositories.
     ///
     /// `pack_id_offset` is used to allow multiple compound databases to be used for lookups without their pack-ids clashing.

git-odb/src/store/compound/mod.rs → git-odb/src/store_impls/compound/mod.rs

@@ -1,5 +1,5 @@
 //! An object database delegating object access to multiple contained object databases with loose and packed objects.
-use crate::{pack, store::loose};
+use crate::{pack, store_impls::loose};
 
 ///
 pub mod find;

git-odb/src/store/compound/write.rs → git-odb/src/store_impls/compound/write.rs

@@ -2,7 +2,7 @@ use std::io::Read;
 
 use git_object::Kind;
 
-use crate::store::{compound, loose};
+use crate::store_impls::{compound, loose};
 
 impl crate::traits::Write for compound::Store {
     type Error = loose::write::Error;

git-odb/src/store/dynamic/find.rs → git-odb/src/store_impls/dynamic/find.rs

@@ -4,7 +4,7 @@ use git_hash::oid;
 use git_object::Data;
 use git_pack::{cache::DecodeEntry, data::entry::Location};
 
-use crate::dynamic::handle;
+use crate::store::handle;
 
 mod error {
     use crate::{loose, pack};
@@ -18,14 +18,14 @@ mod error {
         #[error("An error occurred while obtaining an object from the packed object store")]
         Pack(#[from] pack::data::decode_entry::Error),
         #[error(transparent)]
-        LoadIndex(#[from] crate::dynamic::load_index::Error),
+        LoadIndex(#[from] crate::store::load_index::Error),
         #[error(transparent)]
         LoadPack(#[from] std::io::Error),
     }
 }
 pub use error::Error;
 
-use crate::dynamic;
+use crate::store::types::PackId;
 
 impl<S> crate::pack::Find for super::Handle<S>
 where
@@ -244,7 +244,7 @@ where
             matches!(self.token.as_ref(), Some(handle::Mode::KeepDeletedPacksAvailable)),
             "BUG: handle must be configured to `prevent_pack_unload()` before using this method"
         );
-        let pack_id = dynamic::types::PackId::from_intrinsic_pack_id(pack_id);
+        let pack_id = PackId::from_intrinsic_pack_id(pack_id);
         loop {
             let snapshot = self.snapshot.borrow();
             {
@@ -270,7 +270,7 @@ where
             matches!(self.token.as_ref(), Some(handle::Mode::KeepDeletedPacksAvailable)),
             "BUG: handle must be configured to `prevent_pack_unload()` before using this method"
         );
-        let pack_id = dynamic::types::PackId::from_intrinsic_pack_id(location.pack_id);
+        let pack_id = PackId::from_intrinsic_pack_id(location.pack_id);
         'outer: loop {
             let mut snapshot = self.snapshot.borrow_mut();
             {

git-odb/src/store/dynamic/handle.rs → git-odb/src/store_impls/dynamic/handle.rs

@@ -6,7 +6,7 @@ use std::{
 
 use git_features::threading::OwnShared;
 
-use crate::dynamic::{handle, types};
+use crate::store::{handle, types};
 
 pub(crate) mod multi_index {
     // TODO: replace this one with an actual implementation of a multi-pack index.
@@ -42,7 +42,7 @@ pub(crate) mod index_lookup {
 
     use git_hash::oid;
 
-    use crate::dynamic::{handle, types};
+    use crate::store::{handle, types};
 
     pub(crate) struct Outcome<'a> {
         pub object_index: handle::IndexForObjectInPack,

git-odb/src/store/dynamic/init.rs → git-odb/src/store_impls/dynamic/init.rs

@@ -2,7 +2,7 @@ use std::{iter::FromIterator, path::PathBuf, sync::Arc};
 
 use arc_swap::ArcSwap;
 
-use crate::dynamic::types::{MutableIndexAndPack, SlotMapIndex};
+use crate::store::types::{MutableIndexAndPack, SlotMapIndex};
 
 /// Configures the amount of slots in the index slotmap, which is fixed throughout the existence of the store.
 pub enum Slots {

git-odb/src/store/dynamic/iter.rs → git-odb/src/store_impls/dynamic/iter.rs

@@ -2,7 +2,7 @@ use std::{ops::Deref, option::Option::None, sync::Arc, vec::IntoIter};
 
 use git_hash::ObjectId;
 
-use crate::{dynamic::handle, loose, store::dynamic};
+use crate::{loose, store::handle, store_impls::dynamic};
 
 enum State {
     Pack {
@@ -27,7 +27,7 @@ pub struct AllObjects {
 
 impl AllObjects {
     /// Create a new iterator from a general database, which will be forced to load all indices eagerly.
-    pub fn new(db: &dynamic::Store) -> Result<Self, crate::dynamic::load_index::Error> {
+    pub fn new(db: &dynamic::Store) -> Result<Self, crate::store::load_index::Error> {
         let mut snapshot = db.collect_snapshot();
         while let Some(new_snapshot) = db.load_one_index(crate::RefreshMode::Never, snapshot.marker)? {
             snapshot = new_snapshot

git-odb/src/store/dynamic/load_index.rs → git-odb/src/store_impls/dynamic/load_index.rs

@@ -11,7 +11,7 @@ use std::{
 };
 
 use crate::{
-    dynamic::{handle, types},
+    store::{handle, types},
     RefreshMode,
 };
 
@@ -52,7 +52,7 @@ mod error {
 
 pub use error::Error;
 
-use crate::dynamic::types::{Generation, IndexAndPacks, MutableIndexAndPack, SlotMapIndex};
+use crate::store::types::{Generation, IndexAndPacks, MutableIndexAndPack, SlotMapIndex};
 
 impl super::Store {
     /// If `None` is returned, there is new indices and the caller should give up. This is a possibility even if it's allowed to refresh

git-odb/src/store/dynamic/load_pack.rs → git-odb/src/store_impls/dynamic/load_pack.rs

@@ -1,6 +1,6 @@
 use std::{path::Path, sync::Arc};
 
-use crate::dynamic::types;
+use crate::store::types;
 
 impl super::Store {
     /// If Ok(None) is returned, the pack-id was stale and referred to an unloaded pack or a pack which couldn't be

git-odb/src/store/dynamic/metrics.rs → git-odb/src/store_impls/dynamic/metrics.rs

@@ -1,6 +1,6 @@
 use std::sync::atomic::Ordering;
 
-use crate::dynamic::{types, types::IndexAndPacks};
+use crate::store::{types, types::IndexAndPacks};
 
 impl super::Store {
     pub fn metrics(&self) -> types::Metrics {