Ocean: Cleanup secondary index data access

lib/ain-macros/src/lib.rs

@@ -103,6 +103,8 @@ pub fn repository_derive(input: TokenStream) -> TokenStream {
     // Generate the implementation
     let expanded = quote! {
         impl RepositoryOps<#key_type_ident, #value_type_ident> for #name {
+            type ListItem = std::result::Result<(#key_type_ident, #value_type_ident), ain_db::DBError>;
+
             fn get(&self, id: &#key_type_ident) -> Result<Option<#value_type_ident>> {
                 Ok(self.col.get(id)?)
             }
@@ -115,8 +117,7 @@ pub fn repository_derive(input: TokenStream) -> TokenStream {
                 Ok(self.col.delete(id)?)
             }
 
-            fn list<'a>(&'a self, from: Option<#key_type_ident>, dir: crate::storage::SortOrder) -> crate::repository::ListResult<#key_type_ident, #value_type_ident>
-            {
+            fn list<'a>(&'a self, from: Option<#key_type_ident>, dir: crate::storage::SortOrder) -> Result<Box<dyn Iterator<Item = Self::ListItem> + 'a>> {
                 let it = self.col.iter(from, dir.into())?;
                 Ok(Box::new(it))
             }

lib/ain-ocean/src/api/block.rs

@@ -18,7 +18,9 @@ use crate::{
     api::common::Paginate,
     error::{ApiError, Error},
     model::{Block, BlockContext, Transaction},
-    repository::RepositoryOps,
+    repository::{
+        InitialKeyProvider, RepositoryOps, SecondaryIndex, TransactionByBlockHashRepository,
+    },
     storage::SortOrder,
     Result,
 };
@@ -100,23 +102,11 @@ async fn list_blocks(
         })
         .transpose()?;
 
-    let blocks = ctx
-        .services
-        .block
-        .by_height
+    let repository = &ctx.services.block.by_height;
+    let blocks = repository
         .list(next, SortOrder::Descending)?
         .paginate(&query)
-        .map(|item| {
-            let (_, id) = item?;
-            let b = ctx
-                .services
-                .block
-                .by_id
-                .get(&id)?
-                .ok_or("Missing block index")?;
-
-            Ok(b)
-        })
+        .map(|e| repository.retrieve_primary_value(e))
         .collect::<Result<Vec<_>>>()?;
 
     Ok(ApiPagedResponse::of(blocks, query.size, |block| {
@@ -147,34 +137,27 @@ async fn get_transactions(
     Query(query): Query<PaginationQuery>,
     Extension(ctx): Extension<Arc<AppContext>>,
 ) -> Result<ApiPagedResponse<TransactionResponse>> {
-    let next = query.next.as_ref().map_or(Ok((hash, 0)), |q| {
-        let height = q
-            .parse::<usize>()
-            .map_err(|_| format_err!("Invalid height"))?;
-        Ok::<(BlockHash, usize), Error>((hash, height))
-    })?;
-
-    let txs = ctx
-        .services
-        .transaction
-        .by_block_hash
+    let repository = &ctx.services.transaction.by_block_hash;
+
+    let next = query.next.as_ref().map_or(
+        Ok(TransactionByBlockHashRepository::initial_key(hash)),
+        |q| {
+            let height = q
+                .parse::<usize>()
+                .map_err(|_| format_err!("Invalid height"))?;
+            Ok::<(BlockHash, usize), Error>((hash, height))
+        },
+    )?;
+
+    let txs = repository
         .list(Some(next), SortOrder::Ascending)?
         .paginate(&query)
         .take_while(|item| match item {
             Ok(((h, _), _)) => h == &hash,
             _ => true,
         })
-        .map(|item| {
-            let (_, id) = item?;
-            let tx = ctx
-                .services
-                .transaction
-                .by_id
-                .get(&id)?
-                .ok_or("Missing tx index")?;
-
-            Ok(tx.into())
-        })
+        .map(|el| repository.retrieve_primary_value(el))
+        .map(|v| v.map(TransactionResponse::from))
         .collect::<Result<Vec<_>>>()?;
 
     Ok(ApiPagedResponse::of(txs, query.size, |tx| tx.order))

lib/ain-ocean/src/api/masternode.rs

@@ -19,7 +19,7 @@ use crate::{
     api::common::Paginate,
     error::{ApiError, Error, NotFoundKind},
     model::Masternode,
-    repository::RepositoryOps,
+    repository::{RepositoryOps, SecondaryIndex},
     storage::SortOrder,
     Result,
 };
@@ -105,6 +105,7 @@ async fn list_masternodes(
     Query(query): Query<PaginationQuery>,
     Extension(ctx): Extension<Arc<AppContext>>,
 ) -> Result<ApiPagedResponse<MasternodeData>> {
+    let repository = &ctx.services.masternode.by_height;
     let next = query
         .next
         .as_ref()
@@ -120,23 +121,11 @@ async fn list_masternodes(
         })
         .transpose()?;
 
-    let masternodes = ctx
-        .services
-        .masternode
-        .by_height
+    let masternodes = repository
         .list(next, SortOrder::Descending)?
         .paginate(&query)
-        .map(|item| {
-            let ((_, id), _) = item?;
-            let mn = ctx
-                .services
-                .masternode
-                .by_id
-                .get(&id)?
-                .ok_or("Missing masternode index")?;
-
-            Ok(mn.into())
-        })
+        .map(|el| repository.retrieve_primary_value(el))
+        .map(|v| v.map(MasternodeData::from))
         .collect::<Result<Vec<_>>>()?;
 
     Ok(ApiPagedResponse::of(

lib/ain-ocean/src/api/transactions.rs

@@ -7,11 +7,10 @@ use serde::Deserialize;
 
 use super::{path::Path, query::PaginationQuery, response::ApiPagedResponse, AppContext};
 use crate::{
-    api::common::Paginate,
-    api::response::Response,
+    api::{common::Paginate, response::Response},
     error::ApiError,
     model::{Transaction, TransactionVin, TransactionVout},
-    repository::RepositoryOps,
+    repository::{InitialKeyProvider, RepositoryOps, TransactionVinRepository},
     storage::SortOrder,
     Result,
 };
@@ -37,7 +36,10 @@ async fn get_vins(
     Query(query): Query<PaginationQuery>,
     Extension(ctx): Extension<Arc<AppContext>>,
 ) -> Result<ApiPagedResponse<TransactionVin>> {
-    let next = query.next.clone().unwrap_or(format!("{}00", id));
+    let next = query
+        .next
+        .clone()
+        .unwrap_or(TransactionVinRepository::initial_key(id));
 
     let list = ctx
         .services
@@ -50,14 +52,7 @@ async fn get_vins(
             _ => true,
         })
         .map(|item| {
-            let (id, _) = item?;
-            let v = ctx
-                .services
-                .transaction
-                .vin_by_id
-                .get(&id)?
-                .ok_or("Missing vin index")?;
-
+            let (_, v) = item?;
             Ok(v)
         })
         .collect::<Result<Vec<_>>>()?;
@@ -85,14 +80,7 @@ async fn get_vouts(
             _ => true,
         })
         .map(|item| {
-            let (id, _) = item?;
-            let v = ctx
-                .services
-                .transaction
-                .vout_by_id
-                .get(&id)?
-                .ok_or("Missing vout index")?;
-
+            let (_, v) = item?;
             Ok(v)
         })
         .collect::<Result<Vec<_>>>()?;

lib/ain-ocean/src/error.rs

@@ -44,6 +44,8 @@ pub enum Error {
     #[error("Unable to find {0:}")]
     NotFound(NotFoundKind),
     #[error("Decimal conversion error")]
+    SecondaryIndex,
+    #[error("Error fetching primary value")]
     DecimalError,
     #[error(transparent)]
     Other(#[from] anyhow::Error),

lib/ain-ocean/src/repository/block.rs

@@ -4,11 +4,11 @@ use ain_db::LedgerColumn;
 use ain_macros::Repository;
 use bitcoin::BlockHash;
 
-use super::RepositoryOps;
+use super::{RepositoryOps, SecondaryIndex};
 use crate::{
     model::Block,
     storage::{columns, ocean_store::OceanStore, SortOrder},
-    Result,
+    Error, Result,
 };
 
 #[derive(Repository)]
@@ -55,3 +55,16 @@ impl BlockByHeightRepository {
         }
     }
 }
+
+impl SecondaryIndex<u32, BlockHash> for BlockByHeightRepository {
+    type Value = Block;
+
+    fn retrieve_primary_value(&self, el: Self::ListItem) -> Result<Self::Value> {
+        let (_, id) = el?;
+
+        let col = self.store.column::<columns::Block>();
+        let tx = col.get(&id)?.ok_or(Error::SecondaryIndex)?;
+
+        Ok(tx)
+    }
+}

lib/ain-ocean/src/repository/masternode.rs

@@ -4,11 +4,11 @@ use ain_db::LedgerColumn;
 use ain_macros::Repository;
 use bitcoin::Txid;
 
-use super::RepositoryOps;
+use super::{RepositoryOps, SecondaryIndex};
 use crate::{
     model::Masternode,
     storage::{columns, ocean_store::OceanStore},
-    Result,
+    Error, Result,
 };
 
 #[derive(Repository)]
@@ -44,3 +44,14 @@ impl MasternodeByHeightRepository {
         }
     }
 }
+
+impl SecondaryIndex<MasternodeByHeightKey, u8> for MasternodeByHeightRepository {
+    type Value = Masternode;
+
+    fn retrieve_primary_value(&self, el: Self::ListItem) -> Result<Self::Value> {
+        let ((_, id), _) = el?;
+        let col = self.store.column::<columns::Masternode>();
+        let tx = col.get(&id)?.ok_or(Error::SecondaryIndex)?;
+        Ok(tx)
+    }
+}

lib/ain-ocean/src/repository/mod.rs

@@ -44,12 +44,24 @@ pub use transaction_vout::*;
 pub use tx_result::*;
 pub use vault_auction_batch_history::*;
 
-pub type ListResult<'a, K, V> =
-    Result<Box<dyn Iterator<Item = std::result::Result<(K, V), ain_db::DBError>> + 'a>>;
-
 pub trait RepositoryOps<K, V> {
+    type ListItem;
     fn get(&self, key: &K) -> Result<Option<V>>;
     fn put(&self, key: &K, value: &V) -> Result<()>;
     fn delete(&self, key: &K) -> Result<()>;
-    fn list(&self, from: Option<K>, direction: SortOrder) -> ListResult<K, V>;
+    fn list<'a>(
+        &'a self,
+        from: Option<K>,
+        direction: SortOrder,
+    ) -> Result<Box<dyn Iterator<Item = Self::ListItem> + 'a>>;
+}
+
+pub trait InitialKeyProvider<K, V>: RepositoryOps<K, V> {
+    type PartialKey;
+    fn initial_key(pk: Self::PartialKey) -> K;
+}
+
+pub trait SecondaryIndex<K, V>: RepositoryOps<K, V> {
+    type Value;
+    fn retrieve_primary_value(&self, el: Self::ListItem) -> Result<Self::Value>;
 }

lib/ain-ocean/src/repository/transaction.rs

@@ -2,13 +2,13 @@ use std::sync::Arc;
 
 use ain_db::LedgerColumn;
 use ain_macros::Repository;
-use bitcoin::Txid;
+use bitcoin::{BlockHash, Txid};
 
-use super::RepositoryOps;
+use super::{InitialKeyProvider, RepositoryOps, SecondaryIndex};
 use crate::{
     model::{Transaction, TransactionByBlockHashKey},
     storage::{columns, ocean_store::OceanStore},
-    Result,
+    Error, Result,
 };
 
 #[derive(Repository)]
@@ -42,3 +42,22 @@ impl TransactionByBlockHashRepository {
         }
     }
 }
+
+impl InitialKeyProvider<TransactionByBlockHashKey, Txid> for TransactionByBlockHashRepository {
+    type PartialKey = BlockHash;
+
+    fn initial_key(pk: Self::PartialKey) -> TransactionByBlockHashKey {
+        (pk, 0)
+    }
+}
+
+impl SecondaryIndex<TransactionByBlockHashKey, Txid> for TransactionByBlockHashRepository {
+    type Value = Transaction;
+
+    fn retrieve_primary_value(&self, el: Self::ListItem) -> Result<Self::Value> {
+        let (_, id) = el?;
+        let col = self.store.column::<columns::Transaction>();
+        let tx = col.get(&id)?.ok_or(Error::SecondaryIndex)?;
+        Ok(tx)
+    }
+}

lib/ain-ocean/src/repository/transaction_vin.rs

@@ -2,11 +2,15 @@ use std::sync::Arc;
 
 use ain_db::LedgerColumn;
 use ain_macros::Repository;
+use bitcoin::Txid;
 
-use super::RepositoryOps;
+use super::{InitialKeyProvider, RepositoryOps};
 use crate::{
     model::TransactionVin,
-    storage::{columns, ocean_store::OceanStore},
+    storage::{
+        columns::{self},
+        ocean_store::OceanStore,
+    },
     Result,
 };
 
@@ -25,3 +29,11 @@ impl TransactionVinRepository {
         }
     }
 }
+
+impl InitialKeyProvider<String, TransactionVin> for TransactionVinRepository {
+    type PartialKey = Txid;
+
+    fn initial_key(pk: Self::PartialKey) -> String {
+        format!("{}00", pk)
+    }
+}