Restore debug_dumpdb RPC call (#2570)

* debug_dumpdb

* Format

* Fix doc

* Remove format

* Lowercase arg

* Default limit to 100

lib/ain-evm/src/evm.rs

@@ -8,8 +8,11 @@ use anyhow::format_err;
 use ethereum::{Block, PartialHeader, ReceiptV3};
 use ethereum_types::{Bloom, H160, H256, H64, U256};
 use log::{debug, trace};
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver, UnboundedSender},
+    RwLock,
+};
 
-use crate::log::Notification;
 use crate::{
     backend::{EVMBackend, Vicinity},
     block::BlockService,
@@ -22,7 +25,7 @@ use crate::{
     core::{EVMCoreService, XHash},
     executor::AinExecutor,
     filters::FilterService,
-    log::LogService,
+    log::{LogService, Notification},
     receipt::ReceiptService,
     storage::{traits::BlockStorage, Storage},
     transaction::{
@@ -33,8 +36,6 @@ use crate::{
     txqueue::{BlockData, QueueTx},
     EVMError, Result,
 };
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::sync::RwLock;
 
 pub struct NotificationChannel<T> {
     pub sender: UnboundedSender<T>,

lib/ain-evm/src/storage/block_store.rs

@@ -1,12 +1,13 @@
-use std::{collections::HashMap, fs, marker::PhantomData, path::Path, sync::Arc};
+use std::{collections::HashMap, fs, marker::PhantomData, path::Path, str::FromStr, sync::Arc};
 
 use anyhow::format_err;
 use ethereum::{BlockAny, TransactionV2};
 use ethereum_types::{H160, H256, U256};
 use log::debug;
+use serde::{Deserialize, Serialize};
 
 use super::{
-    db::{Column, ColumnName, LedgerColumn, Rocks},
+    db::{Column, ColumnName, LedgerColumn, Rocks, TypedColumn},
     traits::{BlockStorage, FlushableStorage, ReceiptStorage, Rollback, TransactionStorage},
 };
 use crate::{
@@ -127,7 +128,7 @@ impl BlockStorage for BlockStore {
     fn get_latest_block(&self) -> Result<Option<BlockAny>> {
         let latest_block_cf = self.column::<columns::LatestBlockNumber>();
 
-        match latest_block_cf.get(&()) {
+        match latest_block_cf.get(&"") {
             Ok(Some(block_number)) => self.get_block_by_number(&block_number),
             Ok(None) => Ok(None),
             Err(e) => Err(e),
@@ -138,7 +139,7 @@ impl BlockStorage for BlockStore {
         if let Some(block) = block {
             let latest_block_cf = self.column::<columns::LatestBlockNumber>();
             let block_number = block.header.number;
-            latest_block_cf.put(&(), &block_number)?;
+            latest_block_cf.put(&"latest_block", &block_number)?;
         }
         Ok(())
     }
@@ -217,9 +218,64 @@ impl Rollback for BlockStore {
 
             if let Some(block) = self.get_block_by_hash(&block.header.parent_hash)? {
                 let latest_block_cf = self.column::<columns::LatestBlockNumber>();
-                latest_block_cf.put(&(), &block.header.number)?;
+                latest_block_cf.put(&"latest_block", &block.header.number)?;
             }
         }
         Ok(())
     }
 }
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum DumpArg {
+    All,
+    Blocks,
+    Txs,
+    Receipts,
+    BlockMap,
+    Logs,
+}
+
+impl BlockStore {
+    pub fn dump(&self, arg: &DumpArg, from: Option<&str>, limit: usize) {
+        let s_to_u256 = |s| {
+            U256::from_str_radix(s, 10)
+                .or(U256::from_str_radix(s, 16))
+                .unwrap_or_else(|_| U256::zero())
+        };
+        let s_to_h256 = |s: &str| H256::from_str(&s).unwrap_or(H256::zero());
+
+        match arg {
+            DumpArg::All => {
+                self.dump_all(limit);
+            }
+            DumpArg::Blocks => self.dump_column(columns::Blocks, from.map(s_to_u256), limit),
+            DumpArg::Txs => self.dump_column(columns::Transactions, from.map(s_to_h256), limit),
+            DumpArg::Receipts => self.dump_column(columns::Receipts, from.map(s_to_h256), limit),
+            DumpArg::BlockMap => self.dump_column(columns::BlockMap, from.map(s_to_h256), limit),
+            DumpArg::Logs => self.dump_column(columns::AddressLogsMap, from.map(s_to_u256), limit),
+        }
+    }
+
+    fn dump_all(&self, limit: usize) {
+        for arg in &[
+            DumpArg::Blocks,
+            DumpArg::Txs,
+            DumpArg::Receipts,
+            DumpArg::BlockMap,
+            DumpArg::Logs,
+        ] {
+            self.dump(arg, None, limit);
+        }
+    }
+
+    fn dump_column<C>(&self, _: C, from: Option<C::Index>, limit: usize)
+    where
+        C: TypedColumn + ColumnName,
+    {
+        println!("{}", C::NAME);
+        for (k, v) in self.column::<C>().iter(from, limit) {
+            println!("{:?}: {:#?}", k, v);
+        }
+    }
+}

lib/ain-evm/src/storage/db.rs

@@ -1,5 +1,7 @@
 use std::{
     collections::HashMap,
+    fmt::Debug,
+    iter::Iterator,
     marker::PhantomData,
     path::{Path, PathBuf},
     sync::Arc,
@@ -8,7 +10,10 @@ use std::{
 use bincode;
 use ethereum::{BlockAny, TransactionV2};
 use ethereum_types::{H160, H256, U256};
-use rocksdb::{BlockBasedOptions, Cache, ColumnFamily, ColumnFamilyDescriptor, Options, DB};
+use rocksdb::{
+    BlockBasedOptions, Cache, ColumnFamily, ColumnFamilyDescriptor, DBIterator, IteratorMode,
+    Options, DB,
+};
 use serde::{de::DeserializeOwned, Serialize};
 
 use crate::{log::LogIndex, receipt::Receipt, Result};
@@ -97,6 +102,13 @@ impl Rocks {
         Ok(())
     }
 
+    pub fn iterator_cf<C>(&self, cf: &ColumnFamily, iterator_mode: IteratorMode) -> DBIterator
+    where
+        C: Column,
+    {
+        self.0.iterator_cf(cf, iterator_mode)
+    }
+
     pub fn flush(&self) -> Result<()> {
         self.0.flush()?;
         Ok(())
@@ -209,9 +221,11 @@ impl ColumnName for columns::CodeMap {
 // Column trait. Define associated index type
 //
 pub trait Column {
-    type Index;
+    type Index: Debug;
 
     fn key(index: &Self::Index) -> Vec<u8>;
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index;
 }
 
 //
@@ -224,6 +238,10 @@ impl Column for columns::Transactions {
     fn key(index: &Self::Index) -> Vec<u8> {
         index.as_bytes().to_vec()
     }
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index {
+        Self::Index::from_slice(&raw_key)
+    }
 }
 
 impl Column for columns::Blocks {
@@ -234,6 +252,10 @@ impl Column for columns::Blocks {
         index.to_big_endian(&mut bytes);
         bytes.to_vec()
     }
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index {
+        Self::Index::from(&*raw_key)
+    }
 }
 
 impl Column for columns::Receipts {
@@ -242,21 +264,33 @@ impl Column for columns::Receipts {
     fn key(index: &Self::Index) -> Vec<u8> {
         index.to_fixed_bytes().to_vec()
     }
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index {
+        Self::Index::from_slice(&raw_key)
+    }
 }
 impl Column for columns::BlockMap {
     type Index = H256;
 
     fn key(index: &Self::Index) -> Vec<u8> {
         index.to_fixed_bytes().to_vec()
     }
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index {
+        Self::Index::from_slice(&raw_key)
+    }
 }
 
 impl Column for columns::LatestBlockNumber {
-    type Index = ();
+    type Index = &'static str;
 
     fn key(_index: &Self::Index) -> Vec<u8> {
         b"latest".to_vec()
     }
+
+    fn get_key(_raw_key: Box<[u8]>) -> Self::Index {
+        "latest"
+    }
 }
 
 impl Column for columns::AddressLogsMap {
@@ -267,6 +301,10 @@ impl Column for columns::AddressLogsMap {
         index.to_big_endian(&mut bytes);
         bytes.to_vec()
     }
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index {
+        Self::Index::from(&*raw_key)
+    }
 }
 
 impl Column for columns::CodeMap {
@@ -275,13 +313,17 @@ impl Column for columns::CodeMap {
     fn key(index: &Self::Index) -> Vec<u8> {
         index.to_fixed_bytes().to_vec()
     }
+
+    fn get_key(raw_key: Box<[u8]>) -> Self::Index {
+        H256::from_slice(&raw_key)
+    }
 }
 
 //
 // TypedColumn trait. Define associated value type
 //
 pub trait TypedColumn: Column {
-    type Type: Serialize + DeserializeOwned;
+    type Type: Serialize + DeserializeOwned + Debug;
 }
 
 //
@@ -334,4 +376,26 @@ where
     pub fn delete(&self, key: &C::Index) -> Result<()> {
         self.backend.delete_cf(self.handle(), &C::key(key))
     }
+
+    pub fn iter(
+        &self,
+        from: Option<C::Index>,
+        limit: usize,
+    ) -> impl Iterator<Item = (C::Index, C::Type)> + '_ {
+        let index = from.as_ref().map(|i| C::key(i)).unwrap_or_default();
+        let iterator_mode = from.map_or(IteratorMode::Start, |_| {
+            IteratorMode::From(&index, rocksdb::Direction::Forward)
+        });
+        self.backend
+            .iterator_cf::<C>(self.handle(), iterator_mode)
+            .into_iter()
+            .filter_map(|k| {
+                k.ok().and_then(|(k, v)| {
+                    let value = bincode::deserialize(&v).ok()?;
+                    let key = C::get_key(k);
+                    Some((key, value))
+                })
+            })
+            .take(limit)
+    }
 }

lib/ain-evm/src/storage/mod.rs

@@ -1,4 +1,4 @@
-mod block_store;
+pub mod block_store;
 mod cache;
 mod db;
 pub mod traits;
@@ -10,7 +10,7 @@ use ethereum::{BlockAny, TransactionV2};
 use ethereum_types::{H160, H256, U256};
 
 use self::{
-    block_store::BlockStore,
+    block_store::{BlockStore, DumpArg},
     cache::Cache,
     traits::{
         AttributesStorage, BlockStorage, FlushableStorage, ReceiptStorage, Rollback,
@@ -206,8 +206,9 @@ impl Storage {
 }
 
 impl Storage {
-    pub fn dump_db(&self) {
-        // println!("self.block_data_handler : {:#?}", self.blockstore);
+    pub fn dump_db(&self, arg: DumpArg, from: Option<&str>, limit: usize) {
+        println!("[dump_db]");
+        self.blockstore.dump(&arg, from, limit)
     }
 }
 

lib/ain-evm/src/weiamount.rs

@@ -69,11 +69,13 @@ pub fn try_from_satoshi(satoshi: U256) -> Result<WeiAmount> {
 
 #[cfg(test)]
 mod tests {
+    use std::error::Error;
+
+    use ethereum_types::U256;
+
     use crate::weiamount::{
         try_from_satoshi, WeiAmount, MAX_MONEY_SATS, MAX_MONEY_SATS_DOUBLE, WEI_TO_SATS,
     };
-    use ethereum_types::U256;
-    use std::error::Error;
 
     #[test]
     fn test_satoshi_double_conversion() -> Result<(), Box<dyn Error>> {

lib/ain-grpc/src/lib.rs

@@ -29,28 +29,29 @@ use std::{
 
 use ain_evm::services::{Services, IS_SERVICES_INIT_CALL, SERVICES};
 use anyhow::{format_err, Result};
-use hyper::header::HeaderValue;
-use hyper::Method;
+use hyper::{header::HeaderValue, Method};
 use jsonrpsee::core::server::rpc_module::Methods;
 use jsonrpsee_server::ServerBuilder;
 use log::info;
 use logging::CppLogTarget;
 use tower_http::cors::CorsLayer;
 
-use crate::rpc::{
-    debug::{MetachainDebugRPCModule, MetachainDebugRPCServer},
-    eth::{MetachainRPCModule, MetachainRPCServer},
-    net::{MetachainNetRPCModule, MetachainNetRPCServer},
-    web3::{MetachainWeb3RPCModule, MetachainWeb3RPCServer},
-};
-use crate::subscription::{
-    eth::{MetachainPubSubModule, MetachainPubSubServer},
-    MetachainSubIdProvider,
+use crate::{
+    rpc::{
+        debug::{MetachainDebugRPCModule, MetachainDebugRPCServer},
+        eth::{MetachainRPCModule, MetachainRPCServer},
+        net::{MetachainNetRPCModule, MetachainNetRPCServer},
+        web3::{MetachainWeb3RPCModule, MetachainWeb3RPCServer},
+    },
+    subscription::{
+        eth::{MetachainPubSubModule, MetachainPubSubServer},
+        MetachainSubIdProvider,
+    },
 };
 
 // TODO: Ideally most of the below and SERVICES needs to go into its own core crate now,
 // and this crate be dedicated to network services.
-// Note: This cannot just move to rs-exports, since rs-exports cannot cannot have reverse
+// Note: This cannot just move to rs-exports, since rs-exports cannot have reverse
 // deps that depend on it.
 
 pub fn preinit() {}

lib/ain-grpc/src/receipt.rs

@@ -1,5 +1,4 @@
-use ain_evm::log::LogIndex;
-use ain_evm::receipt::Receipt;
+use ain_evm::{log::LogIndex, receipt::Receipt};
 use ethereum::{EIP658ReceiptData, Log};
 use ethereum_types::{H160, H256, U256};