Client IPC Interface (#1493)

* btree map serializer

* serde tests

* state diff serialization

* basic layout

* more missing serializaers

* uncle returns rlp

* block queue info

* sorting with transaction result

* sorting out util imports

* transaction import result sorting also

* sorting filters & ranges

* error sorting out

* deriving ipc service compiling

* rpc & sync recompile

* sorting rpc using uncles

* fix compilation

* fix merging bugs

* fix unused imports

* fix all warnings

* tests stub

* some merge bugs

* ethcore compilation

* fix rpc compilation

* deriving attribute

* tests (and fixes)

* rpc test working

* fix warnings again

* rs.in -> rs

* missing attribute

* refactored tree changes

* paste reformat mess fix

* pub mod actually

* intendation fix

ethcore/Cargo.toml

@@ -31,6 +31,8 @@ bloomchain = "0.1"
 "ethcore-ipc" = { path = "../ipc/rpc" }
 rayon = "0.3.1"
 ethstore = { path = "../ethstore" }
+semver = "0.2"
+ethcore-ipc-nano = { path = "../ipc/nano" }
 
 [dependencies.hyper]
 git = "https://github.com/ethcore/hyper"

ethcore/build.rs

@@ -30,4 +30,18 @@ fn main() {
 		codegen::register(&mut registry);
 		registry.expand("", &src, &dst).unwrap();
 	}
+
+	// client interface
+	{
+		let src = Path::new("src/client/client.rs");
+		let intermediate = Path::new(&out_dir).join("client.intermediate.rs.in");
+		let mut registry = syntex::Registry::new();
+		codegen::register(&mut registry);
+		registry.expand("", &src, &intermediate).unwrap();
+
+		let dst = Path::new(&out_dir).join("client.ipc.rs");
+		let mut registry = syntex::Registry::new();
+		codegen::register(&mut registry);
+		registry.expand("", &intermediate, &dst).unwrap();
+	}
 }

ethcore/src/block_queue.rs

@@ -28,6 +28,8 @@ use service::*;
 use client::BlockStatus;
 use util::panics::*;
 
+pub use types::block_queue_info::BlockQueueInfo;
+
 known_heap_size!(0, UnverifiedBlock, VerifyingBlock, PreverifiedBlock);
 
 const MIN_MEM_LIMIT: usize = 16384;
@@ -53,22 +55,6 @@ impl Default for BlockQueueConfig {
 	}
 }
 
-/// Block queue status
-#[derive(Debug)]
-pub struct BlockQueueInfo {
-	/// Number of queued blocks pending verification
-	pub unverified_queue_size: usize,
-	/// Number of verified queued blocks pending import
-	pub verified_queue_size: usize,
-	/// Number of blocks being verified
-	pub verifying_queue_size: usize,
-	/// Configured maximum number of blocks in the queue
-	pub max_queue_size: usize,
-	/// Configured maximum number of bytes to use
-	pub max_mem_use: usize,
-	/// Heap memory used in bytes
-	pub mem_used: usize,
-}
 
 impl BlockQueueInfo {
 	/// The total size of the queues.

ethcore/src/blockchain/block_info.rs

@@ -17,8 +17,6 @@
 use util::numbers::{U256,H256};
 use header::BlockNumber;
 
-use util::bytes::{FromRawBytesVariable, FromBytesError, ToBytesWithMap};
-
 /// Brief info about inserted block.
 #[derive(Clone)]
 pub struct BlockInfo {
@@ -54,43 +52,3 @@ pub struct BranchBecomingCanonChainData {
 	/// Hashes of the blocks which were invalidated.
 	pub retracted: Vec<H256>,
 }
-
-impl FromRawBytesVariable for BranchBecomingCanonChainData {
-	fn from_bytes_variable(bytes: &[u8]) -> Result<BranchBecomingCanonChainData, FromBytesError> {
-		type Tuple = (Vec<H256>, Vec<H256>, H256);
-		let (enacted, retracted, ancestor) = try!(Tuple::from_bytes_variable(bytes));
-		Ok(BranchBecomingCanonChainData { ancestor: ancestor, enacted: enacted, retracted: retracted })
-	}
-}
-
-impl FromRawBytesVariable for BlockLocation {
-	fn from_bytes_variable(bytes: &[u8]) -> Result<BlockLocation, FromBytesError> {
-		match bytes[0] {
-			0 => Ok(BlockLocation::CanonChain),
-			1 => Ok(BlockLocation::Branch),
-			2 => Ok(BlockLocation::BranchBecomingCanonChain(
-				try!(BranchBecomingCanonChainData::from_bytes_variable(&bytes[1..bytes.len()])))),
-			_ => Err(FromBytesError::UnknownMarker)
-		}
-	}
-}
-
-impl ToBytesWithMap for BranchBecomingCanonChainData {
-	fn to_bytes_map(&self) -> Vec<u8> {
-		(&self.enacted, &self.retracted, &self.ancestor).to_bytes_map()
-	}
-}
-
-impl ToBytesWithMap for BlockLocation {
-	fn to_bytes_map(&self) -> Vec<u8> {
-		match *self {
-			BlockLocation::CanonChain => vec![0u8],
-			BlockLocation::Branch => vec![1u8],
-			BlockLocation::BranchBecomingCanonChain(ref data) => {
-				let mut bytes = (&data.enacted, &data.retracted, &data.ancestor).to_bytes_map();
-				bytes.insert(0, 2u8);
-				bytes
-			}
-		}
-	}
-}

ethcore/src/client/client.rs

@@ -14,16 +14,35 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Blockchain database client.
-
 use std::path::PathBuf;
+use std::collections::{HashSet, HashMap};
+use std::ops::Deref;
+use std::mem;
+use std::collections::VecDeque;
+use std::sync::*;
+use std::path::Path;
+use std::fmt;
+use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
 use std::time::Instant;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
-use util::*;
+
+// util
+use util::numbers::*;
 use util::panics::*;
+use util::network::*;
+use util::io::*;
+use util::rlp;
+use util::sha3::*;
+use util::{Bytes};
+use util::rlp::{RlpStream, Rlp, UntrustedRlp};
+use util::journaldb;
+use util::journaldb::JournalDB;
+use util::kvdb::*;
+use util::{Applyable, Stream, View, PerfTimer, Itertools, Colour};
+
+// other
 use views::BlockView;
 use error::{ImportError, ExecutionError, BlockError, ImportResult};
-use header::{BlockNumber};
+use header::BlockNumber;
 use state::State;
 use spec::Spec;
 use engine::Engine;
@@ -35,24 +54,29 @@ use verification::{PreverifiedBlock, Verifier};
 use block::*;
 use transaction::{LocalizedTransaction, SignedTransaction, Action};
 use blockchain::extras::TransactionAddress;
-use filter::Filter;
+use types::filter::Filter;
 use log_entry::LocalizedLogEntry;
 use block_queue::{BlockQueue, BlockQueueInfo};
 use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute};
-use client::{BlockID, TransactionID, UncleID, TraceId, Mode, ClientConfig, DatabaseCompactionProfile,
-	BlockChainClient, MiningBlockChainClient, TraceFilter, CallAnalytics,
-	BlockImportError};
+use client::{BlockID, TransactionID, UncleID, TraceId, ClientConfig,
+	DatabaseCompactionProfile, BlockChainClient, MiningBlockChainClient,
+	TraceFilter, CallAnalytics, BlockImportError, Mode};
 use client::Error as ClientError;
 use env_info::EnvInfo;
 use executive::{Executive, Executed, TransactOptions, contract_address};
 use receipt::LocalizedReceipt;
-pub use blockchain::CacheSize as BlockChainCacheSize;
 use trace::{TraceDB, ImportRequest as TraceImportRequest, LocalizedTrace, Database as TraceDatabase};
 use trace;
-pub use types::blockchain_info::BlockChainInfo;
-pub use types::block_status::BlockStatus;
 use evm::Factory as EvmFactory;
 use miner::{Miner, MinerService};
+use util::TrieFactory;
+use ipc::IpcConfig;
+use ipc::binary::{BinaryConvertError};
+
+// re-export
+pub use types::blockchain_info::BlockChainInfo;
+pub use types::block_status::BlockStatus;
+pub use blockchain::CacheSize as BlockChainCacheSize;
 
 const MAX_TX_QUEUE_SIZE: usize = 4096;
 const MAX_QUEUE_SIZE_TO_SLEEP_ON: usize = 2;
@@ -452,7 +476,7 @@ impl Client {
 			HeaderView::new(&self.best_block_header()).state_root(),
 			self.engine.account_start_nonce(),
 			self.trie_factory.clone())
-			.expect("State root of best block header always valid.")
+		.expect("State root of best block header always valid.")
 	}
 
 	/// Get info on the cache.
@@ -472,12 +496,12 @@ impl Client {
 	pub fn tick(&self) {
 		self.chain.collect_garbage();
 		self.block_queue.collect_garbage();
-		
+
 		match self.mode {
 			Mode::Dark(timeout) => {
 				let mut ss = self.sleep_state.lock().unwrap();
 				if let Some(t) = ss.last_activity {
-					if Instant::now() > t + timeout { 
+					if Instant::now() > t + timeout {
 						self.sleep();
 						ss.last_activity = None;
 					}
@@ -487,14 +511,14 @@ impl Client {
 				let mut ss = self.sleep_state.lock().unwrap();
 				let now = Instant::now();
 				if let Some(t) = ss.last_activity {
-					if now > t + timeout { 
+					if now > t + timeout {
 						self.sleep();
 						ss.last_activity = None;
 						ss.last_autosleep = Some(now);
 					}
 				}
-				if let Some(t) = ss.last_autosleep { 
-					if now > t + wakeup_after { 
+				if let Some(t) = ss.last_autosleep {
+					if now > t + wakeup_after {
 						self.wake_up();
 						ss.last_activity = Some(now);
 						ss.last_autosleep = None;
@@ -575,6 +599,8 @@ impl Client {
 	}
 }
 
+#[derive(Ipc)]
+#[ipc(client_ident="RemoteClient")]
 impl BlockChainClient for Client {
 	fn call(&self, t: &SignedTransaction, analytics: CallAnalytics) -> Result<Executed, ExecutionError> {
 		let header = self.block_header(BlockID::Latest).unwrap();
@@ -811,8 +837,8 @@ impl BlockChainClient for Client {
 						receipt.logs.into_iter()
 							.enumerate()
 							.filter(|tuple| filter.matches(&tuple.1))
-						 	.map(|(i, log)| LocalizedLogEntry {
-							 	entry: log,
+							.map(|(i, log)| LocalizedLogEntry {
+								entry: log,
 								block_hash: hash.clone(),
 								block_number: number,
 								transaction_hash: hashes.get(index).cloned().unwrap_or_else(H256::new),
@@ -822,7 +848,6 @@ impl BlockChainClient for Client {
 							.collect::<Vec<LocalizedLogEntry>>()
 					})
 					.collect::<Vec<LocalizedLogEntry>>()
-
 			})
 			.collect()
 	}
@@ -966,3 +991,5 @@ impl MayPanic for Client {
 		self.panic_handler.on_panic(closure);
 	}
 }
+
+impl IpcConfig for Client { }

ethcore/src/client/mod.rs

@@ -16,7 +16,6 @@
 
 //! Blockchain database client.
 
-mod client;
 mod config;
 mod error;
 mod test_client;
@@ -27,7 +26,7 @@ pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockQueue
 pub use self::error::Error;
 pub use types::ids::*;
 pub use self::test_client::{TestBlockChainClient, EachBlockWith};
-pub use self::trace::Filter as TraceFilter;
+pub use types::trace_filter::Filter as TraceFilter;
 pub use executive::{Executed, Executive, TransactOptions};
 pub use env_info::{LastHashes, EnvInfo};
 
@@ -47,23 +46,21 @@ use error::{ImportResult, ExecutionError};
 use receipt::LocalizedReceipt;
 use trace::LocalizedTrace;
 use evm::Factory as EvmFactory;
+pub use types::call_analytics::CallAnalytics;
 pub use block_import_error::BlockImportError;
 pub use transaction_import::TransactionImportResult;
+pub use transaction_import::TransactionImportError;
 
-/// Options concerning what analytics we run on the call.
-#[derive(Eq, PartialEq, Default, Clone, Copy, Debug)]
-pub struct CallAnalytics {
-	/// Make a transaction trace.
-	pub transaction_tracing: bool,
-	/// Make a VM trace.
-	pub vm_tracing: bool,
-	/// Make a diff.
-	pub state_diffing: bool,
+pub mod client {
+	//! Blockchain database client.
+
+	#![allow(dead_code, unused_assignments, unused_variables, missing_docs)] // codegen issues
+	include!(concat!(env!("OUT_DIR"), "/client.ipc.rs"));
 }
 
 /// Blockchain database client. Owns and manages a blockchain and a block queue.
 pub trait BlockChainClient : Sync + Send {
-	
+
 	/// Should be called by any external-facing interface when actively using the client.
 	/// To minimise chatter, there's no need to call more than once every 30s.
 	fn keep_alive(&self) {}

ethcore/src/client/test_client.rs

@@ -36,7 +36,7 @@ use spec::Spec;
 use block_queue::BlockQueueInfo;
 use block::{OpenBlock, SealedBlock};
 use executive::Executed;
-use error::{ExecutionError};
+use error::ExecutionError;
 use trace::LocalizedTrace;
 
 /// Test client.

ethcore/src/client/trace.rs

@@ -1,13 +1,12 @@
 
 //! Bridge between Tracedb and Blockchain.
 
-use std::ops::Range;
-use util::{Address, H256};
+use util::{H256};
 use header::BlockNumber;
 use trace::DatabaseExtras as TraceDatabaseExtras;
 use blockchain::{BlockChain, BlockProvider};
 use blockchain::extras::TransactionAddress;
-use super::BlockID;
+pub use types::trace_filter::Filter;
 
 impl TraceDatabaseExtras for BlockChain {
 	fn block_hash(&self, block_number: BlockNumber) -> Option<H256> {
@@ -26,13 +25,3 @@ impl TraceDatabaseExtras for BlockChain {
 			.map(|tx| tx.hash())
 	}
 }
-
-/// Easy to use trace filter.
-pub struct Filter {
-	/// Range of filtering.
-	pub range: Range<BlockID>,
-	/// From address.
-	pub from_address: Vec<Address>,
-	/// To address.
-	pub to_address: Vec<Address>,
-}

ethcore/src/error.rs

@@ -20,9 +20,9 @@ use util::*;
 use header::BlockNumber;
 use basic_types::LogBloom;
 use client::Error as ClientError;
-pub use types::executed::ExecutionError;
 use ipc::binary::{BinaryConvertError, BinaryConvertable};
 use types::block_import_error::BlockImportError;
+pub use types::executed::ExecutionError;
 
 #[derive(Debug, PartialEq, Clone)]
 /// Errors concerning transaction processing.
@@ -327,7 +327,6 @@ binary_fixed_size!(BlockError);
 binary_fixed_size!(ImportError);
 binary_fixed_size!(TransactionError);
 
-
 // TODO: uncomment below once https://github.com/rust-lang/rust/issues/27336 sorted.
 /*#![feature(concat_idents)]
 macro_rules! assimilate {