Merge pull request #173 from gavofyork/ark

Multithreaded block queue

Cargo.toml

@@ -19,6 +19,7 @@ time = "0.1"
 #interpolate_idents = { git = "https://github.com/SkylerLipthay/interpolate_idents" }
 evmjit = { path = "rust-evmjit", optional = true }
 ethash = { path = "ethash" }
+num_cpus = "0.2"
 
 [features]
 jit = ["evmjit"]

src/block.rs

@@ -1,6 +1,7 @@
 use common::*;
 use engine::*;
 use state::*;
+use verification::PreVerifiedBlock;
 
 /// A transaction/receipt execution entry.
 pub struct Entry {
@@ -263,30 +264,39 @@ impl IsBlock for SealedBlock {
 	fn block(&self) -> &Block { &self.block }
 }
 
-/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
-pub fn enact<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result<ClosedBlock<'x, 'y>, Error> {
+/// Enact the block given by block header, transactions and uncles
+pub fn enact<'x, 'y>(header: &Header, transactions: &[Transaction], uncles: &[Header], engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result<ClosedBlock<'x, 'y>, Error> {
 	{
-		let header = BlockView::new(block_bytes).header_view();
 		let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce());
 		trace!("enact(): root={}, author={}, author_balance={}\n", s.root(), header.author(), s.balance(&header.author()));
 	}
 
-	let block = BlockView::new(block_bytes);
-	let header = block.header_view();
-	let mut b = OpenBlock::new(engine, db, parent, last_hashes, header.author(), header.extra_data());
-	b.set_difficulty(header.difficulty());
-	b.set_gas_limit(header.gas_limit());
+	let mut b = OpenBlock::new(engine, db, parent, last_hashes, header.author().clone(), header.extra_data().clone());
+	b.set_difficulty(*header.difficulty());
+	b.set_gas_limit(*header.gas_limit());
 	b.set_timestamp(header.timestamp());
-//	info!("enact: Enacting #{}. env_info={:?}", header.number(), b.env_info());
-	for t in block.transactions().into_iter() { try!(b.push_transaction(t, None)); }
-	for u in block.uncles().into_iter() { try!(b.push_uncle(u)); }
+	for t in transactions { try!(b.push_transaction(t.clone(), None)); }
+	for u in uncles { try!(b.push_uncle(u.clone())); }
 	Ok(b.close())
 }
 
+/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
+pub fn enact_bytes<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result<ClosedBlock<'x, 'y>, Error> {
+	let block = BlockView::new(block_bytes);
+	let header = block.header();
+	enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes)
+}
+
+/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
+pub fn enact_verified<'x, 'y>(block: &PreVerifiedBlock, engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result<ClosedBlock<'x, 'y>, Error> {
+	let view = BlockView::new(&block.bytes);
+	enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes)
+}
+
 /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards
 pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: OverlayDB, parent: &Header, last_hashes: &LastHashes) -> Result<SealedBlock, Error> {
 	let header = BlockView::new(block_bytes).header_view();
-	Ok(try!(try!(enact(block_bytes, engine, db, parent, last_hashes)).seal(header.seal())))
+	Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(header.seal())))
 }
 
 #[test]

src/client.rs

@@ -88,7 +88,7 @@ pub trait BlockChainClient : Sync + Send {
 	fn block_receipts(&self, hash: &H256) -> Option<Bytes>;
 
 	/// Import a block into the blockchain.
-	fn import_block(&mut self, byte: &[u8]) -> ImportResult;
+	fn import_block(&mut self, bytes: Bytes) -> ImportResult;
 
 	/// Get block queue information.
 	fn queue_status(&self) -> BlockQueueStatus;
@@ -152,58 +152,75 @@ impl Client {
 	}
 
 	/// This is triggered by a message coming from a block queue when the block is ready for insertion
-	pub fn import_verified_block(&mut self, bytes: Bytes) {
-		let block = BlockView::new(&bytes);
-		let header = block.header();
-		if let Err(e) = verify_block_family(&header, &bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) {
-			warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
-			self.queue.mark_as_bad(&header.hash());
+	pub fn import_verified_blocks(&mut self) {
+
+		let mut bad = HashSet::new();
+		let blocks = self.queue.drain(128); 
+		if blocks.is_empty() {
 			return;
-		};
-		let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) {
-			Some(p) => p,
-			None => {
-				warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash);
+		}
+
+		for block in blocks {
+			if bad.contains(&block.header.parent_hash) {
+				self.queue.mark_as_bad(&block.header.hash());
+				bad.insert(block.header.hash());
+				continue;
+			}
+
+			let header = &block.header;
+			if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) {
+				warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 				self.queue.mark_as_bad(&header.hash());
+				bad.insert(block.header.hash());
 				return;
-			},
-		};
-		// build last hashes
-		let mut last_hashes = LastHashes::new();
-		last_hashes.resize(256, H256::new());
-		last_hashes[0] = header.parent_hash.clone();
-		for i in 0..255 {
-			match self.chain.read().unwrap().block_details(&last_hashes[i]) {
-				Some(details) => {
-					last_hashes[i + 1] = details.parent.clone();
+			};
+			let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) {
+				Some(p) => p,
+				None => {
+					warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash);
+					self.queue.mark_as_bad(&header.hash());
+					bad.insert(block.header.hash());
+					return;
 				},
-				None => break,
+			};
+			// build last hashes
+			let mut last_hashes = LastHashes::new();
+			last_hashes.resize(256, H256::new());
+			last_hashes[0] = header.parent_hash.clone();
+			for i in 0..255 {
+				match self.chain.read().unwrap().block_details(&last_hashes[i]) {
+					Some(details) => {
+						last_hashes[i + 1] = details.parent.clone();
+					},
+					None => break,
+				}
 			}
-		}
 
-		let result = match enact(&bytes, self.engine.deref().deref(), self.state_db.clone(), &parent, &last_hashes) {
-			Ok(b) => b,
-			Err(e) => {
-				warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
+			let result = match enact_verified(&block, self.engine.deref().deref(), self.state_db.clone(), &parent, &last_hashes) {
+				Ok(b) => b,
+				Err(e) => {
+					warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
+				bad.insert(block.header.hash());
+					self.queue.mark_as_bad(&header.hash());
+					return;
+				}
+			};
+			if let Err(e) = verify_block_final(&header, result.block().header()) {
+				warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 				self.queue.mark_as_bad(&header.hash());
 				return;
 			}
-		};
-		if let Err(e) = verify_block_final(&header, result.block().header()) {
-			warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
-			self.queue.mark_as_bad(&header.hash());
-			return;
-		}
 
-		self.chain.write().unwrap().insert_block(&bytes); //TODO: err here?
-		match result.drain().commit() {
-			Ok(_) => (),
-			Err(e) => {
-				warn!(target: "client", "State DB commit failed: {:?}", e);
-				return;
+			self.chain.write().unwrap().insert_block(&block.bytes); //TODO: err here?
+			match result.drain().commit() {
+				Ok(_) => (),
+				Err(e) => {
+					warn!(target: "client", "State DB commit failed: {:?}", e);
+					return;
+				}
 			}
+			info!(target: "client", "Imported #{} ({})", header.number(), header.hash());
 		}
-		info!(target: "client", "Imported #{} ({})", header.number(), header.hash());
 	}
 }
 
@@ -261,8 +278,8 @@ impl BlockChainClient for Client {
 		unimplemented!();
 	}
 
-	fn import_block(&mut self, bytes: &[u8]) -> ImportResult {
-		let header = BlockView::new(bytes).header();
+	fn import_block(&mut self, bytes: Bytes) -> ImportResult {
+		let header = BlockView::new(&bytes).header();
 		if self.chain.read().unwrap().is_known(&header.hash()) {
 			return Err(ImportError::AlreadyInChain);
 		}

src/ethereum/ethash.rs

@@ -146,6 +146,10 @@ impl Engine for Ethash {
 		}
 		Ok(())
 	}
+
+	fn verify_transaction(&self, t: &Transaction, _header: &Header) -> Result<(), Error> {
+		t.sender().map(|_|()) // Perform EC recovery and cache sender
+	}
 }
 
 impl Ethash {

src/lib.rs

@@ -83,6 +83,7 @@ extern crate heapsize;
 extern crate crypto;
 extern crate time;
 extern crate env_logger;
+extern crate num_cpus;
 #[cfg(feature = "jit" )]
 extern crate evmjit;
 #[macro_use]