Asynchronous RPC support (#2017)

* Async RPC

* Limiting number of transactions in queue

* Fixing tests

* Bumping serde and jsonrpc-core

* serde updated to 0.8

* fixed failing tests

* Bumping ipc server

* Fixing API for endpoints

* Experimenting with tests without --release mode

.travis.yml

@@ -16,7 +16,7 @@ git:
 matrix:
   include:
   - rust: stable
-    env: RUN_TESTS="true"
+    env: RUN_TESTS="true" TEST_OPTIONS="--no-release"
   - rust: beta
     env: RUN_COVERAGE="true"
   - rust: stable
@@ -30,6 +30,7 @@ env:
   - RUN_TESTS="false"
   - RUN_COVERAGE="false"
   - RUN_DOCS="false"
+  - TEST_OPTIONS=""
   # GH_TOKEN for documentation
   - secure: bumJASbZSU8bxJ0EyPUJmu16AiV9EXOpyOj86Jlq/Ty9CfwGqsSXt96uDyE+OUJf34RUFQMsw0nk37/zC4lcn6kqk2wpuH3N/o85Zo/cVZY/NusBWLQqtT5VbYWsV+u2Ua4Tmmsw8yVYQhYwU2ZOejNpflL+Cs9XGgORp1L+/gMRMC2y5Se6ZhwnKPQlRJ8LGsG1dzjQULxzADIt3/zuspNBS8a2urJwlHfGMkvHDoUWCviP/GXoSqw3TZR7FmKyxE19I8n9+iSvm9+oZZquvcgfUxMHn8Gq/b44UbPvjtFOg2yam4xdWXF/RyWCHdc/R9EHorSABeCbefIsm+zcUF3/YQxwpSxM4IZEeH2rTiC7dcrsKw3XsO16xFQz5YI5Bay+CT/wTdMmJd7DdYz7Dyf+pOvcM9WOf/zorxYWSBOMYy0uzbusU2iyIghQ82s7E/Ahg+WARtPgkuTLSB5aL1oCTBKHqQscMr7lo5Ti6RpWLxEdTQMBznc+bMr+6dEtkEcG9zqc6cE9XX+ox3wTU6+HVMfQ1ltCntJ4UKcw3A6INEbw9wgocQa812CIASQ2fE+SCAbz6JxBjIAlFUnD1lUB7S8PdMPwn9plfQgKQ2A5YZqg6FnBdf0rQXIJYxQWKHXj/rBHSUCT0tHACDlzTA+EwWggvkP5AGIxRxm8jhw=
   - KCOV_CMD="./kcov-master/tmp/usr/local/bin/kcov"
@@ -64,7 +65,7 @@ install:
     )
 
 script:
-  - if [ "$RUN_TESTS" = "true" ]; then ./test.sh --verbose; fi
+  - if [ "$RUN_TESTS" = "true" ]; then ./test.sh $TEST_OPTIONS --verbose; fi
   - if [ "$RUN_COVERAGE" = "true" ]; then ./scripts/cov.sh "$KCOV_CMD"; fi
 
 after_success: |

dapps/Cargo.toml

@@ -11,17 +11,17 @@ build = "build.rs"
 [dependencies]
 rand = "0.3.14"
 log = "0.3"
-jsonrpc-core = "2.1"
+jsonrpc-core = "3.0"
 jsonrpc-http-server = { git = "https://github.com/ethcore/jsonrpc-http-server.git" }
 hyper = { default-features = false, git = "https://github.com/ethcore/hyper" }
 unicase = "1.3"
 url = "1.0"
 rustc-serialize = "0.3"
-serde = "0.7.0"
-serde_json = "0.7.0"
-serde_macros = { version = "0.7.0", optional = true }
+serde = "0.8"
+serde_json = "0.8"
+serde_macros = { version = "0.8", optional = true }
 zip = { version = "0.1", default-features = false }
-ethabi = "0.2.1"
+ethabi = "0.2.2"
 linked-hash-map = "0.3"
 ethcore-rpc = { path = "../rpc" }
 ethcore-util = { path = "../util" }
@@ -34,7 +34,7 @@ mime_guess = { version = "1.6.1" }
 clippy = { version = "0.0.85", optional = true}
 
 [build-dependencies]
-serde_codegen = { version = "0.7.0", optional = true }
+serde_codegen = { version = "0.8", optional = true }
 
 [features]
 default = ["serde_codegen", "extra-dapps"]

dapps/src/apps/fetcher.rs

@@ -26,7 +26,7 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicBool};
 use rustc_serialize::hex::FromHex;
 
-use hyper::Control;
+use hyper;
 use hyper::status::StatusCode;
 
 use random_filename;
@@ -85,7 +85,7 @@ impl<R: URLHint> AppFetcher<R> {
 		}
 	}
 
-	pub fn to_handler(&self, path: EndpointPath, control: Control) -> Box<Handler> {
+	pub fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler> {
 		let mut dapps = self.dapps.lock();
 		let app_id = path.app_id.clone();
 
@@ -94,7 +94,7 @@ impl<R: URLHint> AppFetcher<R> {
 			match status {
 				// Just server dapp
 				Some(&mut ContentStatus::Ready(ref endpoint)) => {
-					(None, endpoint.to_handler(path))
+					(None, endpoint.to_async_handler(path, control))
 				},
 				// App is already being fetched
 				Some(&mut ContentStatus::Fetching(_)) => {

dapps/src/endpoint.rs

@@ -16,7 +16,7 @@
 
 //! URL Endpoint traits
 
-use hyper::{server, net};
+use hyper::{self, server, net};
 use std::collections::BTreeMap;
 
 #[derive(Debug, PartialEq, Default, Clone)]
@@ -43,4 +43,8 @@ pub trait Endpoint : Send + Sync {
 	fn info(&self) -> Option<&EndpointInfo> { None }
 
 	fn to_handler(&self, path: EndpointPath) -> Box<Handler>;
+
+	fn to_async_handler(&self, path: EndpointPath, _control: hyper::Control) -> Box<Handler> {
+		self.to_handler(path)
+	}
 }

dapps/src/router/mod.rs

@@ -73,20 +73,20 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
 		// Choose proper handler depending on path / domain
 		let url = extract_url(&req);
 		let endpoint = extract_endpoint(&url);
+		let control = self.control.take().expect("on_request is called only once; control is always defined at start; qed");
 
 		self.handler = match endpoint {
 			// First check special endpoints
 			(ref path, ref endpoint) if self.special.contains_key(endpoint) => {
-				self.special.get(endpoint).unwrap().to_handler(path.clone().unwrap_or_default())
+				self.special.get(endpoint).unwrap().to_async_handler(path.clone().unwrap_or_default(), control)
 			},
 			// Then delegate to dapp
 			(Some(ref path), _) if self.endpoints.contains_key(&path.app_id) => {
-				self.endpoints.get(&path.app_id).unwrap().to_handler(path.clone())
+				self.endpoints.get(&path.app_id).unwrap().to_async_handler(path.clone(), control)
 			},
 			// Try to resolve and fetch the dapp
 			(Some(ref path), _) if self.fetch.contains(&path.app_id) => {
-				let control = self.control.take().expect("on_request is called only once, thus control is always defined.");
-				self.fetch.to_handler(path.clone(), control)
+				self.fetch.to_async_handler(path.clone(), control)
 			},
 			// Redirection to main page (maybe 404 instead?)
 			(Some(ref path), _) if *req.method() == hyper::method::Method::Get => {
@@ -100,7 +100,7 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
 			},
 			// RPC by default
 			_ => {
-				self.special.get(&SpecialEndpoint::Rpc).unwrap().to_handler(EndpointPath::default())
+				self.special.get(&SpecialEndpoint::Rpc).unwrap().to_async_handler(EndpointPath::default(), control)
 			}
 		};
 
@@ -135,7 +135,7 @@ impl<A: Authorization> Router<A> {
 		allowed_hosts: Option<Vec<String>>,
 		) -> Self {
 
-		let handler = special.get(&SpecialEndpoint::Rpc).unwrap().to_handler(EndpointPath::default());
+		let handler = special.get(&SpecialEndpoint::Api).unwrap().to_handler(EndpointPath::default());
 		Router {
 			control: Some(control),
 			main_page: main_page,

dapps/src/rpc.rs

@@ -15,6 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::sync::{Arc, Mutex};
+use hyper;
 use jsonrpc_core::IoHandler;
 use jsonrpc_http_server::{ServerHandler, PanicHandler, AccessControlAllowOrigin};
 use endpoint::{Endpoint, EndpointPath, Handler};
@@ -38,7 +39,17 @@ struct RpcEndpoint {
 
 impl Endpoint for RpcEndpoint {
 	fn to_handler(&self, _path: EndpointPath) -> Box<Handler> {
+		panic!("RPC Endpoint is asynchronous and requires Control object.");
+	}
+
+	fn to_async_handler(&self, _path: EndpointPath, control: hyper::Control) -> Box<Handler> {
 		let panic_handler = PanicHandler { handler: self.panic_handler.clone() };
-		Box::new(ServerHandler::new(self.handler.clone(), self.cors_domain.clone(), self.allowed_hosts.clone(), panic_handler))
+		Box::new(ServerHandler::new(
+				self.handler.clone(),
+				self.cors_domain.clone(),
+				self.allowed_hosts.clone(),
+				panic_handler,
+				control,
+		))
 	}
 }

ethstore/Cargo.toml

@@ -8,9 +8,9 @@ build = "build.rs"
 libc = "0.2.11"
 rand = "0.3.14"
 ethkey = { path = "../ethkey" }
-serde = "0.7"
-serde_json = "0.7"
-serde_macros = { version = "0.7", optional = true }
+serde = "0.8"
+serde_json = "0.8"
+serde_macros = { version = "0.8", optional = true }
 rustc-serialize = "0.3"
 rust-crypto = "0.2.36"
 tiny-keccak = "1.0"
@@ -21,7 +21,7 @@ itertools = "0.4"
 ethcrypto = { path = "../ethcrypto" }
 
 [build-dependencies]
-serde_codegen = { version = "0.7", optional = true }
+serde_codegen = { version = "0.8", optional = true }
 
 [features]
 default = ["serde_codegen"]

ethstore/src/json/crypto.rs

@@ -16,7 +16,6 @@
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer, Error};
 use serde::de::{Visitor, MapVisitor};
-use serde::ser;
 use super::{Cipher, CipherSer, CipherSerParams, Kdf, KdfSer, KdfSerParams, H256};
 
 #[derive(Debug, PartialEq)]
@@ -141,60 +140,26 @@ impl Serialize for Crypto {
 	fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
 		where S: Serializer
 	{
-		serializer.serialize_struct("Crypto", CryptoMapVisitor {
-			value: self,
-			state: 0,
-		})
-	}
-}
-
-struct CryptoMapVisitor<'a> {
-	value: &'a Crypto,
-	state: u8,
-}
-
-impl<'a> ser::MapVisitor for CryptoMapVisitor<'a> {
-	fn visit<S>(&mut self, serializer: &mut S) -> Result<Option<()>, S::Error>
-		where S: Serializer
-	{
-		match self.state {
-			0 => {
-				self.state += 1;
-				match self.value.cipher {
-					Cipher::Aes128Ctr(_) => Ok(Some(try!(serializer.serialize_struct_elt("cipher", &CipherSer::Aes128Ctr)))),
-				}
+		let mut state = try!(serializer.serialize_struct("Crypto", 6));
+		match self.cipher {
+			Cipher::Aes128Ctr(ref params) => {
+				try!(serializer.serialize_struct_elt(&mut state, "cipher", &CipherSer::Aes128Ctr));
+				try!(serializer.serialize_struct_elt(&mut state, "cipherparams", params));
 			},
-			1 => {
-				self.state += 1;
-				match self.value.cipher {
-					Cipher::Aes128Ctr(ref params) => Ok(Some(try!(serializer.serialize_struct_elt("cipherparams", params)))),
-				}
-			},
-			2 => {
-				self.state += 1;
-				Ok(Some(try!(serializer.serialize_struct_elt("ciphertext", &self.value.ciphertext))))
-			},
-			3 => {
-				self.state += 1;
-				match self.value.kdf {
-					Kdf::Pbkdf2(_) => Ok(Some(try!(serializer.serialize_struct_elt("kdf", &KdfSer::Pbkdf2)))),
-					Kdf::Scrypt(_) => Ok(Some(try!(serializer.serialize_struct_elt("kdf", &KdfSer::Scrypt)))),
-				}
-			},
-			4 => {
-				self.state += 1;
-				match self.value.kdf {
-					Kdf::Pbkdf2(ref params) => Ok(Some(try!(serializer.serialize_struct_elt("kdfparams", params)))),
-					Kdf::Scrypt(ref params) => Ok(Some(try!(serializer.serialize_struct_elt("kdfparams", params)))),
-				}
+		}
+		try!(serializer.serialize_struct_elt(&mut state, "ciphertext", &self.ciphertext));
+		match self.kdf {
+			Kdf::Pbkdf2(ref params) => {
+				try!(serializer.serialize_struct_elt(&mut state, "kdf", &KdfSer::Pbkdf2));
+				try!(serializer.serialize_struct_elt(&mut state, "kdfparams", params));
 			},
-			5 => {
-				self.state += 1;
-				Ok(Some(try!(serializer.serialize_struct_elt("mac", &self.value.mac))))
+			Kdf::Scrypt(ref params) => {
+				try!(serializer.serialize_struct_elt(&mut state, "kdf", &KdfSer::Scrypt));
+				try!(serializer.serialize_struct_elt(&mut state, "kdfparams", params));
 			},
-			_ => {
-				Ok(None)
-			}
 		}
+
+		try!(serializer.serialize_struct_elt(&mut state, "mac", &self.mac));
+		serializer.serialize_struct_end(state)
 	}
 }

json/Cargo.toml

@@ -7,13 +7,13 @@ build = "build.rs"
 [dependencies]
 ethcore-util = { path = "../util" }
 rustc-serialize = "0.3"
-serde = "0.7.0"
-serde_json = "0.7.0"
-serde_macros = { version = "0.7.0", optional = true }
+serde = "0.8"
+serde_json = "0.8"
+serde_macros = { version = "0.8", optional = true }
 clippy = { version = "0.0.85", optional = true}
 
 [build-dependencies]
-serde_codegen = { version = "0.7.0", optional = true }
+serde_codegen = { version = "0.8", optional = true }
 
 [features]
 default = ["serde_codegen"]

json/src/blockchain/blockchain.rs

@@ -178,5 +178,5 @@ mod tests {
 		}"#;
 		let _deserialized: BlockChain = serde_json::from_str(s).unwrap();
 		// TODO: validate all fields
-	//}
+	}
 }

json/src/bytes.rs

@@ -56,7 +56,7 @@ impl FromStr for Bytes {
 			2 if value.starts_with("0x") => vec![],
 			_ if value.starts_with("0x") && value.len() % 2 == 1 => {
 				let v = "0".to_owned() + &value[2..];
-				FromHex::from_hex(v.as_ref() as &str).unwrap_or(vec![]),
+				FromHex::from_hex(v.as_str()).unwrap_or(vec![])
 			},
 			_ if value.starts_with("0x") => FromHex::from_hex(&value[2..]).unwrap_or(vec![]),
 			_ => FromHex::from_hex(value).unwrap_or(vec![]),

json/src/vm/call.rs

@@ -39,9 +39,8 @@ pub struct Call {
 mod tests {
 	use serde_json;
 	use vm::Call;
-	use util::U256;
+	use util::{U256, H160 as Hash160};
 	use uint::Uint;
-	use util::hash::Address as Hash160;
 	use hash::Address;
 	use maybe::MaybeEmpty;
 	use std::str::FromStr;

rpc/Cargo.toml

@@ -10,9 +10,9 @@ build = "build.rs"
 
 [dependencies]
 log = "0.3"
-serde = "0.7.0"
-serde_json = "0.7.0"
-jsonrpc-core = "2.1"
+serde = "0.8"
+serde_json = "0.8"
+jsonrpc-core = "3.0"
 jsonrpc-http-server = { git = "https://github.com/ethcore/jsonrpc-http-server.git" }
 ethcore-io = { path = "../util/io" }
 ethcore-util = { path = "../util" }
@@ -25,14 +25,14 @@ ethjson = { path = "../json" }
 ethcore-devtools = { path = "../devtools" }
 rustc-serialize = "0.3"
 transient-hashmap = "0.1"
-serde_macros = { version = "0.7.0", optional = true }
+serde_macros = { version = "0.8.0", optional = true }
 clippy = { version = "0.0.85", optional = true}
 json-ipc-server = { git = "https://github.com/ethcore/json-ipc-server.git" }
 ethcore-ipc = { path = "../ipc/rpc" }
 time = "0.1"
 
 [build-dependencies]
-serde_codegen = { version = "0.7.0", optional = true }
+serde_codegen = { version = "0.8.0", optional = true }
 
 [features]
 default = ["serde_codegen"]

rpc/src/v1/helpers/dispatch.rs

@@ -50,13 +50,13 @@ pub fn dispatch_transaction<C, M>(client: &C, miner: &M, signed_transaction: Sig
 
 	import
 		.map_err(errors::from_transaction_error)
-		.and_then(|_| to_value(&hash))
+		.map(|_| to_value(&hash))
 }
 
 pub fn signature_with_password(accounts: &AccountProvider, address: Address, hash: H256, pass: String) -> Result<Value, Error> {
 	accounts.sign_with_password(address, pass, hash)
 		.map_err(errors::from_password_error)
-		.and_then(|hash| to_value(&RpcH520::from(hash)))
+		.map(|hash| to_value(&RpcH520::from(hash)))
 }
 
 pub fn unlock_sign_and_dispatch<C, M>(client: &C, miner: &M, request: TransactionRequest, account_provider: &AccountProvider, password: String) -> Result<Value, Error>

rpc/src/v1/helpers/errors.rs

@@ -38,7 +38,8 @@ mod codes {
 	pub const ACCOUNT_ERROR: i64 = -32023;
 	pub const SIGNER_DISABLED: i64 = -32030;
 	pub const REQUEST_REJECTED: i64 = -32040;
-	pub const REQUEST_NOT_FOUND: i64 = -32041;
+	pub const REQUEST_REJECTED_LIMIT: i64 = -32041;
+	pub const REQUEST_NOT_FOUND: i64 = -32042;
 	pub const COMPILATION_ERROR: i64 = -32050;
 }
 
@@ -66,6 +67,13 @@ pub fn request_rejected() -> Error {
 	}
 }
 
+pub fn request_rejected_limit() -> Error {
+	Error {
+		code: ErrorCode::ServerError(codes::REQUEST_REJECTED_LIMIT),
+		message: "Request has been rejected because of queue limit.".into(),
+		data: None,
+	}
+}
 
 pub fn account<T: fmt::Debug>(error: &str, details: T) -> Error {
 	Error {