Add constantinople conf to EvmTestClient.

Cargo.toml

@@ -89,6 +89,7 @@ daemonize = { git = "https://github.com/paritytech/daemonize" }
 [features]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
+ci-skip-issue = ["ethcore/ci-skip-issue"]
 test-heavy = ["ethcore/test-heavy"]
 evm-debug = ["ethcore/evm-debug"]
 evm-debug-tests = ["ethcore/evm-debug-tests"]

ethcore/Cargo.toml

@@ -102,6 +102,8 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 slow-blocks = []
 # Run JSON consensus tests.
 json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"]
+# Skip JSON consensus tests with pending issues.
+ci-skip-issue = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile benches

ethcore/res/ethereum/constantinople_test.json

@@ -6,7 +6,10 @@
 				"minimumDifficulty": "0x020000",
 				"difficultyBoundDivisor": "0x0800",
 				"durationLimit": "0x0d",
-				"blockReward": "0x1BC16D674EC80000",
+				"blockReward": {
+					"0": "0x29A2241AF62C0000",
+					"5": "0x1BC16D674EC80000"
+				},
 				"homesteadTransition": "0x0",
 				"eip100bTransition": "0x0",
 				"difficultyBombDelays": {
@@ -58,9 +61,9 @@
 		"0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
 		"0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
 		"0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
-		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 100 } } } },
+		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 20 } } } },
 		"0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } },
-		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 2000, "word": 0 } } } },
+		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 40000, "word": 0 } } } },
 		"0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } }
 	}
 }

ethcore/src/client/evm_test_client.rs

@@ -86,9 +86,9 @@ impl<'a> EvmTestClient<'a> {
 			ForkSpec::EIP150 => Some(ethereum::new_eip150_test()),
 			ForkSpec::EIP158 => Some(ethereum::new_eip161_test()),
 			ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()),
+			ForkSpec::Constantinople => Some(ethereum::new_constantinople_test()),
 			ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()),
 			ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None,
-			_ => None,
 		}
 	}
 

ethcore/src/json_tests/chain.rs

@@ -23,6 +23,7 @@ use miner::Miner;
 use io::IoChannel;
 use test_helpers;
 use verification::queue::kind::blocks::Unverified;
+use super::SKIP_TEST_STATE;
 
 use super::HookType;
 
@@ -36,12 +37,20 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
 	::json_tests::test_common::run_test_file(p, json_chain_test, h)
 }
 
+fn skip_test(name: &String) -> bool {
+	SKIP_TEST_STATE.block.iter().any(|block_test|block_test.subtests.contains(name))
+}
+
 pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::blockchain::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, blockchain) in tests.into_iter() {
+		if skip_test(&name) {
+			println!("   - {} | {:?} Ignoring tests because in skip list", name, blockchain.network);
+			continue;
+		}
 		start_stop_hook(&name, HookType::OnStart);
 
 		let mut fail = false;
@@ -122,19 +131,24 @@ mod block_tests {
 	declare_test!{BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"}
 	declare_test!{BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"}
 	declare_test!{BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"}
+	declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"}
 	declare_test!{BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"}
 	declare_test!{BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"}
 	declare_test!{BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"}
 	declare_test!{BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"}
 	declare_test!{BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"}
 
+	declare_test!{BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"}
@@ -149,12 +163,15 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"}
 	declare_test!{heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"}
@@ -166,6 +183,7 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"}
 
 	declare_test!{BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"}
 	declare_test!{BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"}

ethcore/src/json_tests/mod.rs

@@ -24,10 +24,12 @@ mod executive;
 mod state;
 mod chain;
 mod trie;
+mod skip;
 
 #[cfg(test)]
 mod difficulty;
 
+
 pub use self::test_common::HookType;
 
 pub use self::transaction::run_test_path as run_transaction_test_path;
@@ -42,3 +44,4 @@ pub use self::trie::run_generic_test_path as run_generic_trie_test_path;
 pub use self::trie::run_generic_test_file as run_generic_trie_test_file;
 pub use self::trie::run_secure_test_path as run_secure_trie_test_path;
 pub use self::trie::run_secure_test_file as run_secure_trie_test_file;
+use self::skip::SKIP_TEST_STATE;

ethcore/src/json_tests/skip.rs

@@ -0,0 +1,37 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State tests to skip.
+
+use ethjson;
+
+#[cfg(all(not(test), feature = "ci-skip-tests"))]
+compile_error!("ci-skip-tests can only be enabled for testing builds.");
+
+#[cfg(feature="ci-skip-issue")]
+lazy_static!{
+	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
+		let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json");
+		ethjson::test::SkipStates::load(&skip_data[..]).expect("No invalid json allowed")
+	};
+}
+
+#[cfg(not(feature="ci-skip-issue"))]
+lazy_static!{
+	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
+		ethjson::test::SkipStates::empty()
+	};
+}

ethcore/src/json_tests/state.rs

@@ -22,7 +22,7 @@ use client::{EvmTestClient, EvmTestError, TransactResult};
 use ethjson;
 use transaction::SignedTransaction;
 use vm::EnvInfo;
-
+use super::SKIP_TEST_STATE;
 use super::HookType;
 
 /// Run state jsontests on a given folder.
@@ -35,6 +35,18 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
 	::json_tests::test_common::run_test_file(p, json_chain_test, h)
 }
 
+fn skip_test(subname: &str, chain: &String, number: usize) -> bool {
+	SKIP_TEST_STATE.state.iter().any(|state_test|{
+		if let Some(subtest) = state_test.subtests.get(subname) {
+			chain == &subtest.chain &&
+			(subtest.subnumbers[0] == "*"
+				 || subtest.subnumbers.contains(&number.to_string()))
+		} else {
+			false
+		}
+	})
+}
+
 pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::state::test::Test::load(json_data).unwrap();
@@ -60,6 +72,10 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 
 				for (i, state) in states.into_iter().enumerate() {
 					let info = format!("   - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total);
+					if skip_test(&name, &spec.name, i + 1) {
+						println!("{} in skip list : SKIPPED", info);
+						continue;
+					}
 
 					let post_root: H256 = state.hash.into();
 					let transaction: SignedTransaction = multitransaction.select(&state.indexes).into();
@@ -125,11 +141,13 @@ mod state_tests {
 	declare_test!{GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"}
 	declare_test!{GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"}
 	declare_test!{GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"}
+	declare_test!{GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"}
 	declare_test!{GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"}
 	declare_test!{GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"}
 	declare_test!{GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"}
 	declare_test!{GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"}
 	declare_test!{GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"}
+	declare_test!{GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"}
 	declare_test!{GeneralStateTest_stExample, "GeneralStateTests/stExample/"}
 	declare_test!{GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"}
 	declare_test!{GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"}

ethcore/src/json_tests/test_common.rs

@@ -34,6 +34,18 @@ pub fn run_test_path<H: FnMut(&str, HookType)>(
 	p: &Path, skip: &[&'static str],
 	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
 	start_stop_hook: &mut H
+) {
+	let mut errors = Vec::new();
+	run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors);
+	let empty: [String; 0] = [];
+	assert_eq!(errors, empty);
+}
+
+fn run_test_path_inner<H: FnMut(&str, HookType)>(
+	p: &Path, skip: &[&'static str],
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H,
+	errors: &mut Vec<String>
 ) {
 	let path = Path::new(p);
 	let s: HashSet<OsString> = skip.iter().map(|s| {
@@ -50,17 +62,32 @@ pub fn run_test_path<H: FnMut(&str, HookType)>(
 			} else {
 				Some(e.path())
 			}}) {
-			run_test_path(&p, skip, runner, start_stop_hook)
+			run_test_path_inner(&p, skip, runner, start_stop_hook, errors);
 		}
 	} else if extension == Some("swp") || extension == None {
 		// Ignore junk
 	} else {
 		let mut path = p.to_path_buf();
 		path.set_extension("json");
-		run_test_file(&path, runner, start_stop_hook)
+		run_test_file_append(&path, runner, start_stop_hook, errors)
 	}
 }
 
+fn run_test_file_append<H: FnMut(&str, HookType)>(
+	path: &Path,
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H,
+	errors: &mut Vec<String>
+) {
+	let mut data = Vec::new();
+	let mut file = match File::open(&path) {
+		Ok(file) => file,
+		Err(_) => panic!("Error opening test file at: {:?}", path),
+	};
+	file.read_to_end(&mut data).expect("Error reading test file");
+	errors.append(&mut runner(&data, start_stop_hook));
+}
+
 pub fn run_test_file<H: FnMut(&str, HookType)>(
 	path: &Path,
 	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,

json/src/test/mod.rs

@@ -45,6 +45,7 @@ pub struct DifficultyTestCase {
 	#[serde(rename="currentBlockNumber")]
 	pub current_block_number: Uint,
 }
+
 /// Blockchain test deserializer.
 #[derive(Debug, PartialEq, Deserialize)]
 pub struct DifficultyTest(BTreeMap<String, DifficultyTestCase>);
@@ -64,3 +65,59 @@ impl DifficultyTest {
 		serde_json::from_reader(reader)
 	}
 }
+
+/// Test to skip (only if issue ongoing)
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct SkipStates {
+	/// Block tests
+	pub block: Vec<BlockSkipStates>,
+	/// State tests
+	pub state: Vec<StateSkipStates>,
+
+}
+
+/// Block test to skip.
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct BlockSkipStates {
+	/// Issue reference.
+	pub reference: String,
+	/// Test failing name.
+	pub failing: String,
+	/// Items failing for the test.
+	pub subtests: Vec<String>,
+}
+
+/// State test to skip.
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct StateSkipStates {
+	/// Issue reference.
+	pub reference: String,
+	/// Test failing name.
+	pub failing: String,
+	/// Items failing for the test.
+	pub subtests: BTreeMap<String, StateSkipSubStates>
+}
+
+/// State subtest to skip.
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct StateSkipSubStates {
+	/// State test number of this item. Or '*' for all state.
+	pub subnumbers: Vec<String>,
+	/// Chain for this items.
+	pub chain: String,
+}
+
+impl SkipStates {
+	/// Loads skip states from json.
+	pub fn load<R>(reader: R) -> Result<Self, Error> where R: Read {
+		serde_json::from_reader(reader)
+	}
+
+	/// Empty skip states.
+	pub fn empty() -> Self {
+		SkipStates {
+			block: Vec::new(),
+			state: Vec::new(),
+		}
+	}
+}

test.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Running Parity Full Test Suite
 
-FEATURES="json-tests"
+FEATURES="json-tests,ci-skip-issue"
 OPTIONS="--release"
 VALIDATE=1