Merge branch 'master' into dp/chore/extract-clique

* master: Extract the Engine trait (#10958) Better error message for rpc gas price errors (#10931) [.gitlab.yml] cargo check ethcore benches (#10965) Verify transaction against its block during import (#10954) [evmbin] fix compilation (#10976) Update to latest trie version. (#10972) [blooms-db] Fix benchmarks (#10974) Fix ethcore/benches build. (#10964) tx-pool: accept local tx with higher gas price when pool full (#10901) Disable unsyncable expanse chain (#10926)
openethereum · Aug 15, 2019 · 42e1d57 · 42e1d57
2 parents c79b7e2 + 6a9de9b
commit 42e1d57
Show file tree

Hide file tree

Showing 71 changed files with 576 additions and 287 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -46,7 +46,7 @@ variables:
         sccache --start-server
     - sccache -s
   after_script:
-    # sccache debug info 
+    # sccache debug info
     - if test -e sccache_debug.log;
       then
         echo "_____All crate-types:_____";
@@ -100,6 +100,13 @@ cargo-check 2 3:
     - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always
     - sccache -s
 
+cargo-check-ethcore-benches:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check -p ethcore --benches --target $CARGO_TARGET --locked --verbose --color=always
+    - sccache -s
+
 cargo-audit:
   stage:                           test
   <<:                              *docker-cache-status
@@ -223,7 +230,7 @@ publish-docker:
     DOCKER_DRIVER:                 overlay2
     GIT_STRATEGY:                  none
     # DOCKERFILE:                  tools/Dockerfile
-    # CONTAINER_IMAGE:             parity/parity 
+    # CONTAINER_IMAGE:             parity/parity
   script:
     # we stopped pushing nightlies to dockerhub, will push to own registry prb.
     - ./tools/publish-docker.sh

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -74,7 +74,7 @@ ethcore-secretstore = { path = "secret-store", optional = true }
 
 registrar = { path = "util/registrar" }
 
-parity-util-mem = { version = "0.1", features = ["jemalloc-global"] }
+parity-util-mem = { version = "0.2.0", features = ["jemalloc-global"] }
 
 [build-dependencies]
 rustc_version = "0.2"

diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml
@@ -38,7 +38,8 @@ ethkey = { path = "../accounts/ethkey" }
 evm = { path = "evm" }
 executive-state = { path = "executive-state" }
 futures = "0.1"
-hash-db = "0.12.4"
+hash-db = "0.15.0"
+parity-util-mem = "0.2.0"
 instant-seal = { path = "./engines/instant-seal" }
 itertools = "0.5"
 journaldb = { path = "../util/journaldb" }
@@ -57,8 +58,8 @@ null-engine = { path = "./engines/null-engine" }
 num_cpus = "1.2"
 parity-bytes = "0.1"
 parity-snappy = "0.1"
-parity-util-mem = "0.1"
 parking_lot = "0.8"
+trie-db = "0.15.0"
 patricia-trie-ethereum = { path = "../util/patricia-trie-ethereum" }
 pod = { path = "pod" }
 rand = "0.6"
@@ -75,7 +76,6 @@ tempdir = { version = "0.3", optional = true }
 time-utils = { path = "../util/time-utils" }
 trace = { path = "trace" }
 trace-time = "0.1"
-trie-db = "0.12.4"
 trie-vm-factories = { path = "trie-vm-factories" }
 triehash-ethereum = { version = "0.2",  path = "../util/triehash-ethereum" }
 unexpected = { path = "../util/unexpected" }
@@ -90,13 +90,12 @@ env_logger = "0.5"
 ethcore-accounts = { path = "../accounts" }
 fetch = { path = "../util/fetch" }
 kvdb-rocksdb = "0.1.3"
-macros = { path = "../util/macros" }
 machine = { path = "./machine", features = ["test-helpers"] }
 parity-runtime = { path = "../util/runtime" }
 rlp_compress = { path = "../util/rlp-compress" }
 serde_json = "1.0"
 tempdir = "0.3"
-trie-standardmap = "0.12.4"
+trie-standardmap = "0.15.0"
 
 [features]
 parity = ["work-notify", "price-info", "stratum"]
@@ -120,7 +119,7 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 # EVM debug traces are printed.
 slow-blocks = []
 # Run JSON consensus tests.
-json-tests = ["env_logger", "test-helpers", "macros"]
+json-tests = ["env_logger", "test-helpers", "machine/test-helpers"]
 # Skip JSON consensus tests with pending issues.
 ci-skip-tests = []
 # Run memory/cpu heavy tests.

diff --git a/ethcore/account-db/Cargo.toml b/ethcore/account-db/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2018"
 
 [dependencies]
 ethereum-types = "0.6"
-hash-db = "0.12.4"
+hash-db = "0.15.0"
 keccak-hash = "0.2.0"
 keccak-hasher = { path = "../../util/keccak-hasher" }
 kvdb = "0.1"

diff --git a/ethcore/account-state/Cargo.toml b/ethcore/account-state/Cargo.toml
@@ -12,22 +12,22 @@ derive_more = "0.15.0"
 ethereum-types = "0.6.0"
 ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" }
 trie-vm-factories = { path = "../trie-vm-factories" }
-hash-db = "0.12.4"
+hash-db = "0.15.0"
 journaldb = { path = "../../util/journaldb" }
 keccak-hash = "0.2.0"
 keccak-hasher = { path = "../../util/keccak-hasher" }
 kvdb = "0.1.0"
 log = "0.4"
 lru-cache = "0.1.2"
-memory-db = "0.12.4"
+memory-db = "0.15.0"
 parity-bytes = "0.1.0"
-parity-util-mem = "0.1.0"
+parity-util-mem = "0.2.0"
 parking_lot = "0.8.0"
 pod = { path = "../pod" }
 rlp = "0.4.0"
 serde = { version = "1.0", features = ["derive"] }
 trace = { path = "../trace" }
-trie-db = "0.12.4"
+trie-db = "0.15.0"
 
 [dev-dependencies]
 account-db = { path = "../account-db" }

diff --git a/ethcore/benches/builtin.rs b/ethcore/benches/builtin.rs
@@ -20,17 +20,19 @@ extern crate criterion;
 #[macro_use]
 extern crate lazy_static;
 
+extern crate machine;
 extern crate ethcore;
+extern crate ethcore_builtin;
 extern crate ethereum_types;
 extern crate parity_bytes as bytes;
 extern crate rustc_hex;
 
 use criterion::{Criterion, Bencher};
 use bytes::BytesRef;
-use ethcore::builtin::Builtin;
-use ethcore::machine::Machine;
+use ethcore_builtin::Builtin;
 use ethereum_types::H160;
-use ethcore::ethereum::new_byzantium_test_machine;
+use machine::Machine;
+use machine::test_helpers::new_byzantium_test_machine;
 use rustc_hex::FromHex;
 
 lazy_static! {

diff --git a/ethcore/blockchain/Cargo.toml b/ethcore/blockchain/Cargo.toml
@@ -14,7 +14,7 @@ common-types = { path = "../types" }
 ethcore-db = { path = "../db" }
 ethereum-types = "0.6.0"
 keccak-hash = "0.2.0"
-parity-util-mem = "0.1"
+parity-util-mem = "0.2.0"
 itertools = "0.5"
 kvdb = "0.1"
 log = "0.4"

diff --git a/ethcore/db/Cargo.toml b/ethcore/db/Cargo.toml
@@ -11,7 +11,7 @@ edition = "2018"
 common-types = { path = "../types" }
 ethereum-types = "0.6.0"
 kvdb = "0.1"
-parity-util-mem = "0.1"
+parity-util-mem = "0.2.0"
 parking_lot = "0.8"
 rlp = "0.4.0"
 rlp_derive = { path = "../../util/rlp-derive" }
diff --git a/ethcore/engine/src/snapshot.rs b/ethcore/engine/src/snapshot.rs
@@ -0,0 +1,90 @@
+// Copyright 2015-2019 Parity Technologies (UK) Ltd.
+// This file is part of Parity Ethereum.
+
+// Parity Ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This module contains traits used while creating/restoring snapshots. They
+//! are here because they use and are used by the Engine trait itself.
+
+use std::sync::{Arc, atomic::AtomicBool};
+
+use ethereum_types::H256;
+
+use blockchain::{BlockChain, BlockChainDB};
+use common_types::{
+	errors::{EthcoreError as Error, SnapshotError},
+	snapshot::{ManifestData, ChunkSink, Progress},
+};
+
+use crate::engine::Engine;
+
+/// Restore from secondary snapshot chunks.
+pub trait Rebuilder: Send {
+	/// Feed a chunk, potentially out of order.
+	///
+	/// Check `abort_flag` periodically while doing heavy work. If set to `false`, should bail with
+	/// `Error::RestorationAborted`.
+	fn feed(
+		&mut self,
+		chunk: &[u8],
+		engine: &dyn Engine,
+		abort_flag: &AtomicBool,
+	) -> Result<(), Error>;
+
+	/// Finalize the restoration. Will be done after all chunks have been
+	/// fed successfully.
+	///
+	/// This should apply the necessary "glue" between chunks,
+	/// and verify against the restored state.
+	fn finalize(&mut self) -> Result<(), Error>;
+}
+
+/// Components necessary for snapshot creation and restoration.
+pub trait SnapshotComponents: Send {
+	/// Create secondary snapshot chunks; these corroborate the state data
+	/// in the state chunks.
+	///
+	/// Chunks shouldn't exceed the given preferred size, and should be fed
+	/// uncompressed into the sink.
+	///
+	/// This will vary by consensus engine, so it's exposed as a trait.
+	fn chunk_all(
+		&mut self,
+		chain: &BlockChain,
+		block_at: H256,
+		chunk_sink: &mut ChunkSink,
+		progress: &Progress,
+		preferred_size: usize,
+	) -> Result<(), SnapshotError>;
+
+	/// Create a rebuilder, which will have chunks fed into it in arbitrary
+	/// order and then be finalized.
+	///
+	/// The manifest, a database, and fresh `BlockChain` are supplied.
+	///
+	/// The engine passed to the `Rebuilder` methods will be the same instance
+	/// that created the `SnapshotComponents`.
+	fn rebuilder(
+		&self,
+		chain: BlockChain,
+		db: Arc<dyn BlockChainDB>,
+		manifest: &ManifestData,
+	) -> Result<Box<dyn Rebuilder>, Error>;
+
+	/// Minimum supported snapshot version number.
+	fn min_supported_version(&self) -> u64;
+
+	/// Current version number
+	fn current_version(&self) -> u64;
+}
diff --git a/ethcore/engines/validator-set/Cargo.toml b/ethcore/engines/validator-set/Cargo.toml
@@ -23,7 +23,7 @@ log = "0.4.8"
 machine = { path = "../../machine" }
 memory-cache = { path = "../../../util/memory-cache" }
 parity-bytes = "0.1.0"
-parity-util-mem = "0.1"
+parity-util-mem = "0.2.0"
 parking_lot = "0.8"
 rlp = "0.4.2"
 triehash = { package = "triehash-ethereum", version = "0.2",  path = "../../../util/triehash-ethereum" }

diff --git a/ethcore/evm/Cargo.toml b/ethcore/evm/Cargo.toml
@@ -8,7 +8,7 @@ authors = ["Parity Technologies <[email protected]>"]
 bit-set = "0.4"
 parity-bytes = "0.1"
 ethereum-types = "0.6.0"
-parity-util-mem = "0.1"
+parity-util-mem = "0.2.0"
 lazy_static = "1.0"
 log = "0.4"
 vm = { path = "../vm" }

diff --git a/ethcore/executive-state/Cargo.toml b/ethcore/executive-state/Cargo.toml
@@ -12,7 +12,7 @@ account-state = { path = "../account-state" }
 bytes = { package = "parity-bytes", version = "0.1.0" }
 common-types = { path = "../types" }
 ethereum-types = "0.6.0"
-hash-db = "0.12.4"
+hash-db = "0.15.0"
 keccak-hasher = { path = "../../util/keccak-hasher" }
 kvdb = "0.1.0"
 log = "0.4.8"

diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml
@@ -19,14 +19,14 @@ ethcore-blockchain = { path = "../blockchain" }
 ethereum-types = "0.6.0"
 executive-state = { path = "../executive-state" }
 machine = { path = "../machine" }
-memory-db = "0.12.4"
-trie-db = "0.12.4"
+memory-db = "0.15.0"
+trie-db = "0.15.0"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 ethcore-network = { path = "../../util/network" }
 ethcore-miner = { path = "../../miner" }
 ethcore-io = { path = "../../util/io" }
-hash-db = "0.12.4"
-parity-util-mem = "0.1"
+hash-db = "0.15.0"
+parity-util-mem = "0.2.0"
 vm = { path = "../vm" }
 fastmap = { path = "../../util/fastmap" }
 failsafe = { version = "0.3.0", default-features = false, features = ["parking_lot_mutex"] }

diff --git a/ethcore/pod/Cargo.toml b/ethcore/pod/Cargo.toml
@@ -11,7 +11,7 @@ common-types = { path = "../types" }
 ethereum-types = "0.6"
 ethjson = { path = "../../json" }
 ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" }
-hash-db = "0.12"
+hash-db = "0.15.0"
 itertools = "0.8"
 keccak-hash = "0.2.0"
 keccak-hasher = { path = "../../util/keccak-hasher" }
@@ -21,7 +21,7 @@ parity-bytes = "0.1.0"
 rlp = "0.4"
 rustc-hex = "1"
 serde = { version = "1.0", features = ["derive"] }
-trie-db = "0.12.4"
+trie-db = "0.15.0"
 triehash = { package = "triehash-ethereum", version = "0.2",  path = "../../util/triehash-ethereum" }
 
 [dev-dependencies]

diff --git a/ethcore/pod/src/account.rs b/ethcore/pod/src/account.rs
@@ -26,7 +26,7 @@ use keccak_hasher::KeccakHasher;
 use triehash::sec_trie_root;
 use parity_bytes::Bytes;
 use trie_db::TrieFactory;
-use ethtrie::RlpCodec;
+use ethtrie::Layout;
 use ethjson;
 use common_types::account_diff::*;
 use rlp::{self, RlpStream};
@@ -70,7 +70,7 @@ impl PodAccount {
 	}
 
 	/// Place additional data into given hash DB.
-	pub fn insert_additional(&self, db: &mut dyn HashDB<KeccakHasher, DBValue>, factory: &TrieFactory<KeccakHasher, RlpCodec>) {
+	pub fn insert_additional(&self, db: &mut dyn HashDB<KeccakHasher, DBValue>, factory: &TrieFactory<Layout>) {
 		match self.code {
 			Some(ref c) if !c.is_empty() => { db.insert(hash_db::EMPTY_PREFIX, c); }
 			_ => {}

diff --git a/ethcore/private-tx/Cargo.toml b/ethcore/private-tx/Cargo.toml
@@ -22,14 +22,14 @@ ethjson = { path = "../../json" }
 ethkey = { path = "../../accounts/ethkey" }
 fetch = { path = "../../util/fetch" }
 futures = "0.1"
-parity-util-mem = "0.1"
+parity-util-mem = "0.2.0"
 keccak-hash = "0.2.0"
 log = "0.4"
 machine = { path = "../machine" }
 parity-bytes = "0.1"
 parity-crypto = "0.4.0"
 parking_lot = "0.8"
-trie-db = "0.12.4"
+trie-db = "0.15.0"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 rand = "0.3"
 rlp = "0.4.0"

diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs
@@ -29,9 +29,12 @@ use blockchain::{BlockChainDB, BlockChainDBHandler};
 use ethcore::client::{Client, ClientConfig, ChainNotify, ClientIoMessage};
 use ethcore::miner::Miner;
 use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams};
-use ethcore::snapshot::{SnapshotService as _SnapshotService, RestorationStatus};
+use ethcore::snapshot::{SnapshotService as _SnapshotService};
 use ethcore::spec::Spec;
-use common_types::errors::{EthcoreError, SnapshotError};
+use common_types::{
+	errors::{EthcoreError, SnapshotError},
+	snapshot::RestorationStatus,
+};
 
 
 use ethcore_private_tx::{self, Importer, Signer};

diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs
@@ -97,7 +97,7 @@ use verification::queue::kind::BlockLike;
 use verification::{Verifier, BlockQueue};
 use verification;
 use ansi_term::Colour;
-
+use ethtrie::Layout;
 // re-export
 pub use blockchain::CacheSize as BlockChainCacheSize;
 use db::{Writable, Readable, keys::BlockDetails};
@@ -729,7 +729,7 @@ impl Client {
 			false => TrieSpec::Secure,
 		};
 
-		let trie_factory = TrieFactory::new(trie_spec);
+		let trie_factory = TrieFactory::new(trie_spec, Layout);
 		let factories = Factories {
 			vm: VmFactory::new(config.vm_type.clone(), config.jump_table_size),
 			trie: trie_factory,