diff --git a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/BlockMetadata.java b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/BlockMetadata.java deleted file mode 100644 index 026ad5cf79..0000000000 --- a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/BlockMetadata.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.core; - -import tech.pegasys.pantheon.ethereum.rlp.RLP; -import tech.pegasys.pantheon.ethereum.rlp.RLPException; -import tech.pegasys.pantheon.ethereum.rlp.RLPInput; -import tech.pegasys.pantheon.ethereum.rlp.RLPOutput; -import tech.pegasys.pantheon.util.bytes.BytesValue; -import tech.pegasys.pantheon.util.uint.UInt256; - -public class BlockMetadata { - private static final BlockMetadata EMPTY = new BlockMetadata(null); - private final UInt256 totalDifficulty; - - public BlockMetadata(final UInt256 totalDifficulty) { - this.totalDifficulty = totalDifficulty; - } - - public static BlockMetadata empty() { - return EMPTY; - } - - public static BlockMetadata fromRlp(final BytesValue bytes) { - return readFrom(RLP.input(bytes)); - } - - public static BlockMetadata readFrom(final RLPInput in) throws RLPException { - in.enterList(); - - final UInt256 totalDifficulty = in.readUInt256Scalar(); - - in.leaveList(); - - return new BlockMetadata(totalDifficulty); - } - - public UInt256 getTotalDifficulty() { - return totalDifficulty; - } - - public BytesValue toRlp() { - return RLP.encode(this::writeTo); - } - - public void writeTo(final RLPOutput out) { - out.startList(); - - out.writeUInt256Scalar(totalDifficulty); - - out.endList(); - } -} diff --git a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/TransactionBuilder.java b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/TransactionBuilder.java deleted file mode 100644 index 888327e7cc..0000000000 --- a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/TransactionBuilder.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.core; - -import tech.pegasys.pantheon.crypto.SECP256K1; -import tech.pegasys.pantheon.crypto.SECP256K1.Signature; -import tech.pegasys.pantheon.ethereum.rlp.RLPInput; -import tech.pegasys.pantheon.util.bytes.BytesValue; - -/** Convenience object for building {@link Transaction}s. */ -public interface TransactionBuilder { - - /** @return A {@link Transaction} populated with the accumulated state. */ - Transaction build(); - - /** - * Constructs a {@link SECP256K1.Signature} based on the accumulated state and then builds a - * corresponding {@link Transaction}. - * - * @param keys The keys to construct the transaction signature with. - * @return A {@link Transaction} populated with the accumulated state. - */ - Transaction signAndBuild(SECP256K1.KeyPair keys); - - /** - * Populates the {@link TransactionBuilder} based on the RLP-encoded transaction and builds a - * {@link Transaction}. - * - *

Note: the fields from the RLP-transaction will be extracted and replace any previously - * populated fields. - * - * @param in The RLP-encoded transaction. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder populateFrom(RLPInput in); - - /** - * Sets the chain id for the {@link Transaction}. - * - * @param chainId The chain id. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder chainId(int chainId); - - /** - * Sets the gas limit for the {@link Transaction}. - * - * @param gasLimit The gas limit. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder gasLimit(long gasLimit); - - /** - * Sets the gas price for the {@link Transaction}. - * - * @param gasPrice The gas price. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder gasPrice(Wei gasPrice); - - /** - * Sets the nonce for the {@link Transaction}. - * - * @param nonce The nonce. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder nonce(long nonce); - - /** - * Sets the payload for the {@link Transaction}. - * - * @param payload The payload. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder payload(BytesValue payload); - - /** - * Sets the sender of the {@link Transaction}. - * - * @param sender The sender. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder sender(Address sender); - - /** - * Sets the signature of the {@link Transaction}. - * - * @param signature The signature. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder signature(Signature signature); - - /** - * Sets the recipient of the {@link Transaction}. - * - * @param to The recipent (can be null). - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder to(Address to); - - /** - * Sets the {@link Wei} transfer value of the {@link Transaction}. - * - * @param value The transfer value. - * @return The updated {@link TransactionBuilder}. - */ - TransactionBuilder value(Wei value); -} diff --git a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/Wei.java b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/Wei.java index 3df4f9e164..c2b5e6b392 100644 --- a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/Wei.java +++ b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/Wei.java @@ -64,10 +64,6 @@ public static Wei fromEth(final long eth) { return Wei.of(BigInteger.valueOf(eth).multiply(BigInteger.TEN.pow(18))); } - public static Counter newCounter() { - return new WeiCounter(); - } - private static class WeiCounter extends Counter { private WeiCounter() { super(Wei::new); diff --git a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetContractCreationProcessor.java b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetContractCreationProcessor.java index 517fef1ad6..12987dadd8 100644 --- a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetContractCreationProcessor.java +++ b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetContractCreationProcessor.java @@ -75,10 +75,6 @@ private static boolean accountExists(final Account account) { return account.getNonce() > 0 || !account.getCode().isEmpty(); } - protected GasCalculator gasCalculator() { - return gasCalculator; - } - @Override public void start(final MessageFrame frame) { if (LOG.isTraceEnabled()) { diff --git a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetTransactionValidator.java b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetTransactionValidator.java index 4c6f608937..d982562e61 100644 --- a/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetTransactionValidator.java +++ b/ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/MainnetTransactionValidator.java @@ -38,10 +38,6 @@ */ public class MainnetTransactionValidator implements TransactionValidator { - public static MainnetTransactionValidator create() { - return new MainnetTransactionValidator(new FrontierGasCalculator(), false, Optional.empty()); - } - private final GasCalculator gasCalculator; private final boolean disallowSignatureMalleability; diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/EthereumWireProtocolConfiguration.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/EthereumWireProtocolConfiguration.java index 822cbea274..dd6aa1ef5f 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/EthereumWireProtocolConfiguration.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/EthereumWireProtocolConfiguration.java @@ -14,6 +14,9 @@ import tech.pegasys.pantheon.util.number.PositiveNumber; +import java.util.Objects; + +import com.google.common.base.MoreObjects; import picocli.CommandLine; public class EthereumWireProtocolConfiguration { @@ -68,27 +71,33 @@ public int getMaxGetNodeData() { } @Override - public boolean equals(final Object obj) { - if (!(obj instanceof EthereumWireProtocolConfiguration)) { + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { return false; } - EthereumWireProtocolConfiguration other = ((EthereumWireProtocolConfiguration) obj); - return maxGetBlockHeaders == other.maxGetBlockHeaders - && maxGetBlockBodies == other.maxGetBlockBodies - && maxGetReceipts == other.maxGetReceipts - && maxGetNodeData == other.maxGetNodeData; + final EthereumWireProtocolConfiguration that = (EthereumWireProtocolConfiguration) o; + return maxGetBlockHeaders == that.maxGetBlockHeaders + && maxGetBlockBodies == that.maxGetBlockBodies + && maxGetReceipts == that.maxGetReceipts + && maxGetNodeData == that.maxGetNodeData; } @Override public int hashCode() { - return super.hashCode(); + return Objects.hash(maxGetBlockHeaders, maxGetBlockBodies, maxGetReceipts, maxGetNodeData); } @Override public String toString() { - return String.format( - "maxGetBlockHeaders=%s\tmaxGetBlockBodies=%s\tmaxGetReceipts=%s\tmaxGetReceipts=%s", - maxGetBlockHeaders, maxGetBlockBodies, maxGetReceipts, maxGetNodeData); + return MoreObjects.toStringHelper(this) + .add("maxGetBlockHeaders", maxGetBlockHeaders) + .add("maxGetBlockBodies", maxGetBlockBodies) + .add("maxGetReceipts", maxGetReceipts) + .add("maxGetNodeData", maxGetNodeData) + .toString(); } public static class Builder { diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContext.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContext.java index fb67ae6e42..644ea94001 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContext.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContext.java @@ -14,26 +14,17 @@ public class EthContext { - private final String protocolName; private final EthPeers ethPeers; private final EthMessages ethMessages; private final EthScheduler scheduler; public EthContext( - final String protocolName, - final EthPeers ethPeers, - final EthMessages ethMessages, - final EthScheduler scheduler) { - this.protocolName = protocolName; + final EthPeers ethPeers, final EthMessages ethMessages, final EthScheduler scheduler) { this.ethPeers = ethPeers; this.ethMessages = ethMessages; this.scheduler = scheduler; } - public String getProtocolName() { - return protocolName; - } - public EthPeers getEthPeers() { return ethPeers; } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthMessages.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthMessages.java index f0583eb120..be5d6bf04f 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthMessages.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthMessages.java @@ -15,7 +15,6 @@ import tech.pegasys.pantheon.util.Subscribers; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; public class EthMessages { @@ -31,18 +30,8 @@ void dispatch(final EthMessage message) { listeners.forEach(callback -> callback.exec(message)); } - public long subscribe(final int messageCode, final MessageCallback callback) { - return listenersByCode - .computeIfAbsent(messageCode, key -> new Subscribers<>()) - .subscribe(callback); - } - - public void unsubscribe(final long listenerId) { - for (final Entry> entry : listenersByCode.entrySet()) { - if (entry.getValue().unsubscribe(listenerId)) { - break; - } - } + public void subscribe(final int messageCode, final MessageCallback callback) { + listenersByCode.computeIfAbsent(messageCode, key -> new Subscribers<>()).subscribe(callback); } @FunctionalInterface diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index 7691f94ef7..670469dfd9 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -27,7 +27,6 @@ import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection; import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection.PeerNotConnected; import tech.pegasys.pantheon.ethereum.p2p.wire.messages.DisconnectMessage.DisconnectReason; -import tech.pegasys.pantheon.util.Subscribers; import tech.pegasys.pantheon.util.bytes.BytesValue; import tech.pegasys.pantheon.util.uint.UInt256; @@ -63,7 +62,6 @@ public class EthPeer { private final AtomicReference> onStatusesExchanged = new AtomicReference<>(); private final PeerReputation reputation = new PeerReputation(); - private final Subscribers disconnectCallbacks = new Subscribers<>(); EthPeer( final PeerConnection connection, @@ -110,14 +108,6 @@ public void disconnect(final DisconnectReason reason) { connection.disconnect(reason); } - public long subscribeDisconnect(final DisconnectCallback callback) { - return disconnectCallbacks.subscribe(callback); - } - - public void unsubscribeDisconnect(final long id) { - disconnectCallbacks.unsubscribe(id); - } - public ResponseStream send(final MessageData messageData) throws PeerNotConnected { switch (messageData.getCode()) { case EthPV62.GET_BLOCK_HEADERS: @@ -258,7 +248,6 @@ void handleDisconnect() { bodiesRequestManager.close(); receiptsRequestManager.close(); nodeDataRequestManager.close(); - disconnectCallbacks.forEach(callback -> callback.onDisconnect(this)); } public void registerKnownBlock(final Hash hash) { diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeers.java index 3b48745013..b33bec9ab2 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeers.java @@ -73,18 +73,14 @@ public void unsubscribeConnect(final long id) { connectCallbacks.unsubscribe(id); } - public long subscribeDisconnect(final DisconnectCallback callback) { - return disconnectCallbacks.subscribe(callback); + public void subscribeDisconnect(final DisconnectCallback callback) { + disconnectCallbacks.subscribe(callback); } public int peerCount() { return connections.size(); } - public int availablePeerCount() { - return (int) availablePeers().count(); - } - public Stream availablePeers() { return connections.values().stream().filter(EthPeer::readyForRequests); } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java index da04086647..201a606cfb 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java @@ -82,7 +82,7 @@ public EthProtocolManager( ethPeers = new EthPeers(getSupportedProtocol()); ethMessages = new EthMessages(); - ethContext = new EthContext(getSupportedProtocol(), ethPeers, ethMessages, scheduler); + ethContext = new EthContext(ethPeers, ethMessages, scheduler); this.blockBroadcaster = new BlockBroadcaster(ethContext); diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthScheduler.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthScheduler.java index 49354289c7..c36f5404a1 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthScheduler.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthScheduler.java @@ -242,10 +242,6 @@ private CompletableFuture failAfterTimeout(final Duration timeout) { return promise; } - public void failAfterTimeout(final CompletableFuture promise) { - failAfterTimeout(promise, defaultTimeout); - } - public void failAfterTimeout(final CompletableFuture promise, final Duration timeout) { final long delay = timeout.toMillis(); final TimeUnit unit = TimeUnit.MILLISECONDS; diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthServer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthServer.java index 22a6c7dfe0..7967521ea2 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthServer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthServer.java @@ -65,17 +65,6 @@ class EthServer { this.setupListeners(); } - EthServer( - final Blockchain blockchain, - final WorldStateArchive worldStateArchive, - final EthMessages ethMessages) { - this( - blockchain, - worldStateArchive, - ethMessages, - EthereumWireProtocolConfiguration.defaultConfig()); - } - private void setupListeners() { ethMessages.subscribe(EthPV62.GET_BLOCK_HEADERS, this::handleGetBlockHeaders); ethMessages.subscribe(EthPV62.GET_BLOCK_BODIES, this::handleGetBlockBodies); diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/RequestManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/RequestManager.java index aa626db960..625f67686a 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/RequestManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/RequestManager.java @@ -148,10 +148,6 @@ public void close() { dispatchBufferedResponses(); } - public EthPeer peer() { - return peer; - } - private void processMessage(final MessageData message) { if (closed) { return; diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractEthTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractEthTask.java index adc476c502..27ceb21d55 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractEthTask.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractEthTask.java @@ -25,7 +25,6 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -130,21 +129,6 @@ protected final CompletableFuture executeSubTask( } } - /** - * Utility for registering completable futures for cleanup if this EthTask is cancelled. - * - * @param the type of data returned from the CompletableFuture - * @param subTaskFuture the future to be registered. - */ - protected final void registerSubTask(final CompletableFuture subTaskFuture) { - synchronized (result) { - if (!isCancelled()) { - subTaskFutures.add(subTaskFuture); - subTaskFuture.whenComplete((r, t) -> subTaskFutures.remove(subTaskFuture)); - } - } - } - /** * Helper method for sending subTask to worker that will clean up if this EthTask is cancelled. * @@ -158,17 +142,6 @@ protected final CompletableFuture executeWorkerSubTask( return executeSubTask(() -> scheduler.scheduleSyncWorkerTask(subTask)); } - public final T result() { - if (!isSucceeded()) { - return null; - } - try { - return result.get().get(); - } catch (final InterruptedException | ExecutionException e) { - return null; - } - } - /** Execute core task logic. */ protected abstract void executeTask(); diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractPipelinedTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractPipelinedTask.java deleted file mode 100644 index 4f41f2bc8b..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/AbstractPipelinedTask.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.manager.task; - -import tech.pegasys.pantheon.ethereum.eth.manager.exceptions.EthTaskException; -import tech.pegasys.pantheon.metrics.Counter; -import tech.pegasys.pantheon.metrics.MetricCategory; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.util.ExceptionUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public abstract class AbstractPipelinedTask extends AbstractEthTask> { - private static final Logger LOG = LogManager.getLogger(); - - static final int TIMEOUT_MS = 1000; - - private final BlockingQueue inboundQueue; - private final BlockingQueue outboundQueue; - private final List results; - - private boolean shuttingDown = false; - private final AtomicReference processingException = new AtomicReference<>(null); - - private final Counter inboundQueueCounter; - private final Counter outboundQueueCounter; - - protected AbstractPipelinedTask( - final BlockingQueue inboundQueue, - final int outboundBacklogSize, - final MetricsSystem metricsSystem) { - super(metricsSystem); - this.inboundQueue = inboundQueue; - outboundQueue = new LinkedBlockingQueue<>(outboundBacklogSize); - results = new ArrayList<>(); - this.inboundQueueCounter = - metricsSystem - .createLabelledCounter( - MetricCategory.SYNCHRONIZER, - "inboundQueueCounter", - "count of queue items that started processing", - "taskName") - .labels(getClass().getSimpleName()); - this.outboundQueueCounter = - metricsSystem - .createLabelledCounter( - MetricCategory.SYNCHRONIZER, - "outboundQueueCounter", - "count of queue items that finished processing", - "taskName") - .labels(getClass().getSimpleName()); - } - - @Override - protected void executeTask() { - Optional previousInput = Optional.empty(); - try { - while (!isDone() && processingException.get() == null) { - if (shuttingDown && inboundQueue.isEmpty()) { - break; - } - final I input; - try { - input = inboundQueue.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS); - if (input == null) { - // timed out waiting for a result - continue; - } - inboundQueueCounter.inc(); - } catch (final InterruptedException e) { - // this is expected - continue; - } - final Optional output = processStep(input, previousInput); - output.ifPresent( - o -> { - while (!isDone()) { - try { - if (outboundQueue.offer(o, 1, TimeUnit.SECONDS)) { - outboundQueueCounter.inc(); - results.add(o); - break; - } - } catch (final InterruptedException e) { - processingException.compareAndSet(null, e); - break; - } - } - }); - previousInput = Optional.of(input); - } - } catch (final RuntimeException e) { - processingException.compareAndSet(null, e); - } - if (processingException.get() == null) { - result.get().complete(results); - } else { - result.get().completeExceptionally(processingException.get()); - } - } - - public BlockingQueue getOutboundQueue() { - return outboundQueue; - } - - public void shutdown() { - this.shuttingDown = true; - } - - protected void failExceptionally(final Throwable t) { - Throwable rootCause = ExceptionUtils.rootCause(t); - if (rootCause instanceof InterruptedException || rootCause instanceof EthTaskException) { - LOG.debug("Task Failure: {}", t.toString()); - } else { - LOG.error("Task Failure", t); - } - - processingException.compareAndSet(null, t); - result.get().completeExceptionally(t); - cancel(); - } - - protected abstract Optional processStep(I input, Optional previousInput); -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByHashTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByHashTask.java index 76f7f46ad3..3c48e6a9b4 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByHashTask.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByHashTask.java @@ -110,25 +110,6 @@ public static AbstractGetHeadersFromPeerTask endingAtHash( metricsSystem); } - public static AbstractGetHeadersFromPeerTask endingAtHash( - final ProtocolSchedule protocolSchedule, - final EthContext ethContext, - final Hash lastHash, - final long lastBlockNumber, - final int segmentLength, - final int skip, - final MetricsSystem metricsSystem) { - return new GetHeadersFromPeerByHashTask( - protocolSchedule, - ethContext, - lastHash, - lastBlockNumber, - segmentLength, - skip, - true, - metricsSystem); - } - public static AbstractGetHeadersFromPeerTask forSingleHash( final ProtocolSchedule protocolSchedule, final EthContext ethContext, diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByNumberTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByNumberTask.java index 5d096b7231..1d4f5b4df3 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByNumberTask.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/task/GetHeadersFromPeerByNumberTask.java @@ -53,16 +53,6 @@ public static AbstractGetHeadersFromPeerTask startingAtNumber( protocolSchedule, ethContext, firstBlockNumber, segmentLength, 0, false, metricsSystem); } - public static AbstractGetHeadersFromPeerTask endingAtNumber( - final ProtocolSchedule protocolSchedule, - final EthContext ethContext, - final long lastlockNumber, - final int segmentLength, - final MetricsSystem metricsSystem) { - return new GetHeadersFromPeerByNumberTask( - protocolSchedule, ethContext, lastlockNumber, segmentLength, 0, true, metricsSystem); - } - public static AbstractGetHeadersFromPeerTask endingAtNumber( final ProtocolSchedule protocolSchedule, final EthContext ethContext, diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockHandler.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockHandler.java deleted file mode 100644 index 52433b5e07..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockHandler.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync; - -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -public interface BlockHandler { - CompletableFuture> downloadBlocks(List headers); - - CompletableFuture> validateAndImportBlocks(List blocks); - - long extractBlockNumber(B block); - - Hash extractBlockHash(B block); - - CompletableFuture executeParallelCalculations(List blocks); -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/CheckpointHeaderManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/CheckpointHeaderManager.java deleted file mode 100644 index 9f74daf6a4..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/CheckpointHeaderManager.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync; - -import static java.util.Collections.emptyList; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.Blockchain; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPeerTask.PeerTaskResult; -import tech.pegasys.pantheon.ethereum.eth.manager.task.GetHeadersFromPeerByHashTask; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncTarget; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.util.ExceptionUtils; - -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.TimeoutException; - -import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class CheckpointHeaderManager { - - private static final Logger LOG = LogManager.getLogger(); - - private final Deque checkpointHeaders = new ConcurrentLinkedDeque<>(); - private final SynchronizerConfiguration config; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final SyncState syncState; - private final ProtocolSchedule protocolSchedule; - private final MetricsSystem metricsSystem; - - private int checkpointTimeouts = 0; - - public CheckpointHeaderManager( - final SynchronizerConfiguration config, - final ProtocolContext protocolContext, - final EthContext ethContext, - final SyncState syncState, - final ProtocolSchedule protocolSchedule, - final MetricsSystem metricsSystem) { - this.config = config; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.syncState = syncState; - this.protocolSchedule = protocolSchedule; - this.metricsSystem = metricsSystem; - } - - public CompletableFuture> pullCheckpointHeaders(final SyncTarget syncTarget) { - if (!shouldDownloadMoreCheckpoints()) { - return CompletableFuture.completedFuture(getCheckpointsAwaitingImport()); - } - - final BlockHeader lastHeader = - checkpointHeaders.size() > 0 ? checkpointHeaders.getLast() : syncTarget.commonAncestor(); - // Try to pull more checkpoint headers - return getAdditionalCheckpointHeaders(syncTarget, lastHeader) - .thenApply( - additionalCheckpoints -> { - if (!additionalCheckpoints.isEmpty()) { - checkpointTimeouts = 0; - checkpointHeaders.addAll(additionalCheckpoints); - LOG.debug("Tracking {} checkpoint headers", checkpointHeaders.size()); - } - return getCheckpointsAwaitingImport(); - }); - } - - protected CompletableFuture> getAdditionalCheckpointHeaders( - final SyncTarget syncTarget, final BlockHeader lastHeader) { - return requestAdditionalCheckpointHeaders(lastHeader, syncTarget) - .handle( - (headers, t) -> { - t = ExceptionUtils.rootCause(t); - if (t instanceof TimeoutException) { - checkpointTimeouts++; - return emptyList(); - } else if (t != null) { - // An error occurred, so no new checkpoints to add. - return emptyList(); - } - if (headers.size() > 0 - && checkpointHeaders.size() > 0 - && checkpointHeaders.getLast().equals(headers.get(0))) { - // Don't push header that is already tracked - headers.remove(0); - } - if (headers.isEmpty()) { - checkpointTimeouts++; - } - return headers; - }); - } - - private CompletableFuture> requestAdditionalCheckpointHeaders( - final BlockHeader lastHeader, final SyncTarget syncTarget) { - LOG.debug("Requesting checkpoint headers from {}", lastHeader.getNumber()); - final int skip = config.downloaderChainSegmentSize() - 1; - final int additionalHeaderCount = - calculateAdditionalCheckpointHeadersToRequest(lastHeader, skip); - if (additionalHeaderCount <= 0) { - return CompletableFuture.completedFuture(emptyList()); - } - return GetHeadersFromPeerByHashTask.startingAtHash( - protocolSchedule, - ethContext, - lastHeader.getHash(), - lastHeader.getNumber(), - // + 1 because lastHeader will be returned as well. - additionalHeaderCount + 1, - skip, - metricsSystem) - .assignPeer(syncTarget.peer()) - .run() - .thenApply(PeerTaskResult::getResult); - } - - protected int calculateAdditionalCheckpointHeadersToRequest( - final BlockHeader lastHeader, final int skip) { - return config.downloaderHeaderRequestSize(); - } - - protected boolean shouldDownloadMoreCheckpoints() { - return checkpointHeaders.size() < config.downloaderHeaderRequestSize() - && checkpointTimeouts < config.downloaderCheckpointTimeoutsPermitted(); - } - - public boolean checkpointsHaveTimedOut() { - // We have no more checkpoints, and have been unable to pull any new checkpoints for - // several cycles. - return checkpointHeaders.size() == 0 - && checkpointTimeouts >= config.downloaderCheckpointTimeoutsPermitted(); - } - - public void clearSyncTarget() { - checkpointTimeouts = 0; - checkpointHeaders.clear(); - } - - public boolean clearImportedCheckpointHeaders() { - final Blockchain blockchain = protocolContext.getBlockchain(); - // Update checkpoint headers to reflect if any checkpoints were imported. - final List imported = new ArrayList<>(); - while (!checkpointHeaders.isEmpty() - && blockchain.contains(checkpointHeaders.peekFirst().getHash())) { - imported.add(checkpointHeaders.removeFirst()); - } - final BlockHeader lastImportedCheckpointHeader = imported.get(imported.size() - 1); - // The first checkpoint header is always present in the blockchain. - checkpointHeaders.addFirst(lastImportedCheckpointHeader); - syncState.setCommonAncestor(lastImportedCheckpointHeader); - return imported.size() > 1; - } - - public BlockHeader allCheckpointsImported() { - final BlockHeader lastImportedCheckpoint = checkpointHeaders.getLast(); - checkpointHeaders.clear(); - return lastImportedCheckpoint; - } - - private List getCheckpointsAwaitingImport() { - return Lists.newArrayList(checkpointHeaders); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/EthTaskChainDownloader.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/EthTaskChainDownloader.java deleted file mode 100644 index 2b0c5638ed..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/EthTaskChainDownloader.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync; - -import static java.util.Collections.emptyList; - -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.exceptions.EthTaskException; -import tech.pegasys.pantheon.ethereum.eth.manager.task.WaitForPeersTask; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncTarget; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.exceptions.InvalidBlockException; -import tech.pegasys.pantheon.ethereum.p2p.wire.messages.DisconnectMessage.DisconnectReason; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.util.ExceptionUtils; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class EthTaskChainDownloader implements ChainDownloader { - private static final Logger LOG = LogManager.getLogger(); - - private final SynchronizerConfiguration config; - private final EthContext ethContext; - private final SyncState syncState; - private final SyncTargetManager syncTargetManager; - private final CheckpointHeaderManager checkpointHeaderManager; - private final BlockImportTaskFactory blockImportTaskFactory; - private final MetricsSystem metricsSystem; - private final CompletableFuture downloadFuture = new CompletableFuture<>(); - - private int chainSegmentTimeouts = 0; - - private final AtomicBoolean started = new AtomicBoolean(false); - private CompletableFuture currentTask; - - public EthTaskChainDownloader( - final SynchronizerConfiguration config, - final EthContext ethContext, - final SyncState syncState, - final SyncTargetManager syncTargetManager, - final CheckpointHeaderManager checkpointHeaderManager, - final BlockImportTaskFactory blockImportTaskFactory, - final MetricsSystem metricsSystem) { - this.metricsSystem = metricsSystem; - this.config = config; - this.ethContext = ethContext; - - this.syncState = syncState; - this.syncTargetManager = syncTargetManager; - this.checkpointHeaderManager = checkpointHeaderManager; - this.blockImportTaskFactory = blockImportTaskFactory; - } - - @Override - public CompletableFuture start() { - if (started.compareAndSet(false, true)) { - executeDownload(); - return downloadFuture; - } else { - throw new IllegalStateException( - "Attempt to start an already started " + this.getClass().getSimpleName() + "."); - } - } - - @Override - public void cancel() { - downloadFuture.cancel(true); - } - - @VisibleForTesting - public CompletableFuture getCurrentTask() { - return currentTask; - } - - private void executeDownload() { - if (downloadFuture.isDone()) { - return; - } - // Find target, pull checkpoint headers, import, repeat - currentTask = - waitForPeers() - .thenCompose(r -> syncTargetManager.findSyncTarget(syncState.syncTarget())) - .thenApply(this::updateSyncState) - .thenCompose(this::pullCheckpointHeaders) - .thenCompose(this::importBlocks) - .thenCompose(r -> checkSyncTarget()) - .whenComplete( - (r, t) -> { - if (t != null) { - final Throwable rootCause = ExceptionUtils.rootCause(t); - if (rootCause instanceof CancellationException) { - LOG.trace("Download cancelled", t); - } else if (rootCause instanceof InvalidBlockException) { - LOG.debug("Invalid block downloaded", t); - } else if (rootCause instanceof EthTaskException) { - LOG.debug(rootCause.toString()); - } else if (rootCause instanceof InterruptedException) { - LOG.trace("Interrupted while downloading chain", rootCause); - } else { - LOG.error("Error encountered while downloading", t); - } - // On error, wait a bit before retrying - ethContext - .getScheduler() - .scheduleFutureTask(this::executeDownload, Duration.ofSeconds(2)); - } else if (syncTargetManager.shouldContinueDownloading()) { - executeDownload(); - } else { - LOG.info("Chain download complete"); - downloadFuture.complete(null); - } - }); - } - - private SyncTarget updateSyncState(final SyncTarget newTarget) { - if (isSameAsCurrentTarget(newTarget)) { - return syncState.syncTarget().get(); - } - return syncState.setSyncTarget(newTarget.peer(), newTarget.commonAncestor()); - } - - private Boolean isSameAsCurrentTarget(final SyncTarget newTarget) { - return syncState - .syncTarget() - .map(currentTarget -> currentTarget.equals(newTarget)) - .orElse(false); - } - - private CompletableFuture> pullCheckpointHeaders(final SyncTarget syncTarget) { - return syncTarget.peer().isDisconnected() - ? CompletableFuture.completedFuture(emptyList()) - : checkpointHeaderManager.pullCheckpointHeaders(syncTarget); - } - - private CompletableFuture waitForPeers() { - return WaitForPeersTask.create(ethContext, 1, metricsSystem).run(); - } - - private CompletableFuture checkSyncTarget() { - final Optional maybeSyncTarget = syncState.syncTarget(); - if (!maybeSyncTarget.isPresent()) { - // No sync target, so nothing to check. - return CompletableFuture.completedFuture(null); - } - - final SyncTarget syncTarget = maybeSyncTarget.get(); - if (syncTargetManager.shouldSwitchSyncTarget(syncTarget)) { - LOG.info("Better sync target found, clear current sync target: {}.", syncTarget); - clearSyncTarget(syncTarget); - return CompletableFuture.completedFuture(null); - } - if (finishedSyncingToCurrentTarget(syncTarget)) { - LOG.info("Finished syncing to target: {}.", syncTarget); - clearSyncTarget(syncTarget); - // Wait a bit before checking for a new sync target - final CompletableFuture future = new CompletableFuture<>(); - ethContext - .getScheduler() - .scheduleFutureTask(() -> future.complete(null), Duration.ofSeconds(10)); - return future; - } - return CompletableFuture.completedFuture(null); - } - - private boolean finishedSyncingToCurrentTarget(final SyncTarget syncTarget) { - return !syncTargetManager.syncTargetCanProvideMoreBlocks(syncTarget) - || checkpointHeaderManager.checkpointsHaveTimedOut() - || chainSegmentsHaveTimedOut(); - } - - private boolean chainSegmentsHaveTimedOut() { - return chainSegmentTimeouts >= config.downloaderChainSegmentTimeoutsPermitted(); - } - - private void clearSyncTarget() { - syncState.syncTarget().ifPresent(this::clearSyncTarget); - } - - private void clearSyncTarget(final SyncTarget syncTarget) { - chainSegmentTimeouts = 0; - checkpointHeaderManager.clearSyncTarget(); - syncState.clearSyncTarget(); - } - - private CompletableFuture> importBlocks(final List checkpointHeaders) { - if (checkpointHeaders.isEmpty()) { - // No checkpoints to download - return CompletableFuture.completedFuture(emptyList()); - } - - final CompletableFuture> importedBlocks = - blockImportTaskFactory.importBlocksForCheckpoints(checkpointHeaders); - - return importedBlocks.whenComplete( - (r, t) -> { - t = ExceptionUtils.rootCause(t); - if (t instanceof InvalidBlockException) { - // Blocks were invalid, meaning our checkpoints are wrong - // Reset sync target - final Optional maybeSyncTarget = syncState.syncTarget(); - maybeSyncTarget.ifPresent( - target -> target.peer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL)); - final String peerDescriptor = - maybeSyncTarget - .map(SyncTarget::peer) - .map(EthPeer::toString) - .orElse("(unknown - already disconnected)"); - LOG.warn( - "Invalid block discovered while downloading from peer {}. Disconnect.", - peerDescriptor); - clearSyncTarget(); - } else if (t != null || r.isEmpty()) { - if (t != null) { - final Throwable rootCause = ExceptionUtils.rootCause(t); - if (rootCause instanceof EthTaskException) { - LOG.debug(rootCause.toString()); - } else if (rootCause instanceof InterruptedException) { - LOG.trace("Interrupted while importing blocks", rootCause); - } else { - LOG.error("Encountered error importing blocks", t); - } - } - if (checkpointHeaderManager.clearImportedCheckpointHeaders()) { - chainSegmentTimeouts = 0; - } - if (t instanceof TimeoutException || r != null) { - // Download timed out, or returned no new blocks - chainSegmentTimeouts++; - } - } else { - chainSegmentTimeouts = 0; - - final BlockHeader lastImportedCheckpoint = - checkpointHeaderManager.allCheckpointsImported(); - syncState.setCommonAncestor(lastImportedCheckpoint); - } - }); - } - - public interface BlockImportTaskFactory { - CompletableFuture> importBlocksForCheckpoints( - final List checkpointHeaders); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SyncTargetManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SyncTargetManager.java index 6b9c67b445..1c56906f86 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SyncTargetManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SyncTargetManager.java @@ -120,14 +120,5 @@ private CompletableFuture waitForNewPeer() { .timeout(WaitForPeerTask.create(ethContext, metricsSystem), Duration.ofSeconds(5)); } - public abstract boolean shouldSwitchSyncTarget(final SyncTarget currentTarget); - public abstract boolean shouldContinueDownloading(); - - public abstract boolean isSyncTargetReached(final EthPeer peer); - - public boolean syncTargetCanProvideMoreBlocks(final SyncTarget syncTarget) { - final EthPeer currentSyncingPeer = syncTarget.peer(); - return !currentSyncingPeer.isDisconnected() && !isSyncTargetReached(currentSyncingPeer); - } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java index c7eec2d0d7..539c673a87 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java @@ -54,13 +54,11 @@ public class SynchronizerConfiguration { private final UInt256 downloaderChangeTargetThresholdByTd; private final int downloaderHeaderRequestSize; private final int downloaderCheckpointTimeoutsPermitted; - private final int downloaderChainSegmentTimeoutsPermitted; private final int downloaderChainSegmentSize; private final int downloaderParallelism; private final int transactionsParallelism; private final int computationParallelism; private final int maxTrailingPeers; - private final boolean piplineDownloaderForFullSyncEnabled; private final long worldStateMinMillisBeforeStalling; private SynchronizerConfiguration( @@ -77,13 +75,11 @@ private SynchronizerConfiguration( final UInt256 downloaderChangeTargetThresholdByTd, final int downloaderHeaderRequestSize, final int downloaderCheckpointTimeoutsPermitted, - final int downloaderChainSegmentTimeoutsPermitted, final int downloaderChainSegmentSize, final int downloaderParallelism, final int transactionsParallelism, final int computationParallelism, - final int maxTrailingPeers, - final boolean piplineDownloaderForFullSyncEnabled) { + final int maxTrailingPeers) { this.fastSyncPivotDistance = fastSyncPivotDistance; this.fastSyncFullValidationRate = fastSyncFullValidationRate; this.fastSyncMinimumPeerCount = fastSyncMinimumPeerCount; @@ -97,13 +93,11 @@ private SynchronizerConfiguration( this.downloaderChangeTargetThresholdByTd = downloaderChangeTargetThresholdByTd; this.downloaderHeaderRequestSize = downloaderHeaderRequestSize; this.downloaderCheckpointTimeoutsPermitted = downloaderCheckpointTimeoutsPermitted; - this.downloaderChainSegmentTimeoutsPermitted = downloaderChainSegmentTimeoutsPermitted; this.downloaderChainSegmentSize = downloaderChainSegmentSize; this.downloaderParallelism = downloaderParallelism; this.transactionsParallelism = transactionsParallelism; this.computationParallelism = computationParallelism; this.maxTrailingPeers = maxTrailingPeers; - this.piplineDownloaderForFullSyncEnabled = piplineDownloaderForFullSyncEnabled; } public static Builder builder() { @@ -155,10 +149,6 @@ public int downloaderCheckpointTimeoutsPermitted() { return downloaderCheckpointTimeoutsPermitted; } - public int downloaderChainSegmentTimeoutsPermitted() { - return downloaderChainSegmentTimeoutsPermitted; - } - public int downloaderChainSegmentSize() { return downloaderChainSegmentSize; } @@ -210,10 +200,6 @@ public int getMaxTrailingPeers() { return maxTrailingPeers; } - public boolean isPiplineDownloaderForFullSyncEnabled() { - return piplineDownloaderForFullSyncEnabled; - } - public static class Builder { private SyncMode syncMode = SyncMode.FULL; private int fastSyncMinimumPeerCount = DEFAULT_FAST_SYNC_MINIMUM_PEERS; @@ -273,15 +259,6 @@ public void parseBlockPropagationRange(final String arg) { "Number of tries to attempt to download checkpoints before stopping (default: ${DEFAULT-VALUE})") private int downloaderCheckpointTimeoutsPermitted = 5; - @CommandLine.Option( - names = "--Xsynchronizer-downloader-chain-segment-timeouts-permitted", - hidden = true, - defaultValue = "5", - paramLabel = "", - description = - "Number of times to attempt to download chain segments before stopping (default: ${DEFAULT-VALUE})") - private int downloaderChainSegmentTimeoutsPermitted = 5; - @CommandLine.Option( names = "--Xsynchronizer-downloader-chain-segment-size", hidden = true, @@ -371,14 +348,6 @@ public void parseBlockPropagationRange(final String arg) { "Minimum time in ms without progress before considering a world state download as stalled (default: ${DEFAULT-VALUE})") private long worldStateMinMillisBeforeStalling = DEFAULT_WORLD_STATE_MIN_MILLIS_BEFORE_STALLING; - @CommandLine.Option( - names = "--Xsynchronizer-pipeline-full-sync-enabled", - hidden = true, - defaultValue = "true", - paramLabel = "", - description = "Enable the pipeline based chain downloader during full synchronization") - private Boolean piplineDownloaderForFullSyncEnabled = true; - public Builder fastSyncPivotDistance(final int distance) { fastSyncPivotDistance = distance; return this; @@ -417,12 +386,6 @@ public Builder downloaderCheckpointTimeoutsPermitted( return this; } - public Builder downloaderChainSegmentTimeoutsPermitted( - final int downloaderChainSegmentTimeoutsPermitted) { - this.downloaderChainSegmentTimeoutsPermitted = downloaderChainSegmentTimeoutsPermitted; - return this; - } - public Builder downloaderChainSegmentSize(final int downloaderChainSegmentSize) { this.downloaderChainSegmentSize = downloaderChainSegmentSize; return this; @@ -480,12 +443,6 @@ public Builder maxTrailingPeers(final int maxTailingPeers) { return this; } - public Builder piplineDownloaderForFullSyncEnabled( - final Boolean piplineDownloaderForFullSyncEnabled) { - this.piplineDownloaderForFullSyncEnabled = piplineDownloaderForFullSyncEnabled; - return this; - } - public SynchronizerConfiguration build() { return new SynchronizerConfiguration( fastSyncPivotDistance, @@ -501,13 +458,11 @@ public SynchronizerConfiguration build() { downloaderChangeTargetThresholdByTd, downloaderHeaderRequestSize, downloaderCheckpointTimeoutsPermitted, - downloaderChainSegmentTimeoutsPermitted, downloaderChainSegmentSize, downloaderParallelism, transactionsParallelism, computationParallelism, - maxTrailingPeers, - piplineDownloaderForFullSyncEnabled); + maxTrailingPeers); } } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandler.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandler.java deleted file mode 100644 index aa24d5d9b3..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandler.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; - -import static java.util.Collections.emptyList; -import static tech.pegasys.pantheon.util.FutureUtils.completedExceptionally; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.BlockImporter; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.core.TransactionReceipt; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.sync.BlockHandler; -import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.CompleteBlocksTask; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.GetReceiptsForHeadersTask; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.exceptions.InvalidBlockException; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSpec; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class FastSyncBlockHandler implements BlockHandler { - private static final Logger LOG = LogManager.getLogger(); - - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final MetricsSystem metricsSystem; - private final ValidationPolicy headerValidationPolicy; - private final ValidationPolicy ommerValidationPolicy; - - public FastSyncBlockHandler( - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final MetricsSystem metricsSystem, - final ValidationPolicy headerValidationPolicy, - final ValidationPolicy ommerValidationPolicy) { - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.metricsSystem = metricsSystem; - this.headerValidationPolicy = headerValidationPolicy; - this.ommerValidationPolicy = ommerValidationPolicy; - } - - @Override - public CompletableFuture> downloadBlocks( - final List headers) { - return downloadBodies(headers) - .thenCombine(downloadReceipts(headers), this::combineBlocksAndReceipts); - } - - private CompletableFuture> downloadBodies(final List headers) { - return CompleteBlocksTask.forHeaders(protocolSchedule, ethContext, headers, metricsSystem) - .run(); - } - - private CompletableFuture>> downloadReceipts( - final List headers) { - return GetReceiptsForHeadersTask.forHeaders(ethContext, headers, metricsSystem).run(); - } - - private List combineBlocksAndReceipts( - final List blocks, final Map> receiptsByHeader) { - return blocks.stream() - .map( - block -> { - final List receipts = - receiptsByHeader.getOrDefault(block.getHeader(), emptyList()); - return new BlockWithReceipts(block, receipts); - }) - .collect(Collectors.toList()); - } - - @Override - public CompletableFuture> validateAndImportBlocks( - final List blocksWithReceipts) { - LOG.debug( - "Storing blocks {} to {}", - blocksWithReceipts.get(0).getHeader().getNumber(), - blocksWithReceipts.get(blocksWithReceipts.size() - 1).getHeader().getNumber()); - - for (final BlockWithReceipts blockWithReceipt : blocksWithReceipts) { - final BlockImporter blockImporter = getBlockImporter(blockWithReceipt); - final Block block = blockWithReceipt.getBlock(); - if (!blockImporter.fastImportBlock( - protocolContext, - block, - blockWithReceipt.getReceipts(), - headerValidationPolicy.getValidationModeForNextBlock(), - ommerValidationPolicy.getValidationModeForNextBlock())) { - return invalidBlockFailure(block); - } - } - return CompletableFuture.completedFuture(blocksWithReceipts); - } - - private CompletableFuture> invalidBlockFailure(final Block block) { - return completedExceptionally( - new InvalidBlockException( - "Failed to import block", block.getHeader().getNumber(), block.getHash())); - } - - private BlockImporter getBlockImporter(final BlockWithReceipts blockWithReceipt) { - final ProtocolSpec protocolSpec = - protocolSchedule.getByBlockNumber(blockWithReceipt.getHeader().getNumber()); - - return protocolSpec.getBlockImporter(); - } - - @Override - public long extractBlockNumber(final BlockWithReceipts blockWithReceipt) { - return blockWithReceipt.getHeader().getNumber(); - } - - @Override - public Hash extractBlockHash(final BlockWithReceipts block) { - return block.getHash(); - } - - @Override - public CompletableFuture executeParallelCalculations(final List blocks) { - return CompletableFuture.completedFuture(null); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockImportTaskFactory.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockImportTaskFactory.java deleted file mode 100644 index b0ba9c5fd4..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockImportTaskFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; - -import static java.util.Collections.emptyList; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.sync.EthTaskChainDownloader.BlockImportTaskFactory; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.ParallelImportChainSegmentTask; -import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.Counter; -import tech.pegasys.pantheon.metrics.LabelledMetric; -import tech.pegasys.pantheon.metrics.MetricCategory; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -class FastSyncBlockImportTaskFactory implements BlockImportTaskFactory { - - private final SynchronizerConfiguration config; - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final MetricsSystem metricsSystem; - private final LabelledMetric fastSyncValidationCounter; - - FastSyncBlockImportTaskFactory( - final SynchronizerConfiguration config, - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final MetricsSystem metricsSystem) { - this.config = config; - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.metricsSystem = metricsSystem; - this.fastSyncValidationCounter = - metricsSystem.createLabelledCounter( - MetricCategory.SYNCHRONIZER, - "fast_sync_validation_mode", - "Number of blocks validated using light vs full validation during fast sync", - "validationMode"); - } - - @Override - public CompletableFuture> importBlocksForCheckpoints( - final List checkpointHeaders) { - if (checkpointHeaders.size() < 2) { - return CompletableFuture.completedFuture(emptyList()); - } - final FastSyncValidationPolicy attachedValidationPolicy = - new FastSyncValidationPolicy( - config.fastSyncFullValidationRate(), - HeaderValidationMode.LIGHT_SKIP_DETACHED, - HeaderValidationMode.SKIP_DETACHED, - fastSyncValidationCounter); - final FastSyncValidationPolicy ommerValidationPolicy = - new FastSyncValidationPolicy( - config.fastSyncFullValidationRate(), - HeaderValidationMode.LIGHT, - HeaderValidationMode.FULL, - fastSyncValidationCounter); - final FastSyncValidationPolicy detatchedValidationPolicy = - new FastSyncValidationPolicy( - config.fastSyncFullValidationRate(), - HeaderValidationMode.LIGHT_DETACHED_ONLY, - HeaderValidationMode.DETACHED_ONLY, - fastSyncValidationCounter); - - final ParallelImportChainSegmentTask importTask = - ParallelImportChainSegmentTask.forCheckpoints( - protocolSchedule, - protocolContext, - ethContext, - config.downloaderParallelism(), - new FastSyncBlockHandler<>( - protocolSchedule, - protocolContext, - ethContext, - metricsSystem, - attachedValidationPolicy, - ommerValidationPolicy), - detatchedValidationPolicy, - checkpointHeaders, - metricsSystem); - return importTask.run(); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java index e7005e7daf..a30fe38735 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java @@ -16,7 +16,6 @@ import tech.pegasys.pantheon.ethereum.core.BlockHeader; import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.sync.ChainDownloader; -import tech.pegasys.pantheon.ethereum.eth.sync.EthTaskChainDownloader; import tech.pegasys.pantheon.ethereum.eth.sync.PipelineChainDownloader; import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; @@ -25,8 +24,6 @@ public class FastSyncChainDownloader { - private static final boolean USE_PIPELINE_DOWNLOADER = true; - private FastSyncChainDownloader() {} public static ChainDownloader create( @@ -42,36 +39,12 @@ public static ChainDownloader create( new FastSyncTargetManager<>( config, protocolSchedule, protocolContext, ethContext, metricsSystem, pivotBlockHeader); - if (USE_PIPELINE_DOWNLOADER) { - return new PipelineChainDownloader<>( - syncState, - syncTargetManager, - new FastSyncDownloadPipelineFactory<>( - config, - protocolSchedule, - protocolContext, - ethContext, - pivotBlockHeader, - metricsSystem), - ethContext.getScheduler(), - metricsSystem); - } - - return new EthTaskChainDownloader<>( - config, - ethContext, + return new PipelineChainDownloader<>( syncState, syncTargetManager, - new FastSyncCheckpointHeaderManager<>( - config, - protocolContext, - ethContext, - syncState, - protocolSchedule, - metricsSystem, - pivotBlockHeader), - new FastSyncBlockImportTaskFactory<>( - config, protocolSchedule, protocolContext, ethContext, metricsSystem), + new FastSyncDownloadPipelineFactory<>( + config, protocolSchedule, protocolContext, ethContext, pivotBlockHeader, metricsSystem), + ethContext.getScheduler(), metricsSystem); } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointFilter.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointFilter.java deleted file mode 100644 index e50aa4f644..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointFilter.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; - -import static java.util.stream.Collectors.toList; - -import tech.pegasys.pantheon.ethereum.core.BlockHeader; - -import java.util.List; -import java.util.function.UnaryOperator; - -public class FastSyncCheckpointFilter implements UnaryOperator> { - - private final BlockHeader pivotBlockHeader; - - public FastSyncCheckpointFilter(final BlockHeader pivotBlockHeader) { - this.pivotBlockHeader = pivotBlockHeader; - } - - @Override - public List apply(final List blockHeaders) { - if (blockHeaders.isEmpty()) { - return blockHeaders; - } - if (lastHeaderNumberIn(blockHeaders) > pivotBlockHeader.getNumber()) { - return trimToPivotBlock(blockHeaders); - } - return blockHeaders; - } - - private List trimToPivotBlock(final List blockHeaders) { - final List filteredHeaders = - blockHeaders.stream() - .filter(header -> header.getNumber() <= pivotBlockHeader.getNumber()) - .collect(toList()); - if (filteredHeaders.isEmpty() - || lastHeaderNumberIn(filteredHeaders) != pivotBlockHeader.getNumber()) { - filteredHeaders.add(pivotBlockHeader); - } - return filteredHeaders; - } - - private long lastHeaderNumberIn(final List filteredHeaders) { - return filteredHeaders.get(filteredHeaders.size() - 1).getNumber(); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointHeaderManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointHeaderManager.java deleted file mode 100644 index 54cb689ba5..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointHeaderManager.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.sync.CheckpointHeaderManager; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncTarget; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import com.google.common.collect.Lists; - -class FastSyncCheckpointHeaderManager extends CheckpointHeaderManager { - private final SynchronizerConfiguration config; - private final BlockHeader pivotBlockHeader; - - public FastSyncCheckpointHeaderManager( - final SynchronizerConfiguration config, - final ProtocolContext protocolContext, - final EthContext ethContext, - final SyncState syncState, - final ProtocolSchedule protocolSchedule, - final MetricsSystem metricsSystem, - final BlockHeader pivotBlockHeader) { - super(config, protocolContext, ethContext, syncState, protocolSchedule, metricsSystem); - this.config = config; - this.pivotBlockHeader = pivotBlockHeader; - } - - @Override - protected CompletableFuture> getAdditionalCheckpointHeaders( - final SyncTarget syncTarget, final BlockHeader lastHeader) { - return super.getAdditionalCheckpointHeaders(syncTarget, lastHeader) - .thenApply( - checkpointHeaders -> { - final long lastSegmentEnd = - checkpointHeaders.isEmpty() - ? lastHeader.getNumber() - : checkpointHeaders.get(checkpointHeaders.size() - 1).getNumber(); - if (shouldDownloadMoreCheckpoints() - && nextChainSegmentIncludesPivotBlock(lastSegmentEnd) - && pivotBlockNotAlreadyIncluded(lastSegmentEnd)) { - if (checkpointHeaders.isEmpty()) { - return Lists.newArrayList(lastHeader, pivotBlockHeader); - } - return concat(checkpointHeaders, pivotBlockHeader); - } - return checkpointHeaders; - }); - } - - private List concat( - final List checkpointHeaders, final BlockHeader value) { - return Stream.concat(checkpointHeaders.stream(), Stream.of(value)).collect(Collectors.toList()); - } - - private boolean pivotBlockNotAlreadyIncluded(final long lastSegmentEnd) { - return lastSegmentEnd < pivotBlockHeader.getNumber(); - } - - @Override - protected int calculateAdditionalCheckpointHeadersToRequest( - final BlockHeader lastHeader, final int skip) { - final long startingBlockNumber = lastHeader.getNumber(); - final long blocksUntilPivotBlock = pivotBlockHeader.getNumber() - startingBlockNumber; - - final long toRequest = blocksUntilPivotBlock / (skip + 1); - final int maximumAdditionalCheckpoints = - super.calculateAdditionalCheckpointHeadersToRequest(lastHeader, skip); - return (int) Math.min(maximumAdditionalCheckpoints, toRequest); - } - - private boolean nextChainSegmentIncludesPivotBlock(final long lastSegmentEnd) { - final long nextSegmentEnd = lastSegmentEnd + config.downloaderChainSegmentSize(); - return nextSegmentEnd >= pivotBlockHeader.getNumber(); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncError.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncError.java index 54640b9e3a..03c81b0544 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncError.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncError.java @@ -13,7 +13,6 @@ package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; public enum FastSyncError { - FAST_SYNC_UNAVAILABLE, NO_PEERS_AVAILABLE, CHAIN_TOO_SHORT, PIVOT_BLOCK_HEADER_MISMATCH, diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncTargetManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncTargetManager.java index 48a09b2d9e..988cc5cb61 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncTargetManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncTargetManager.java @@ -16,13 +16,11 @@ import static tech.pegasys.pantheon.ethereum.eth.sync.fastsync.PivotBlockRetriever.MAX_PIVOT_BLOCK_RETRIES; import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; import tech.pegasys.pantheon.ethereum.core.BlockHeader; import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; import tech.pegasys.pantheon.ethereum.eth.sync.SyncTargetManager; import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncTarget; import tech.pegasys.pantheon.ethereum.eth.sync.tasks.RetryingGetHeaderFromPeerByNumberTask; import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; import tech.pegasys.pantheon.ethereum.p2p.wire.messages.DisconnectMessage.DisconnectReason; @@ -108,19 +106,8 @@ private boolean peerHasDifferentPivotBlock(final List result) { return result.size() != 1 || !result.get(0).equals(pivotBlockHeader); } - @Override - public boolean shouldSwitchSyncTarget(final SyncTarget currentTarget) { - return false; - } - @Override public boolean shouldContinueDownloading() { return !protocolContext.getBlockchain().getChainHeadHash().equals(pivotBlockHeader.getHash()); } - - @Override - public boolean isSyncTargetReached(final EthPeer peer) { - final MutableBlockchain blockchain = protocolContext.getBlockchain(); - return blockchain.getChainHeadBlockNumber() >= pivotBlockHeader.getNumber(); - } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/ExtractTxSignaturesTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/ExtractTxSignaturesStep.java similarity index 94% rename from ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/ExtractTxSignaturesTask.java rename to ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/ExtractTxSignaturesStep.java index fc883b4f35..61bfe8e640 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/ExtractTxSignaturesTask.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/ExtractTxSignaturesStep.java @@ -19,7 +19,7 @@ import java.util.function.Function; import java.util.stream.Stream; -public class ExtractTxSignaturesTask implements Function, Stream> { +public class ExtractTxSignaturesStep implements Function, Stream> { @Override public Stream apply(final List blocks) { diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncBlockHandler.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncBlockHandler.java deleted file mode 100644 index 10d86ef4d6..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncBlockHandler.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fullsync; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.core.Transaction; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthScheduler; -import tech.pegasys.pantheon.ethereum.eth.sync.BlockHandler; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.CompleteBlocksTask; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.PersistBlockTask; -import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class FullSyncBlockHandler implements BlockHandler { - private static final Logger LOG = LogManager.getLogger(); - - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final MetricsSystem metricsSystem; - - public FullSyncBlockHandler( - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final MetricsSystem metricsSystem) { - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.metricsSystem = metricsSystem; - } - - @Override - public CompletableFuture> validateAndImportBlocks(final List blocks) { - LOG.debug( - "Validating and importing {} to {}", - blocks.get(0).getHeader().getNumber(), - blocks.get(blocks.size() - 1).getHeader().getNumber()); - return PersistBlockTask.forSequentialBlocks( - protocolSchedule, - protocolContext, - blocks, - HeaderValidationMode.SKIP_DETACHED, - metricsSystem) - .get(); - } - - @Override - public CompletableFuture> downloadBlocks(final List headers) { - return CompleteBlocksTask.forHeaders(protocolSchedule, ethContext, headers, metricsSystem) - .run(); - } - - @Override - public long extractBlockNumber(final Block block) { - return block.getHeader().getNumber(); - } - - @Override - public Hash extractBlockHash(final Block block) { - return block.getHash(); - } - - @Override - public CompletableFuture executeParallelCalculations(final List blocks) { - final EthScheduler ethScheduler = ethContext.getScheduler(); - final List> calculations = new ArrayList<>(); - for (final Block block : blocks) { - for (final Transaction tx : block.getBody().getTransactions()) { - calculations.add(ethScheduler.scheduleComputationTask(tx::getSender)); - } - } - return CompletableFuture.allOf(calculations.toArray(new CompletableFuture[0])); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncBlockImportTaskFactory.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncBlockImportTaskFactory.java deleted file mode 100644 index 35fba43b8e..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncBlockImportTaskFactory.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fullsync; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPeerTask.PeerTaskResult; -import tech.pegasys.pantheon.ethereum.eth.sync.EthTaskChainDownloader.BlockImportTaskFactory; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.ImportBlocksTask; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.ParallelImportChainSegmentTask; -import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -class FullSyncBlockImportTaskFactory implements BlockImportTaskFactory { - - private final SynchronizerConfiguration config; - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final MetricsSystem metricsSystem; - - FullSyncBlockImportTaskFactory( - final SynchronizerConfiguration config, - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final MetricsSystem metricsSystem) { - this.config = config; - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.metricsSystem = metricsSystem; - } - - @Override - public CompletableFuture> importBlocksForCheckpoints( - final List checkpointHeaders) { - final CompletableFuture> importedHashes; - if (checkpointHeaders.size() < 2) { - // Download blocks without constraining the end block - final ImportBlocksTask importTask = - ImportBlocksTask.fromHeader( - protocolSchedule, - protocolContext, - ethContext, - checkpointHeaders.get(0), - config.downloaderChainSegmentSize(), - metricsSystem); - importedHashes = importTask.run().thenApply(PeerTaskResult::getResult); - } else { - final ParallelImportChainSegmentTask importTask = - ParallelImportChainSegmentTask.forCheckpoints( - protocolSchedule, - protocolContext, - ethContext, - config.downloaderParallelism(), - new FullSyncBlockHandler<>( - protocolSchedule, protocolContext, ethContext, metricsSystem), - () -> HeaderValidationMode.DETACHED_ONLY, - checkpointHeaders, - metricsSystem); - importedHashes = importTask.run(); - } - return importedHashes; - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java index 3a88c28b22..5960c6217c 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloader.java @@ -15,20 +15,13 @@ import tech.pegasys.pantheon.ethereum.ProtocolContext; import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.sync.ChainDownloader; -import tech.pegasys.pantheon.ethereum.eth.sync.CheckpointHeaderManager; -import tech.pegasys.pantheon.ethereum.eth.sync.EthTaskChainDownloader; import tech.pegasys.pantheon.ethereum.eth.sync.PipelineChainDownloader; import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; import tech.pegasys.pantheon.metrics.MetricsSystem; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - public class FullSyncChainDownloader { - private static final Logger LOG = LogManager.getLogger(); - private FullSyncChainDownloader() {} public static ChainDownloader create( @@ -43,27 +36,12 @@ public static ChainDownloader create( new FullSyncTargetManager<>( config, protocolSchedule, protocolContext, ethContext, metricsSystem); - if (config.isPiplineDownloaderForFullSyncEnabled()) { - LOG.info("Using PipelineChainDownloader"); - return new PipelineChainDownloader<>( - syncState, - syncTargetManager, - new FullSyncDownloadPipelineFactory<>( - config, protocolSchedule, protocolContext, ethContext, metricsSystem), - ethContext.getScheduler(), - metricsSystem); - } - - LOG.info("Using EthTaskChainDownloader"); - return new EthTaskChainDownloader<>( - config, - ethContext, + return new PipelineChainDownloader<>( syncState, syncTargetManager, - new CheckpointHeaderManager<>( - config, protocolContext, ethContext, syncState, protocolSchedule, metricsSystem), - new FullSyncBlockImportTaskFactory<>( + new FullSyncDownloadPipelineFactory<>( config, protocolSchedule, protocolContext, ethContext, metricsSystem), + ethContext.getScheduler(), metricsSystem); } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java index 4dc02790af..8f047e4c10 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncDownloadPipelineFactory.java @@ -86,7 +86,7 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncTarget target) protocolSchedule, protocolContext, detachedValidationPolicy); final DownloadBodiesStep downloadBodiesStep = new DownloadBodiesStep<>(protocolSchedule, ethContext, metricsSystem); - final ExtractTxSignaturesTask extractTxSignaturesTask = new ExtractTxSignaturesTask(); + final ExtractTxSignaturesStep extractTxSignaturesStep = new ExtractTxSignaturesStep(); final FullImportBlockStep importBlockStep = new FullImportBlockStep<>(protocolSchedule, protocolContext); @@ -104,7 +104,7 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncTarget target) .thenFlatMap("validateHeadersJoin", validateHeadersJoinUpStep, singleHeaderBufferSize) .inBatches(headerRequestSize) .thenProcessAsyncOrdered("downloadBodies", downloadBodiesStep, downloaderParallelism) - .thenFlatMap("extractTxSignatures", extractTxSignaturesTask, singleHeaderBufferSize) + .thenFlatMap("extractTxSignatures", extractTxSignaturesStep, singleHeaderBufferSize) .andFinishWith("importBlock", importBlockStep); } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncTargetManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncTargetManager.java index 32a1e4bcfa..f2228bebf0 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncTargetManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncTargetManager.java @@ -38,7 +38,6 @@ class FullSyncTargetManager extends SyncTargetManager { private static final Logger LOG = LogManager.getLogger(); private final ProtocolContext protocolContext; private final EthContext ethContext; - private final BetterSyncTargetEvaluator betterSyncTargetEvaluator; FullSyncTargetManager( final SynchronizerConfiguration config, @@ -49,7 +48,6 @@ class FullSyncTargetManager extends SyncTargetManager { super(config, protocolSchedule, protocolContext, ethContext, metricsSystem); this.protocolContext = protocolContext; this.ethContext = ethContext; - betterSyncTargetEvaluator = new BetterSyncTargetEvaluator(config, ethContext.getEthPeers()); } @Override @@ -86,8 +84,7 @@ protected CompletableFuture> selectBestAvailableSyncTarget() { } } - @Override - public boolean isSyncTargetReached(final EthPeer peer) { + private boolean isSyncTargetReached(final EthPeer peer) { final long peerHeight = peer.chainState().getEstimatedHeight(); final UInt256 peerTd = peer.chainState().getBestBlock().getTotalDifficulty(); final MutableBlockchain blockchain = protocolContext.getBlockchain(); @@ -96,11 +93,6 @@ public boolean isSyncTargetReached(final EthPeer peer) { && peerHeight <= blockchain.getChainHeadBlockNumber(); } - @Override - public boolean shouldSwitchSyncTarget(final SyncTarget currentTarget) { - return betterSyncTargetEvaluator.shouldSwitchSyncTarget(currentTarget.peer()); - } - @Override public boolean shouldContinueDownloading() { return true; diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncState.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncState.java index 6ef8fa1ece..078e0b9dc4 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncState.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncState.java @@ -74,10 +74,9 @@ public Optional syncTarget() { return syncTarget; } - public SyncTarget setSyncTarget(final EthPeer peer, final BlockHeader commonAncestor) { + public void setSyncTarget(final EthPeer peer, final BlockHeader commonAncestor) { final SyncTarget syncTarget = new SyncTarget(peer, commonAncestor); replaceSyncTarget(Optional.of(syncTarget)); - return syncTarget; } public boolean isInSync() { @@ -90,10 +89,6 @@ public boolean isInSync(final long syncTolerance) { .orElse(true); } - public void setCommonAncestor(final BlockHeader commonAncestor) { - syncTarget.ifPresent(target -> target.setCommonAncestor(commonAncestor)); - } - public void clearSyncTarget() { replaceSyncTarget(Optional.empty()); } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncTarget.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncTarget.java index 014d04abd3..e56d25e090 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncTarget.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/state/SyncTarget.java @@ -24,7 +24,7 @@ public class SyncTarget { private final EthPeer peer; - private BlockHeader commonAncestor; + private final BlockHeader commonAncestor; public SyncTarget(final EthPeer peer, final BlockHeader commonAncestor) { this.peer = peer; @@ -39,10 +39,6 @@ public BlockHeader commonAncestor() { return commonAncestor; } - void setCommonAncestor(final BlockHeader commonAncestor) { - this.commonAncestor = commonAncestor; - } - public long addPeerChainEstimatedHeightListener(final EstimatedHeightListener listener) { return peer.addChainEstimatedHeightListener(listener); } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ImportBlocksTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ImportBlocksTask.java deleted file mode 100644 index 01dbb73de9..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ImportBlocksTask.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPeerTask; -import tech.pegasys.pantheon.ethereum.eth.manager.task.GetHeadersFromPeerByHashTask; -import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection.PeerNotConnected; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Download and import blocks from a peer. - * - * @param the consensus algorithm context - */ -public class ImportBlocksTask extends AbstractPeerTask> { - private static final Logger LOG = LogManager.getLogger(); - - private final ProtocolContext protocolContext; - private final ProtocolSchedule protocolSchedule; - private final long startNumber; - - private final BlockHeader referenceHeader; - private final int maxBlocks; - private final MetricsSystem metricsSystem; - private EthPeer peer; - - protected ImportBlocksTask( - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final BlockHeader referenceHeader, - final int maxBlocks, - final MetricsSystem metricsSystem) { - super(ethContext, metricsSystem); - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.referenceHeader = referenceHeader; - this.maxBlocks = maxBlocks; - this.metricsSystem = metricsSystem; - - this.startNumber = referenceHeader.getNumber(); - } - - public static ImportBlocksTask fromHeader( - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final BlockHeader previousHeader, - final int maxBlocks, - final MetricsSystem metricsSystem) { - return new ImportBlocksTask<>( - protocolSchedule, protocolContext, ethContext, previousHeader, maxBlocks, metricsSystem); - } - - @Override - protected void executeTaskWithPeer(final EthPeer peer) throws PeerNotConnected { - this.peer = peer; - LOG.debug("Importing blocks from {}", startNumber); - downloadHeaders() - .thenCompose(this::completeBlocks) - .thenCompose(this::importBlocks) - .whenComplete( - (r, t) -> { - if (t != null) { - LOG.debug("Import from block {} failed: {}.", startNumber, t); - result.get().completeExceptionally(t); - } else { - LOG.debug("Import from block {} succeeded.", startNumber); - result - .get() - .complete( - new PeerTaskResult<>( - peer, r.stream().map(Block::getHash).collect(Collectors.toList()))); - } - }); - } - - @Override - protected Optional findSuitablePeer() { - return ethContext.getEthPeers().idlePeer(referenceHeader.getNumber()); - } - - private CompletableFuture>> downloadHeaders() { - final AbstractPeerTask> task = - GetHeadersFromPeerByHashTask.startingAtHash( - protocolSchedule, - ethContext, - referenceHeader.getHash(), - referenceHeader.getNumber(), - maxBlocks, - metricsSystem) - .assignPeer(peer); - return executeSubTask(task::run); - } - - private CompletableFuture> completeBlocks( - final PeerTaskResult> headers) { - if (headers.getResult().isEmpty()) { - return CompletableFuture.completedFuture(Collections.emptyList()); - } - final CompleteBlocksTask task = - CompleteBlocksTask.forHeaders( - protocolSchedule, ethContext, headers.getResult(), metricsSystem); - task.assignPeer(peer); - return executeSubTask(() -> ethContext.getScheduler().timeout(task)); - } - - private CompletableFuture> importBlocks(final List blocks) { - // Don't import reference block if we already know about it - if (protocolContext.getBlockchain().contains(referenceHeader.getHash())) { - blocks.removeIf(b -> b.getHash().equals(referenceHeader.getHash())); - } - if (blocks.isEmpty()) { - return CompletableFuture.completedFuture(Collections.emptyList()); - } - final Supplier>> task = - PersistBlockTask.forSequentialBlocks( - protocolSchedule, protocolContext, blocks, HeaderValidationMode.FULL, metricsSystem); - return executeWorkerSubTask(ethContext.getScheduler(), task); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelDownloadBodiesTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelDownloadBodiesTask.java deleted file mode 100644 index b5c445d390..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelDownloadBodiesTask.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPipelinedTask; -import tech.pegasys.pantheon.ethereum.eth.sync.BlockHandler; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ParallelDownloadBodiesTask - extends AbstractPipelinedTask, List> { - private static final Logger LOG = LogManager.getLogger(); - - private final BlockHandler blockHandler; - - ParallelDownloadBodiesTask( - final BlockHandler blockHandler, - final BlockingQueue> inboundQueue, - final int outboundBacklogSize, - final MetricsSystem metricsSystem) { - super(inboundQueue, outboundBacklogSize, metricsSystem); - - this.blockHandler = blockHandler; - } - - @Override - protected Optional> processStep( - final List headers, final Optional> previousHeaders) { - LOG.trace( - "Downloading bodies {} to {}", - headers.get(0).getNumber(), - headers.get(headers.size() - 1).getNumber()); - try { - final List blocks = blockHandler.downloadBlocks(headers).get(); - LOG.debug( - "Downloaded bodies {} to {}", - headers.get(0).getNumber(), - headers.get(headers.size() - 1).getNumber()); - return Optional.of(blocks); - } catch (final InterruptedException | ExecutionException e) { - failExceptionally(e); - return Optional.empty(); - } - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelDownloadHeadersTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelDownloadHeadersTask.java deleted file mode 100644 index 46ceae089a..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelDownloadHeadersTask.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPipelinedTask; -import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - -import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ParallelDownloadHeadersTask - extends AbstractPipelinedTask> { - private static final Logger LOG = LogManager.getLogger(); - - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final ValidationPolicy validationPolicy; - private final MetricsSystem metricsSystem; - - ParallelDownloadHeadersTask( - final BlockingQueue inboundQueue, - final int outboundBacklogSize, - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final ValidationPolicy validationPolicy, - final MetricsSystem metricsSystem) { - super(inboundQueue, outboundBacklogSize, metricsSystem); - - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.validationPolicy = validationPolicy; - this.metricsSystem = metricsSystem; - } - - @Override - protected Optional> processStep( - final BlockHeader nextCheckpointHeader, - final Optional previousCheckpointHeader) { - if (!previousCheckpointHeader.isPresent()) { - return Optional.empty(); - } - final int segmentLength = - (int) (nextCheckpointHeader.getNumber() - previousCheckpointHeader.get().getNumber()) - 1; - LOG.trace( - "Requesting download of {} blocks ending at {}", - segmentLength, - nextCheckpointHeader.getHash()); - final DownloadHeaderSequenceTask downloadTask = - DownloadHeaderSequenceTask.endingAtHeader( - protocolSchedule, - protocolContext, - ethContext, - nextCheckpointHeader, - segmentLength, - validationPolicy, - metricsSystem); - final CompletableFuture> headerFuture = executeSubTask(downloadTask::run); - - final List headers = Lists.newArrayList(previousCheckpointHeader.get()); - try { - headers.addAll(headerFuture.get()); - } catch (final InterruptedException | ExecutionException e) { - result.get().completeExceptionally(e); - return Optional.empty(); - } - headers.add(nextCheckpointHeader); - if (headers.size() > 2) { - LOG.debug( - "Downloaded headers {} to {}", - headers.get(1).getNumber(), - headers.get(headers.size() - 1).getNumber()); - } - return Optional.of(headers); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelExtractTxSignaturesTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelExtractTxSignaturesTask.java deleted file mode 100644 index 9326a5eefc..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelExtractTxSignaturesTask.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPipelinedTask; -import tech.pegasys.pantheon.ethereum.eth.sync.BlockHandler; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -class ParallelExtractTxSignaturesTask extends AbstractPipelinedTask, List> { - private static final Logger LOG = LogManager.getLogger(); - - private final BlockHandler blockHandler; - - ParallelExtractTxSignaturesTask( - final BlockHandler blockHandler, - final BlockingQueue> inboundQueue, - final int outboundBacklogSize, - final MetricsSystem metricsSystem) { - super(inboundQueue, outboundBacklogSize, metricsSystem); - this.blockHandler = blockHandler; - } - - @Override - protected Optional> processStep( - final List bodies, final Optional> previousBodies) { - LOG.trace( - "Calculating fields for transactions between {} to {}", - blockHandler.extractBlockNumber(bodies.get(0)), - blockHandler.extractBlockNumber(bodies.get(bodies.size() - 1))); - - try { - blockHandler.executeParallelCalculations(bodies).get(); - } catch (final InterruptedException | ExecutionException e) { - result.get().completeExceptionally(e); - return Optional.empty(); - } - LOG.debug( - "Calculated fields for transactions between {} to {}", - blockHandler.extractBlockNumber(bodies.get(0)), - blockHandler.extractBlockNumber(bodies.get(bodies.size() - 1))); - return Optional.of(bodies); - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelImportChainSegmentTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelImportChainSegmentTask.java deleted file mode 100644 index eb7bd2ece6..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelImportChainSegmentTask.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthScheduler; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractEthTask; -import tech.pegasys.pantheon.ethereum.eth.sync.BlockHandler; -import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.function.BiConsumer; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ParallelImportChainSegmentTask extends AbstractEthTask> { - private static final Logger LOG = LogManager.getLogger(); - - private final EthContext ethContext; - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - - private final ArrayBlockingQueue checkpointHeaders; - private final int maxActiveChunks; - private final long firstHeaderNumber; - private final long lastHeaderNumber; - - private final BlockHandler blockHandler; - private final ValidationPolicy validationPolicy; - private final MetricsSystem metricsSystem; - - private ParallelImportChainSegmentTask( - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final int maxActiveChunks, - final List checkpointHeaders, - final BlockHandler blockHandler, - final ValidationPolicy validationPolicy, - final MetricsSystem metricsSystem) { - super(metricsSystem); - this.ethContext = ethContext; - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.maxActiveChunks = maxActiveChunks; - this.metricsSystem = metricsSystem; - - if (checkpointHeaders.size() > 1) { - this.firstHeaderNumber = checkpointHeaders.get(0).getNumber(); - this.lastHeaderNumber = checkpointHeaders.get(checkpointHeaders.size() - 1).getNumber(); - } else { - this.firstHeaderNumber = -1; - this.lastHeaderNumber = -1; - } - this.checkpointHeaders = - new ArrayBlockingQueue<>(checkpointHeaders.size(), false, checkpointHeaders); - this.blockHandler = blockHandler; - this.validationPolicy = validationPolicy; - } - - public static ParallelImportChainSegmentTask forCheckpoints( - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final int maxActiveChunks, - final BlockHandler blockHandler, - final ValidationPolicy validationPolicy, - final List checkpointHeaders, - final MetricsSystem metricsSystem) { - return new ParallelImportChainSegmentTask<>( - protocolSchedule, - protocolContext, - ethContext, - maxActiveChunks, - checkpointHeaders, - blockHandler, - validationPolicy, - metricsSystem); - } - - @Override - protected void executeTask() { - if (firstHeaderNumber >= 0) { - LOG.debug("Importing chain segment from {} to {}.", firstHeaderNumber, lastHeaderNumber); - - // build pipeline - final ParallelDownloadHeadersTask downloadHeadersTask = - new ParallelDownloadHeadersTask<>( - checkpointHeaders, - maxActiveChunks, - protocolSchedule, - protocolContext, - ethContext, - validationPolicy, - metricsSystem); - final ParallelValidateHeadersTask validateHeadersTask = - new ParallelValidateHeadersTask<>( - validationPolicy, - downloadHeadersTask.getOutboundQueue(), - maxActiveChunks, - protocolSchedule, - protocolContext, - metricsSystem); - final ParallelDownloadBodiesTask downloadBodiesTask = - new ParallelDownloadBodiesTask<>( - blockHandler, validateHeadersTask.getOutboundQueue(), maxActiveChunks, metricsSystem); - final ParallelExtractTxSignaturesTask extractTxSignaturesTask = - new ParallelExtractTxSignaturesTask<>( - blockHandler, downloadBodiesTask.getOutboundQueue(), maxActiveChunks, metricsSystem); - final ParallelValidateAndImportBodiesTask validateAndImportBodiesTask = - new ParallelValidateAndImportBodiesTask<>( - blockHandler, - extractTxSignaturesTask.getOutboundQueue(), - Integer.MAX_VALUE, - metricsSystem); - - // Start the pipeline. - final EthScheduler scheduler = ethContext.getScheduler(); - final CompletableFuture downloadHeaderFuture = - scheduler.scheduleServiceTask(downloadHeadersTask); - registerSubTask(downloadHeaderFuture); - final CompletableFuture validateHeaderFuture = - scheduler.scheduleServiceTask(validateHeadersTask); - registerSubTask(validateHeaderFuture); - final CompletableFuture downloadBodiesFuture = - scheduler.scheduleServiceTask(downloadBodiesTask); - registerSubTask(downloadBodiesFuture); - final CompletableFuture extractTxSignaturesFuture = - scheduler.scheduleServiceTask(extractTxSignaturesTask); - registerSubTask(extractTxSignaturesFuture); - final CompletableFuture>> validateBodiesFuture = - scheduler.scheduleServiceTask(validateAndImportBodiesTask); - registerSubTask(validateBodiesFuture); - - // Hook in pipeline completion signaling. - downloadHeadersTask.shutdown(); - downloadHeaderFuture.thenRun(validateHeadersTask::shutdown); - validateHeaderFuture.thenRun(downloadBodiesTask::shutdown); - downloadBodiesFuture.thenRun(extractTxSignaturesTask::shutdown); - extractTxSignaturesFuture.thenRun(validateAndImportBodiesTask::shutdown); - - final BiConsumer cancelOnException = - (s, e) -> { - if (e != null && !(e instanceof CancellationException)) { - downloadHeadersTask.cancel(); - validateHeadersTask.cancel(); - downloadBodiesTask.cancel(); - extractTxSignaturesTask.cancel(); - validateAndImportBodiesTask.cancel(); - result.get().completeExceptionally(e); - } - }; - - downloadHeaderFuture.whenComplete(cancelOnException); - validateHeaderFuture.whenComplete(cancelOnException); - downloadBodiesFuture.whenComplete(cancelOnException); - extractTxSignaturesFuture.whenComplete(cancelOnException); - validateBodiesFuture.whenComplete( - (r, e) -> { - if (e != null) { - cancelOnException.accept(null, e); - } else if (r != null) { - try { - final List importedBlocks = - validateBodiesFuture.get().stream() - .flatMap(Collection::stream) - .collect(Collectors.toList()); - result.get().complete(importedBlocks); - } catch (final InterruptedException | ExecutionException ex) { - result.get().completeExceptionally(ex); - } - } - }); - - } else { - LOG.warn("Import task requested with no checkpoint headers."); - } - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelValidateAndImportBodiesTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelValidateAndImportBodiesTask.java deleted file mode 100644 index 0f1876beea..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelValidateAndImportBodiesTask.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPipelinedTask; -import tech.pegasys.pantheon.ethereum.eth.sync.BlockHandler; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ParallelValidateAndImportBodiesTask - extends AbstractPipelinedTask, List> { - private static final Logger LOG = LogManager.getLogger(); - - private final BlockHandler blockHandler; - - ParallelValidateAndImportBodiesTask( - final BlockHandler blockHandler, - final BlockingQueue> inboundQueue, - final int outboundBacklogSize, - final MetricsSystem metricsSystem) { - super(inboundQueue, outboundBacklogSize, metricsSystem); - - this.blockHandler = blockHandler; - } - - @Override - protected Optional> processStep( - final List blocks, final Optional> previousBlocks) { - final long firstBlock = blockHandler.extractBlockNumber(blocks.get(0)); - final long lastBlock = blockHandler.extractBlockNumber(blocks.get(blocks.size() - 1)); - LOG.debug("Starting import of chain segment {} to {}", firstBlock, lastBlock); - final CompletableFuture> importedBlocksFuture = - blockHandler.validateAndImportBlocks(blocks); - try { - final List downloadedHashes = - importedBlocksFuture.get().stream() - .map(blockHandler::extractBlockHash) - .collect(Collectors.toList()); - LOG.info("Completed importing chain segment {} to {}", firstBlock, lastBlock); - return Optional.of(downloadedHashes); - } catch (final InterruptedException | ExecutionException e) { - failExceptionally(e); - return Optional.empty(); - } - } -} diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelValidateHeadersTask.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelValidateHeadersTask.java deleted file mode 100644 index 31c3778c92..0000000000 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ParallelValidateHeadersTask.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPipelinedTask; -import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy; -import tech.pegasys.pantheon.ethereum.eth.sync.tasks.exceptions.InvalidBlockException; -import tech.pegasys.pantheon.ethereum.mainnet.BlockHeaderValidator; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSpec; -import tech.pegasys.pantheon.metrics.MetricsSystem; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ParallelValidateHeadersTask - extends AbstractPipelinedTask, List> { - private static final Logger LOG = LogManager.getLogger(); - - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final ValidationPolicy validationPolicy; - - ParallelValidateHeadersTask( - final ValidationPolicy validationPolicy, - final BlockingQueue> inboundQueue, - final int outboundBacklogSize, - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final MetricsSystem metricsSystem) { - super(inboundQueue, outboundBacklogSize, metricsSystem); - - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.validationPolicy = validationPolicy; - } - - @Override - protected Optional> processStep( - final List headers, final Optional> previousHeaders) { - LOG.debug( - "Validating Headers {} to {}", - headers.get(0).getNumber(), - headers.get(headers.size() - 1).getNumber()); - - final BlockHeader parentHeader = headers.get(0); - final BlockHeader childHeader = headers.get(1); - final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(childHeader.getNumber()); - final BlockHeaderValidator blockHeaderValidator = protocolSpec.getBlockHeaderValidator(); - if (blockHeaderValidator.validateHeader( - childHeader, - parentHeader, - protocolContext, - validationPolicy.getValidationModeForNextBlock())) { - LOG.debug( - "Validated Headers {} to {}", - headers.get(0).getNumber(), - headers.get(headers.size() - 1).getNumber()); - // The first header will be imported by the previous request range. - return Optional.of(headers.subList(1, headers.size())); - } else { - LOG.debug( - "Could not validate Headers {} to {}", - headers.get(0).getNumber(), - headers.get(headers.size() - 1).getNumber()); - // ignore the value, we only want the first exception to be there - failExceptionally( - new InvalidBlockException( - "Provided first header does not connect to last header.", - parentHeader.getNumber(), - parentHeader.getHash())); - return Optional.empty(); - } - } -} diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContextTestUtil.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContextTestUtil.java index f0bb526aba..26b34615ca 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContextTestUtil.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/manager/EthContextTestUtil.java @@ -20,7 +20,6 @@ public class EthContextTestUtil { public static EthContext createTestEthContext(final TimeoutPolicy timeoutPolicy) { return new EthContext( - PROTOCOL_NAME, new EthPeers(PROTOCOL_NAME), new EthMessages(), new DeterministicEthScheduler(timeoutPolicy)); diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 1b600bb76f..9f501fd787 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -580,7 +580,7 @@ public void shouldNotImportBlocksThatAreAlreadyBeingImported() { when(ethScheduler.scheduleSyncWorkerTask(any(Supplier.class))) .thenReturn(new CompletableFuture<>()); final EthContext ethContext = - new EthContext("eth", new EthPeers("eth"), new EthMessages(), ethScheduler); + new EthContext(new EthPeers("eth"), new EthMessages(), ethScheduler); final BlockPropagationManager blockPropagationManager = new BlockPropagationManager<>( syncConfig, diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/CheckpointHeaderManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/CheckpointHeaderManagerTest.java deleted file mode 100644 index b701c42f4f..0000000000 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/CheckpointHeaderManagerTest.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync; - -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.when; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.BlockHeaderTestFixture; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.eth.manager.ChainState; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContextTestUtil; -import tech.pegasys.pantheon.ethereum.eth.manager.EthMessage; -import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.RequestManager; -import tech.pegasys.pantheon.ethereum.eth.manager.RequestManager.ResponseStream; -import tech.pegasys.pantheon.ethereum.eth.messages.BlockHeadersMessage; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncTarget; -import tech.pegasys.pantheon.ethereum.mainnet.MainnetProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection.PeerNotConnected; -import tech.pegasys.pantheon.ethereum.worldstate.WorldStateArchive; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; - -import java.util.concurrent.atomic.AtomicBoolean; - -import org.junit.Before; -import org.junit.Test; - -public class CheckpointHeaderManagerTest { - - private static final BlockHeader GENESIS = block(0); - private static final int SEGMENT_SIZE = 5; - private static final int HEADER_REQUEST_SIZE = 3; - - private static final ProtocolSchedule PROTOCOL_SCHEDULE = MainnetProtocolSchedule.create(); - - private final MutableBlockchain blockchain = mock(MutableBlockchain.class); - private final WorldStateArchive worldStateArchive = mock(WorldStateArchive.class); - private final ProtocolContext protocolContext = - new ProtocolContext<>(blockchain, worldStateArchive, null); - - private final AtomicBoolean timeout = new AtomicBoolean(false); - private final EthContext ethContext = EthContextTestUtil.createTestEthContext(timeout::get); - private final SyncState syncState = new SyncState(blockchain, ethContext.getEthPeers()); - private final MetricsSystem metricsSystem = new NoOpMetricsSystem(); - private final EthPeer syncTargetPeer = mock(EthPeer.class); - private final RequestManager requestManager = new RequestManager(syncTargetPeer); - private SyncTarget syncTarget; - - private final CheckpointHeaderManager checkpointHeaderManager = - new CheckpointHeaderManager<>( - SynchronizerConfiguration.builder() - .downloaderChainSegmentSize(SEGMENT_SIZE) - .downloaderHeadersRequestSize(HEADER_REQUEST_SIZE) - .downloaderCheckpointTimeoutsPermitted(2) - .build(), - protocolContext, - ethContext, - syncState, - PROTOCOL_SCHEDULE, - metricsSystem); - - @Before - public void setUp() { - when(syncTargetPeer.chainState()).thenReturn(new ChainState()); - syncTarget = syncState.setSyncTarget(syncTargetPeer, GENESIS); - } - - @Test - public void shouldHandleErrorsWhenRequestingHeaders() throws Exception { - when(anyHeadersRequested()).thenThrow(new PeerNotConnected("Nope")); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(emptyList()); - } - - @Test - public void shouldHandleTimeouts() throws Exception { - timeout.set(true); - when(anyHeadersRequested()).thenReturn(createResponseStream(), createResponseStream()); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(emptyList()); - assertThat(checkpointHeaderManager.checkpointsHaveTimedOut()).isFalse(); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(emptyList()); - assertThat(checkpointHeaderManager.checkpointsHaveTimedOut()).isTrue(); - } - - @Test - public void shouldResetTimeoutWhenHeadersReceived() throws Exception { - // Timeout - timeout.set(true); - when(anyHeadersRequested()).thenReturn(createResponseStream()); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(emptyList()); - assertThat(checkpointHeaderManager.checkpointsHaveTimedOut()).isFalse(); - - // Receive response - reset(syncTargetPeer); - respondToHeaderRequests(GENESIS, block(5)); - timeout.set(false); - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(GENESIS, block(5))); - assertThat(checkpointHeaderManager.checkpointsHaveTimedOut()).isFalse(); - - // Timeout again but shouldn't have reached threshold - reset(syncTargetPeer); - timeout.set(true); - when(anyHeadersRequested()).thenReturn(createResponseStream()); - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(GENESIS, block(5))); - assertThat(checkpointHeaderManager.checkpointsHaveTimedOut()).isFalse(); - } - - @Test - public void shouldUseReturnedHeadersAsCheckpointHeaders() throws Exception { - respondToHeaderRequests(GENESIS, block(5), block(10)); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(GENESIS, block(5), block(10))); - } - - @Test - public void shouldPullAdditionalCheckpointsWhenRequired() throws Exception { - respondToHeaderRequests(GENESIS, block(5)); - respondToHeaderRequests(block(5), block(10), block(15), block(20)); - - // Pull initial headers - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(GENESIS, block(5))); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(GENESIS, block(5), block(10), block(15), block(20))); - } - - @Test - public void shouldRemoveImportedCheckpointHeaders() throws Exception { - respondToHeaderRequests(GENESIS, block(5), block(10)); - respondToHeaderRequests(block(10)); - - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(GENESIS, block(5), block(10))); - - when(blockchain.contains(GENESIS.getHash())).thenReturn(true); - when(blockchain.contains(block(5).getHash())).thenReturn(true); - when(blockchain.contains(block(10).getHash())).thenReturn(false); - checkpointHeaderManager.clearImportedCheckpointHeaders(); - - // The first checkpoint header should always be in the blockchain (just as geneis was present) - assertThat(checkpointHeaderManager.pullCheckpointHeaders(syncTarget)) - .isCompletedWithValue(asList(block(5), block(10))); - } - - private void respondToHeaderRequests(final BlockHeader... headers) throws Exception { - final ResponseStream responseStream = createResponseStream(); - when(syncTargetPeer.getHeadersByHash( - headers[0].getHash(), HEADER_REQUEST_SIZE + 1, SEGMENT_SIZE - 1, false)) - .thenReturn(responseStream); - requestManager.dispatchResponse( - new EthMessage(syncTargetPeer, BlockHeadersMessage.create(asList(headers)))); - } - - private static BlockHeader block(final int blockNumber) { - return new BlockHeaderTestFixture().number(blockNumber).buildHeader(); - } - - private ResponseStream createResponseStream() throws PeerNotConnected { - return requestManager.dispatchRequest(() -> {}); - } - - private ResponseStream anyHeadersRequested() throws PeerNotConnected { - return syncTargetPeer.getHeadersByHash(any(Hash.class), anyInt(), anyInt(), anyBoolean()); - } -} diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandlerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandlerTest.java deleted file mode 100644 index 3e41a6582c..0000000000 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandlerTest.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; - -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; -import static tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode.LIGHT; -import static tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode.LIGHT_SKIP_DETACHED; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockBody; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.BlockHeaderTestFixture; -import tech.pegasys.pantheon.ethereum.core.BlockImporter; -import tech.pegasys.pantheon.ethereum.eth.manager.DeterministicEthScheduler; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthMessages; -import tech.pegasys.pantheon.ethereum.eth.manager.EthPeers; -import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy; -import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSpec; -import tech.pegasys.pantheon.ethereum.worldstate.WorldStateArchive; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class FastSyncBlockHandlerTest { - private static final String PROTOCOL_NAME = "ETH"; - private static final BlockBody EMPTY_BODY = new BlockBody(emptyList(), emptyList()); - private static final BlockHeader HEADER = new BlockHeaderTestFixture().buildHeader(); - private static final Block BLOCK = new Block(HEADER, EMPTY_BODY); - private static final Block BLOCK2 = - new Block(new BlockHeaderTestFixture().number(2).buildHeader(), EMPTY_BODY); - private static final Block BLOCK3 = - new Block(new BlockHeaderTestFixture().number(3).buildHeader(), EMPTY_BODY); - private static final HeaderValidationMode VALIDATION_MODE = LIGHT_SKIP_DETACHED; - private static final HeaderValidationMode OMMER_VALIDATION_MODE = LIGHT; - - @SuppressWarnings("unchecked") - private final ProtocolSchedule protocolSchedule = mock(ProtocolSchedule.class); - - @SuppressWarnings("unchecked") - private final ProtocolSpec protocolSpec = mock(ProtocolSpec.class); - - @SuppressWarnings("unchecked") - private final BlockImporter blockImporter = mock(BlockImporter.class); - - private final MutableBlockchain blockchain = mock(MutableBlockchain.class); - private final WorldStateArchive worldStateArchive = mock(WorldStateArchive.class); - private final ProtocolContext protocolContext = - new ProtocolContext<>(blockchain, worldStateArchive, null); - private final EthContext ethContext = - new EthContext( - PROTOCOL_NAME, - new EthPeers(PROTOCOL_NAME), - new EthMessages(), - new DeterministicEthScheduler()); - private final MetricsSystem metricsSystem = new NoOpMetricsSystem(); - private final ValidationPolicy validationPolicy = mock(ValidationPolicy.class); - private final ValidationPolicy ommerValidationPolicy = mock(ValidationPolicy.class); - - private final FastSyncBlockHandler blockHandler = - new FastSyncBlockHandler<>( - protocolSchedule, - protocolContext, - ethContext, - metricsSystem, - validationPolicy, - ommerValidationPolicy); - - @Before - public void setUp() { - when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(protocolSpec); - when(protocolSpec.getBlockImporter()).thenReturn(blockImporter); - when(validationPolicy.getValidationModeForNextBlock()).thenReturn(VALIDATION_MODE); - when(ommerValidationPolicy.getValidationModeForNextBlock()).thenReturn(OMMER_VALIDATION_MODE); - } - - @After - public void checkWorldStateIsUnused() { - verifyZeroInteractions(worldStateArchive); - } - - @Test - public void shouldFastImportBlocks() { - when(blockImporter.fastImportBlock( - protocolContext, BLOCK, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE)) - .thenReturn(true); - final List blocksWithReceipts = - singletonList(new BlockWithReceipts(BLOCK, emptyList())); - - final CompletableFuture> result = - blockHandler.validateAndImportBlocks(blocksWithReceipts); - - assertThat(result).isCompleted(); - verify(blockImporter) - .fastImportBlock( - protocolContext, BLOCK, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE); - } - - @Test - public void shouldReturnExceptionallyCompletedFutureWhenBlockImportFails() { - when(blockImporter.fastImportBlock( - protocolContext, BLOCK, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE)) - .thenReturn(false); - - final CompletableFuture> result = - blockHandler.validateAndImportBlocks( - singletonList(new BlockWithReceipts(BLOCK, emptyList()))); - - assertThat(result).isCompletedExceptionally(); - } - - @Test - public void shouldNotContinueImportingBlocksAfterValidationFailure() { - when(blockImporter.fastImportBlock( - protocolContext, BLOCK, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE)) - .thenReturn(true); - when(blockImporter.fastImportBlock( - protocolContext, BLOCK2, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE)) - .thenReturn(false); - - final CompletableFuture> result = - blockHandler.validateAndImportBlocks( - asList( - new BlockWithReceipts(BLOCK, emptyList()), - new BlockWithReceipts(BLOCK2, emptyList()), - new BlockWithReceipts(BLOCK3, emptyList()))); - - assertThat(result).isCompletedExceptionally(); - - verify(blockImporter) - .fastImportBlock( - protocolContext, BLOCK, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE); - verify(blockImporter) - .fastImportBlock( - protocolContext, BLOCK2, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE); - verify(blockImporter, never()) - .fastImportBlock( - protocolContext, BLOCK3, emptyList(), VALIDATION_MODE, OMMER_VALIDATION_MODE); - } -} diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointHeaderManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointHeaderManagerTest.java deleted file mode 100644 index dcf8a5f3ee..0000000000 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncCheckpointHeaderManagerTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fastsync; - -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.spy; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.Blockchain; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthProtocolManager; -import tech.pegasys.pantheon.ethereum.eth.manager.EthProtocolManagerTestUtil; -import tech.pegasys.pantheon.ethereum.eth.manager.RespondingEthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.RespondingEthPeer.Responder; -import tech.pegasys.pantheon.ethereum.eth.manager.ethtaskutils.BlockchainSetupUtil; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncTarget; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import org.junit.Before; -import org.junit.Test; - -public class FastSyncCheckpointHeaderManagerTest { - - protected ProtocolSchedule protocolSchedule; - protected EthProtocolManager ethProtocolManager; - protected EthContext ethContext; - protected ProtocolContext protocolContext; - private SyncState syncState; - - private BlockchainSetupUtil localBlockchainSetup; - protected MutableBlockchain localBlockchain; - private BlockchainSetupUtil otherBlockchainSetup; - protected Blockchain otherBlockchain; - MetricsSystem metricsSystem = new NoOpMetricsSystem();; - private BlockHeader pivotBlockHeader; - private FastSyncCheckpointHeaderManager checkpointHeaderManager; - private RespondingEthPeer peer; - - @Before - public void setupTest() { - localBlockchainSetup = BlockchainSetupUtil.forTesting(); - localBlockchain = spy(localBlockchainSetup.getBlockchain()); - otherBlockchainSetup = BlockchainSetupUtil.forTesting(); - otherBlockchain = otherBlockchainSetup.getBlockchain(); - - protocolSchedule = localBlockchainSetup.getProtocolSchedule(); - protocolContext = localBlockchainSetup.getProtocolContext(); - ethProtocolManager = - EthProtocolManagerTestUtil.create(localBlockchain, localBlockchainSetup.getWorldArchive()); - ethContext = ethProtocolManager.ethContext(); - syncState = new SyncState(protocolContext.getBlockchain(), ethContext.getEthPeers()); - - otherBlockchainSetup.importFirstBlocks(30); - - pivotBlockHeader = block(17); - - peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localBlockchain); - - checkpointHeaderManager = - new FastSyncCheckpointHeaderManager<>( - SynchronizerConfiguration.builder() - .downloaderChainSegmentSize(5) - .downloaderHeadersRequestSize(5) - .build(), - protocolContext, - ethContext, - syncState, - protocolSchedule, - metricsSystem, - pivotBlockHeader); - } - - @Test - public void shouldNotRequestCheckpointHeadersBeyondThePivotBlock() { - final SyncTarget syncTarget = syncState.setSyncTarget(peer.getEthPeer(), block(10)); - assertCheckpointHeaders(syncTarget, asList(block(10), block(15), pivotBlockHeader)); - } - - @Test - public void shouldNotDuplicatePivotBlockAsCheckpoint() { - final SyncTarget syncTarget = syncState.setSyncTarget(peer.getEthPeer(), block(7)); - assertCheckpointHeaders(syncTarget, asList(block(7), block(12), pivotBlockHeader)); - } - - @Test - public void shouldHaveNoCheckpointsWhenCommonAncestorIsPivotBlock() { - final SyncTarget syncTarget = - syncState.setSyncTarget(peer.getEthPeer(), block(pivotBlockHeader.getNumber())); - assertCheckpointHeaders(syncTarget, emptyList()); - } - - @Test - public void shouldHaveNoCheckpointsWhenCommonAncestorIsAfterPivotBlock() { - final SyncTarget syncTarget = - syncState.setSyncTarget(peer.getEthPeer(), block(pivotBlockHeader.getNumber() + 1)); - assertCheckpointHeaders(syncTarget, emptyList()); - } - - @Test - public void shouldHaveCommonAncestorAndPivotBlockWhenCommonAncestorImmediatelyBeforePivotBlock() { - final BlockHeader commonAncestor = block(pivotBlockHeader.getNumber() - 1); - final SyncTarget syncTarget = syncState.setSyncTarget(peer.getEthPeer(), commonAncestor); - assertCheckpointHeaders(syncTarget, asList(commonAncestor, pivotBlockHeader)); - } - - private void assertCheckpointHeaders( - final SyncTarget syncTarget, final List expected) { - final CompletableFuture> result = - checkpointHeaderManager.pullCheckpointHeaders(syncTarget); - - final Responder responder = RespondingEthPeer.blockchainResponder(otherBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(result).isCompletedWithValue(expected); - } - - private BlockHeader block(final long blockNumber) { - return otherBlockchain.getBlockHeader(blockNumber).get(); - } -} diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java index 12ce030ae2..7318a456ce 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncChainDownloaderTest.java @@ -13,7 +13,6 @@ package tech.pegasys.pantheon.ethereum.eth.sync.fullsync; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assumptions.assumeThatObject; import static org.awaitility.Awaitility.await; import static tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider.createInMemoryBlockchain; @@ -23,7 +22,6 @@ import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.BlockBody; import tech.pegasys.pantheon.ethereum.core.BlockDataGenerator; -import tech.pegasys.pantheon.ethereum.core.BlockDataGenerator.BlockOptions; import tech.pegasys.pantheon.ethereum.core.BlockHeader; import tech.pegasys.pantheon.ethereum.core.TransactionReceipt; import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; @@ -36,7 +34,6 @@ import tech.pegasys.pantheon.ethereum.eth.messages.EthPV62; import tech.pegasys.pantheon.ethereum.eth.messages.GetBlockHeadersMessage; import tech.pegasys.pantheon.ethereum.eth.sync.ChainDownloader; -import tech.pegasys.pantheon.ethereum.eth.sync.EthTaskChainDownloader; import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; @@ -49,7 +46,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -57,24 +53,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; -@RunWith(Parameterized.class) public class FullSyncChainDownloaderTest { - @Parameter public boolean usePipelineDownloader; - - @Parameter(1) - public String name; - - @Parameters(name = "{1}") - public static Object[][] params() { - return new Object[][] {{false, "EthTask"}, {true, "Pipeline"}}; - } - protected ProtocolSchedule protocolSchedule; protected EthProtocolManager ethProtocolManager; protected EthContext ethContext; @@ -123,8 +104,7 @@ private ChainDownloader downloader() { } private SynchronizerConfiguration.Builder syncConfigBuilder() { - return SynchronizerConfiguration.builder() - .piplineDownloaderForFullSyncEnabled(usePipelineDownloader); + return SynchronizerConfiguration.builder(); } @Test @@ -306,149 +286,6 @@ public void choosesBestPeerAsSyncTarget_byTdAndHeight() { assertThat(syncState.syncTarget().get().peer()).isEqualTo(peerB.getEthPeer()); } - @Test - public void switchesSyncTarget_betterHeight() { - final UInt256 localTd = localBlockchain.getChainHead().getTotalDifficulty(); - final Responder responder = RespondingEthPeer.blockchainResponder(otherBlockchain); - - // Peer A is initially better - final RespondingEthPeer peerA = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(100), 60); - final RespondingEthPeer peerB = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(100), 50); - - final SynchronizerConfiguration syncConfig = - syncConfigBuilder() - .downloaderChainSegmentSize(5) - .downloaderChangeTargetThresholdByHeight(10) - .build(); - final ChainDownloader downloader = downloader(syncConfig); - downloader.start(); - - // Process until the sync target is selected - peerA.respondWhileOtherThreadsWork(responder, () -> !syncState.syncTarget().isPresent()); - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(peerA.getEthPeer()); - - // Update Peer B so that its a better target and send some responses to push logic forward - peerB.getEthPeer().chainState().update(gen.hash(), 100); - - processUntilSyncTargetChecked(responder, downloader, peerA, peerB); - - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(peerB.getEthPeer()); - } - - @Test - public void doesNotSwitchSyncTarget_betterHeightUnderThreshold() { - otherBlockchainSetup.importFirstBlocks(8); - final UInt256 localTd = localBlockchain.getChainHead().getTotalDifficulty(); - final Responder responder = RespondingEthPeer.blockchainResponder(otherBlockchain); - - final RespondingEthPeer bestPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(200)); - final RespondingEthPeer otherPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(100)); - - final SynchronizerConfiguration syncConfig = - syncConfigBuilder() - .downloaderChainSegmentSize(5) - .downloaderChangeTargetThresholdByHeight(1000) - .build(); - final ChainDownloader downloader = downloader(syncConfig); - downloader.start(); - - // Process until the sync target is selected - bestPeer.respondWhileOtherThreadsWork(responder, () -> !syncState.syncTarget().isPresent()); - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(bestPeer.getEthPeer()); - - // Update otherPeer so that its a better target, but under the threshold to switch - otherPeer.getEthPeer().chainState().update(gen.hash(), 100); - - processUntilSyncTargetChecked(responder, downloader, bestPeer, otherPeer); - - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(bestPeer.getEthPeer()); - } - - @Test - public void switchesSyncTarget_betterTd() { - final UInt256 localTd = localBlockchain.getChainHead().getTotalDifficulty(); - final Responder responder = RespondingEthPeer.blockchainResponder(otherBlockchain); - - // Peer A is initially better - final RespondingEthPeer peerA = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(200)); - final RespondingEthPeer peerB = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(100)); - - final SynchronizerConfiguration syncConfig = - syncConfigBuilder() - .downloaderChainSegmentSize(5) - .downloaderChangeTargetThresholdByTd(UInt256.of(10)) - .build(); - final ChainDownloader downloader = downloader(syncConfig); - downloader.start(); - - // Process until the sync target is selected - peerA.respondWhileOtherThreadsWork(responder, () -> !syncState.syncTarget().isPresent()); - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(peerA.getEthPeer()); - - // Update Peer B so that its a better target and send some responses to push logic forward - peerB - .getEthPeer() - .chainState() - .updateForAnnouncedBlock( - gen.header(), localBlockchain.getChainHead().getTotalDifficulty().plus(300)); - - processUntilSyncTargetChecked(responder, downloader, peerA, peerB); - - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(peerB.getEthPeer()); - } - - @Test - public void doesNotSwitchSyncTarget_betterTdUnderThreshold() { - final long localChainHeadAtStart = localBlockchain.getChainHeadBlockNumber(); - final UInt256 localTd = localBlockchain.getChainHead().getTotalDifficulty(); - otherBlockchainSetup.importFirstBlocks(8); - final Responder responder = RespondingEthPeer.blockchainResponder(otherBlockchain); - - // Sanity check - assertThat(localChainHeadAtStart).isLessThan(otherBlockchain.getChainHeadBlockNumber()); - - final RespondingEthPeer bestPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(200)); - final RespondingEthPeer otherPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, localTd.plus(100)); - - final SynchronizerConfiguration syncConfig = - syncConfigBuilder() - .downloaderChainSegmentSize(5) - .downloaderChangeTargetThresholdByTd(UInt256.of(100_000_000L)) - .build(); - final ChainDownloader downloader = downloader(syncConfig); - downloader.start(); - - // Process until the sync target is selected - bestPeer.respondWhileOtherThreadsWork(responder, () -> !syncState.syncTarget().isPresent()); - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(bestPeer.getEthPeer()); - - // Update otherPeer so that its a better target and send some responses to push logic forward - final BlockHeader newBestBlock = - gen.header(1000, new BlockOptions().setDifficulty(UInt256.ZERO)); - bestPeer.getEthPeer().chainState().updateForAnnouncedBlock(newBestBlock, localTd.plus(201)); - otherPeer.getEthPeer().chainState().updateForAnnouncedBlock(newBestBlock, localTd.plus(300)); - - processUntilSyncTargetChecked(responder, downloader, bestPeer, otherPeer); - - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(bestPeer.getEthPeer()); - } - @Test public void recoversFromSyncTargetDisconnect() { localBlockchainSetup.importFirstBlocks(2); @@ -608,20 +445,4 @@ private MutableBlockchain createShortChain( } return shortChain; } - - @SuppressWarnings("unchecked") - private void processUntilSyncTargetChecked( - final Responder responder, - final ChainDownloader rawDownloader, - final RespondingEthPeer... peers) { - // This breaks encapsulation and depends on the particular semantics of EthTaskChainDownloader - // These cases are now handled by unit tests in BetterSyncTargetEvaluatorTest - assumeThatObject(rawDownloader).isInstanceOf(EthTaskChainDownloader.class); - final EthTaskChainDownloader downloader = (EthTaskChainDownloader) rawDownloader; - // Process through first task cycle - final CompletableFuture firstTask = downloader.getCurrentTask(); - while (downloader.getCurrentTask() == firstTask) { - RespondingEthPeer.respondOnce(responder, peers); - } - } } diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/IncrementerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/IncrementerTest.java deleted file mode 100644 index df2a08baa7..0000000000 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/IncrementerTest.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.fullsync; - -import static org.assertj.core.api.Assertions.assertThat; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.Blockchain; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; -import tech.pegasys.pantheon.ethereum.eth.manager.EthProtocolManager; -import tech.pegasys.pantheon.ethereum.eth.manager.EthProtocolManagerTestUtil; -import tech.pegasys.pantheon.ethereum.eth.manager.EthScheduler; -import tech.pegasys.pantheon.ethereum.eth.manager.RespondingEthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.ethtaskutils.BlockchainSetupUtil; -import tech.pegasys.pantheon.ethereum.eth.sync.ChainDownloader; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.metrics.MetricCategory; -import tech.pegasys.pantheon.metrics.MetricsSystem; -import tech.pegasys.pantheon.metrics.Observation; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; -import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration; -import tech.pegasys.pantheon.metrics.prometheus.PrometheusMetricsSystem; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class IncrementerTest { - private final MetricsConfiguration metricsConfiguration = MetricsConfiguration.createDefault(); - private ProtocolSchedule protocolSchedule; - private EthContext ethContext; - private ProtocolContext protocolContext; - private SyncState syncState; - private MutableBlockchain localBlockchain; - private MetricsSystem metricsSystem; - private EthProtocolManager ethProtocolManager; - private Blockchain otherBlockchain; - private long targetBlock; - - @Before - public void setUp() { - metricsConfiguration.setEnabled(true); - metricsSystem = PrometheusMetricsSystem.init(metricsConfiguration); - - final BlockchainSetupUtil localBlockchainSetup = BlockchainSetupUtil.forTesting(); - localBlockchain = localBlockchainSetup.getBlockchain(); - final BlockchainSetupUtil otherBlockchainSetup = BlockchainSetupUtil.forTesting(); - otherBlockchain = otherBlockchainSetup.getBlockchain(); - - protocolSchedule = localBlockchainSetup.getProtocolSchedule(); - protocolContext = localBlockchainSetup.getProtocolContext(); - ethProtocolManager = - EthProtocolManagerTestUtil.create( - localBlockchain, - localBlockchainSetup.getWorldArchive(), - new EthScheduler(1, 1, 1, new NoOpMetricsSystem())); - ethContext = ethProtocolManager.ethContext(); - syncState = new SyncState(protocolContext.getBlockchain(), ethContext.getEthPeers()); - - otherBlockchainSetup.importFirstBlocks(15); - targetBlock = otherBlockchain.getChainHeadBlockNumber(); - // Sanity check - assertThat(targetBlock).isGreaterThan(localBlockchain.getChainHeadBlockNumber()); - } - - @After - public void tearDown() { - ethProtocolManager.stop(); - } - - @Test - public void parallelDownloadPipelineCounterShouldIncrement() { - final RespondingEthPeer peer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain); - final RespondingEthPeer.Responder responder = - RespondingEthPeer.blockchainResponder(otherBlockchain); - - final SynchronizerConfiguration syncConfig = - SynchronizerConfiguration.builder() - .piplineDownloaderForFullSyncEnabled(false) - .downloaderChainSegmentSize(10) - .build(); - final ChainDownloader downloader = downloader(syncConfig); - downloader.start(); - - peer.respondWhileOtherThreadsWork(responder, () -> !syncState.syncTarget().isPresent()); - assertThat(syncState.syncTarget()).isPresent(); - assertThat(syncState.syncTarget().get().peer()).isEqualTo(peer.getEthPeer()); - - peer.respondWhileOtherThreadsWork( - responder, () -> localBlockchain.getChainHeadBlockNumber() < targetBlock); - - assertThat(localBlockchain.getChainHeadBlockNumber()).isEqualTo(targetBlock); - - final List metrics = - metricsSystem.getMetrics(MetricCategory.SYNCHRONIZER).collect(Collectors.toList()); - - // the first iteration gets the genesis block, which results in no data - // being passed downstream. So observed value is 2. - final Observation headerInboundObservation = - new Observation( - MetricCategory.SYNCHRONIZER, - "inboundQueueCounter", - 2.0, - Collections.singletonList("ParallelDownloadHeadersTask")); - final Observation headerOutboundObservation = - new Observation( - MetricCategory.SYNCHRONIZER, - "outboundQueueCounter", - 1.0, - Collections.singletonList("ParallelDownloadHeadersTask")); - assertThat(metrics).contains(headerInboundObservation, headerOutboundObservation); - - for (final String label : - Arrays.asList( - "ParallelValidateHeadersTask", - "ParallelDownloadBodiesTask", - "ParallelExtractTxSignaturesTask", - "ParallelValidateAndImportBodiesTask")) { - final Observation inboundObservation = - new Observation( - MetricCategory.SYNCHRONIZER, - "inboundQueueCounter", - 1.0, - Collections.singletonList(label)); - final Observation outboundObservation = - new Observation( - MetricCategory.SYNCHRONIZER, - "outboundQueueCounter", - 1.0, - Collections.singletonList(label)); - assertThat(metrics).contains(inboundObservation, outboundObservation); - } - } - - private ChainDownloader downloader(final SynchronizerConfiguration syncConfig) { - return FullSyncChainDownloader.create( - syncConfig, protocolSchedule, protocolContext, ethContext, syncState, metricsSystem); - } -} diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ImportBlocksTaskTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ImportBlocksTaskTest.java deleted file mode 100644 index b7579c3873..0000000000 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/ImportBlocksTaskTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.ethereum.eth.sync.tasks; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider.createInMemoryBlockchain; - -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockBody; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.core.TransactionReceipt; -import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.EthProtocolManagerTestUtil; -import tech.pegasys.pantheon.ethereum.eth.manager.RespondingEthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.RespondingEthPeer.Responder; -import tech.pegasys.pantheon.ethereum.eth.manager.ethtaskutils.AbstractMessageTaskTest; -import tech.pegasys.pantheon.ethereum.eth.manager.exceptions.NoAvailablePeersException; -import tech.pegasys.pantheon.ethereum.eth.manager.task.AbstractPeerTask.PeerTaskResult; -import tech.pegasys.pantheon.ethereum.eth.manager.task.EthTask; -import tech.pegasys.pantheon.ethereum.eth.messages.BlockHeadersMessage; -import tech.pegasys.pantheon.ethereum.eth.messages.EthPV62; -import tech.pegasys.pantheon.ethereum.p2p.api.MessageData; -import tech.pegasys.pantheon.ethereum.p2p.wire.Capability; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; - -import com.google.common.collect.Lists; -import org.junit.Test; - -public class ImportBlocksTaskTest - extends AbstractMessageTaskTest, PeerTaskResult>> { - - @Override - protected List generateDataToBeRequested() { - final long chainHead = blockchain.getChainHeadBlockNumber(); - final long importSize = 5; - final long startNumber = chainHead - importSize + 1; - final List blocksToImport = new ArrayList<>(); - for (long i = 0; i < importSize; i++) { - final BlockHeader header = blockchain.getBlockHeader(startNumber + i).get(); - final BlockBody body = blockchain.getBlockBody(header.getHash()).get(); - blocksToImport.add(new Block(header, body)); - } - return blocksToImport; - } - - @Override - protected EthTask>> createTask(final List requestedData) { - final Block firstBlock = requestedData.get(0); - final MutableBlockchain shortBlockchain = - createShortChain(firstBlock.getHeader().getNumber() - 1); - final ProtocolContext modifiedContext = - new ProtocolContext<>( - shortBlockchain, - protocolContext.getWorldStateArchive(), - protocolContext.getConsensusState()); - return ImportBlocksTask.fromHeader( - protocolSchedule, - modifiedContext, - ethContext, - firstBlock.getHeader(), - requestedData.size(), - metricsSystem); - } - - @Override - protected void assertResultMatchesExpectation( - final List requestedData, - final PeerTaskResult> response, - final EthPeer respondingPeer) { - assertThat(response.getResult()) - .isEqualTo(requestedData.stream().map(Block::getHash).collect(Collectors.toList())); - assertThat(response.getPeer()).isEqualTo(respondingPeer); - } - - @Test - public void completesWhenPeerReturnsPartialResult() { - - // Respond with some headers and all corresponding bodies - final Responder fullResponder = RespondingEthPeer.blockchainResponder(blockchain); - final Responder partialResponder = - (final Capability cap, final MessageData msg) -> { - final Optional fullReponse = fullResponder.respond(cap, msg); - if (msg.getCode() == EthPV62.GET_BLOCK_HEADERS) { - // Return a partial headers response - final BlockHeadersMessage headersMessage = - BlockHeadersMessage.readFrom(fullReponse.get()); - final List originalHeaders = - Lists.newArrayList(headersMessage.getHeaders(protocolSchedule)); - final List partialHeaders = - originalHeaders.subList(0, originalHeaders.size() / 2); - return Optional.of(BlockHeadersMessage.create(partialHeaders)); - } - return fullReponse; - }; - - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager); - - // Execute task - final AtomicReference> actualResult = new AtomicReference<>(); - final AtomicReference actualPeer = new AtomicReference<>(); - final AtomicBoolean done = new AtomicBoolean(false); - final List requestedData = generateDataToBeRequested(); - final List requestedHashes = - requestedData.stream().map(Block::getHash).collect(Collectors.toList()); - final EthTask>> task = createTask(requestedData); - final CompletableFuture>> future = task.run(); - future.whenComplete( - (response, error) -> { - actualResult.set(response.getResult()); - actualPeer.set(response.getPeer()); - done.compareAndSet(false, true); - }); - - // Send partial responses - peer.respondWhile(partialResponder, () -> !future.isDone()); - - assertThat(done).isTrue(); - assertThat(actualPeer.get()).isEqualTo(peer.getEthPeer()); - assertThat(actualResult.get().size()).isLessThan(requestedData.size()); - for (final Hash hash : actualResult.get()) { - assertThat(requestedHashes).contains(hash); - assertThat(blockchain.contains(hash)).isTrue(); - } - } - - @Test - public void completesWhenPeersSendEmptyResponses() { - // Setup a unresponsive peer - final Responder responder = RespondingEthPeer.emptyResponder(); - final RespondingEthPeer respondingEthPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager); - - // Execute task and wait for response - final AtomicBoolean done = new AtomicBoolean(false); - final List requestedData = generateDataToBeRequested(); - final EthTask>> task = createTask(requestedData); - final CompletableFuture>> future = task.run(); - respondingEthPeer.respondWhile(responder, () -> !future.isDone()); - future.whenComplete((response, error) -> done.compareAndSet(false, true)); - assertThat(future.isDone()).isTrue(); - assertThat(future.isCompletedExceptionally()).isFalse(); - } - - @Test - public void shouldNotRequestDataFromPeerBelowExpectedHeight() { - // Setup a unresponsive peer - final Responder responder = RespondingEthPeer.emptyResponder(); - final RespondingEthPeer respondingEthPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 1); - - // Execute task and wait for response - final List requestedData = generateDataToBeRequested(); - final EthTask>> task = createTask(requestedData); - final CompletableFuture>> future = task.run(); - respondingEthPeer.respondWhile(responder, () -> !future.isDone()); - assertThat(future.isDone()).isTrue(); - assertThat(future.isCompletedExceptionally()).isTrue(); - assertThatThrownBy(future::get).hasCauseInstanceOf(NoAvailablePeersException.class); - } - - private MutableBlockchain createShortChain(final long truncateAtBlockNumber) { - final BlockHeader genesisHeader = - blockchain.getBlockHeader(BlockHeader.GENESIS_BLOCK_NUMBER).get(); - final BlockBody genesisBody = blockchain.getBlockBody(genesisHeader.getHash()).get(); - final Block genesisBlock = new Block(genesisHeader, genesisBody); - final MutableBlockchain shortChain = createInMemoryBlockchain(genesisBlock); - long nextBlock = genesisHeader.getNumber() + 1; - while (nextBlock <= truncateAtBlockNumber) { - final BlockHeader header = blockchain.getBlockHeader(nextBlock).get(); - final BlockBody body = blockchain.getBlockBody(header.getHash()).get(); - final List receipts = blockchain.getTxReceipts(header.getHash()).get(); - final Block block = new Block(header, body); - shortChain.appendBlock(block, receipts); - nextBlock++; - } - return shortChain; - } -}