From 476da339905139a07393bf37fb549800066a6f4a Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 7 Feb 2019 10:44:57 -0500 Subject: [PATCH 01/11] o --- .../eth/manager/EthProtocolManager.java | 16 +- .../eth/sync/SynchronizerConfiguration.java | 46 ++- .../eth/sync/BlockPropagationManagerTest.java | 367 +++++++++++++++++- 3 files changed, 390 insertions(+), 39 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java index 53373518f6..06263566e5 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java @@ -44,7 +44,7 @@ import org.apache.logging.log4j.Logger; public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { - static final int DEFAULT_REQUEST_LIMIT = 200; + public static final int DEFAULT_REQUEST_LIMIT = 200; private static final Logger LOG = LogManager.getLogger(); private static final List FAST_SYNC_CAPS = Collections.singletonList(EthProtocol.ETH63); @@ -64,13 +64,13 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { private List supportedCapabilities; private final Blockchain blockchain; - EthProtocolManager( - final Blockchain blockchain, - final WorldStateArchive worldStateArchive, - final int networkId, - final boolean fastSyncEnabled, - final int requestLimit, - final EthScheduler scheduler) { + public EthProtocolManager( + final Blockchain blockchain, + final WorldStateArchive worldStateArchive, + final int networkId, + final boolean fastSyncEnabled, + final int requestLimit, + final EthScheduler scheduler) { this.networkId = networkId; this.scheduler = scheduler; diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java index f8a4a3cc95..f2b43e43aa 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java @@ -64,27 +64,31 @@ public class SynchronizerConfiguration { private final int transactionsParallelism; private final int computationParallelism; - private SynchronizerConfiguration( - final SyncMode requestedSyncMode, - final int fastSyncPivotDistance, - final float fastSyncFullValidationRate, - final int fastSyncMinimumPeerCount, - final Duration fastSyncMaximumPeerWaitTime, - final int worldStateHashCountPerRequest, - final int worldStateRequestParallelism, - final Range blockPropagationRange, - final Optional syncMode, - final long downloaderChangeTargetThresholdByHeight, - final UInt256 downloaderChangeTargetThresholdByTd, - final int downloaderHeaderRequestSize, - final int downloaderCheckpointTimeoutsPermitted, - final int downloaderChainSegmentTimeoutsPermitted, - final int downloaderChainSegmentSize, - final long trailingPeerBlocksBehindThreshold, - final int maxTrailingPeers, - final int downloaderParallelism, - final int transactionsParallelism, - final int computationParallelism) { +// worldStateHashCountPerRequest +// worldStateRequestParallelism +// computationParallelism + + public SynchronizerConfiguration( + final SyncMode requestedSyncMode, + final int fastSyncPivotDistance, + final float fastSyncFullValidationRate, + final int fastSyncMinimumPeerCount, + final Duration fastSyncMaximumPeerWaitTime, + final int worldStateHashCountPerRequest, + final int worldStateRequestParallelism, + final Range blockPropagationRange, + final Optional syncMode, + final long downloaderChangeTargetThresholdByHeight, + final UInt256 downloaderChangeTargetThresholdByTd, + final int downloaderHeaderRequestSize, + final int downloaderCheckpointTimeoutsPermitted, + final int downloaderChainSegmentTimeoutsPermitted, + final int downloaderChainSegmentSize, + final long trailingPeerBlocksBehindThreshold, + final int maxTrailingPeers, + final int downloaderParallelism, + final int transactionsParallelism, + final int computationParallelism) { this.requestedSyncMode = requestedSyncMode; this.fastSyncPivotDistance = fastSyncPivotDistance; this.fastSyncFullValidationRate = fastSyncFullValidationRate; diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 0c411d043f..4270ac42d1 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -12,6 +12,7 @@ */ package tech.pegasys.pantheon.ethereum.eth.sync; +import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -19,13 +20,28 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static sun.security.krb5.Confounder.bytes; +import com.google.common.collect.Range; +import tech.pegasys.pantheon.config.GenesisConfigFile; import tech.pegasys.pantheon.ethereum.ProtocolContext; import tech.pegasys.pantheon.ethereum.chain.Blockchain; +import tech.pegasys.pantheon.ethereum.chain.BlockchainStorage; +import tech.pegasys.pantheon.ethereum.chain.DefaultMutableBlockchain; import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; +import tech.pegasys.pantheon.ethereum.core.Address; import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.core.BlockBody; import tech.pegasys.pantheon.ethereum.core.BlockDataGenerator; import tech.pegasys.pantheon.ethereum.core.BlockDataGenerator.BlockOptions; +import tech.pegasys.pantheon.ethereum.core.BlockHeader; +import tech.pegasys.pantheon.ethereum.core.BlockHeaderBuilder; +import tech.pegasys.pantheon.ethereum.core.Hash; +import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; +import tech.pegasys.pantheon.ethereum.core.LogsBloomFilter; +import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; +import tech.pegasys.pantheon.ethereum.core.TransactionReceipt; +import tech.pegasys.pantheon.ethereum.eth.manager.DeterministicEthScheduler; import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.manager.EthMessages; import tech.pegasys.pantheon.ethereum.eth.manager.EthPeers; @@ -40,14 +56,28 @@ import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockMessage; import tech.pegasys.pantheon.ethereum.eth.sync.state.PendingBlocks; import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; +import tech.pegasys.pantheon.ethereum.mainnet.MainnetBlockHashFunction; +import tech.pegasys.pantheon.ethereum.mainnet.MainnetProtocolSchedule; import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; +import tech.pegasys.pantheon.ethereum.mainnet.ProtocolScheduleBuilder; +import tech.pegasys.pantheon.ethereum.storage.keyvalue.KeyValueStorageWorldStateStorage; +import tech.pegasys.pantheon.ethereum.worldstate.WorldStateArchive; import tech.pegasys.pantheon.metrics.LabelledMetric; import tech.pegasys.pantheon.metrics.OperationTimer; import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; +import tech.pegasys.pantheon.services.kvstore.InMemoryKeyValueStorage; +import tech.pegasys.pantheon.util.bytes.Bytes32; +import tech.pegasys.pantheon.util.bytes.BytesValue; import tech.pegasys.pantheon.util.uint.UInt256; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import java.util.function.Supplier; import org.junit.Before; @@ -121,11 +151,11 @@ public void importsAnnouncedBlocks_aheadOfChainInOrder() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage nextAnnouncement = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockHashesMessage nextNextAnnouncement = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); @@ -156,11 +186,11 @@ public void importsAnnouncedBlocks_aheadOfChainOutOfOrder() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage nextAnnouncement = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockHashesMessage nextNextAnnouncement = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); @@ -261,14 +291,14 @@ public void importsMixedOutOfOrderMessages() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage block1Msg = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(block1.getHash(), block1.getHeader().getNumber()))); final NewBlockMessage block2Msg = NewBlockMessage.create( block2, fullBlockchain.getTotalDifficultyByHash(block2.getHash()).get()); final NewBlockHashesMessage block3Msg = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(block3.getHash(), block3.getHeader().getNumber()))); final NewBlockMessage block4Msg = NewBlockMessage.create( @@ -305,7 +335,7 @@ public void handlesDuplicateAnnouncements() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage newBlockHash = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockMessage newBlock = NewBlockMessage.create( @@ -340,7 +370,7 @@ public void handlesPendingDuplicateAnnouncements() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage newBlockHash = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockMessage newBlock = NewBlockMessage.create( @@ -372,7 +402,7 @@ public void ignoresFutureNewBlockHashAnnouncement() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage futureAnnouncement = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(futureBlock.getHash(), futureBlock.getHeader().getNumber()))); // Broadcast @@ -424,7 +454,7 @@ public void ignoresOldNewBlockHashAnnouncement() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage oldAnnouncement = NewBlockHashesMessage.create( - Collections.singletonList( + singletonList( new NewBlockHash(oldBlock.getHash(), oldBlock.getHeader().getNumber()))); // Broadcast @@ -566,4 +596,321 @@ public void shouldNotImportBlocksThatAreAlreadyBeingImported() { verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); } + + @Test + public void blockPropagationManager_functionalityAssessment() { + + BlockchainStorage kvStore00 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); + BlockHeader genesisHeader00 = + BlockHeaderBuilder.create() + .parentHash(Hash.ZERO) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(0L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody genesisBody00 = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block genesisBlock00 = new Block(genesisHeader00, genesisBody00); + DefaultMutableBlockchain blockchain00 = new DefaultMutableBlockchain(genesisBlock00, kvStore00, new NoOpMetricsSystem()); + + BlockHeader header00A = + BlockHeaderBuilder.create() + .parentHash(genesisBlock00.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(1L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body00A = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block00A = new Block(header00A, body00A); + List receipts00A = Collections.emptyList(); + blockchain00.appendBlock(block00A, receipts00A); + + BlockHeader header00B = + BlockHeaderBuilder.create() + .parentHash(block00A.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(2L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body00B = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block00B = new Block(header00B, body00B); + List receipts00B = Collections.emptyList(); + blockchain00.appendBlock(block00B, receipts00B); + + BlockHeader header00C = + BlockHeaderBuilder.create() + .parentHash(block00B.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(3L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body00C = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block00C = new Block(header00C, body00C); + List receipts00C = Collections.emptyList(); + blockchain00.appendBlock(block00C, receipts00C); + + BlockHeader header00D = + BlockHeaderBuilder.create() + .parentHash(block00C.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(4L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body00D = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block00D = new Block(header00D, body00D); + List receipts00D = Collections.emptyList(); + blockchain00.appendBlock(block00D, receipts00D); + + assertThat(blockchain00.getChainHeadBlockNumber()).isEqualTo(4L); + + /* * */ + + BlockchainStorage kvStore01 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); + BlockHeader genesisHeader01 = + BlockHeaderBuilder.create() + .parentHash(Hash.ZERO) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(0L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody genesisBody01 = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block genesisBlock01 = new Block(genesisHeader01, genesisBody01); + DefaultMutableBlockchain blockchain01 = new DefaultMutableBlockchain(genesisBlock01, kvStore01, new NoOpMetricsSystem()); + + BlockHeader header01A = + BlockHeaderBuilder.create() + .parentHash(genesisBlock01.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(1L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body01A = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block01A = new Block(header01A, body01A); + List receipts01A = Collections.emptyList(); + blockchain01.appendBlock(block01A, receipts01A); + + BlockHeader header01B = + BlockHeaderBuilder.create() + .parentHash(block01A.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(2L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body01B = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block01B = new Block(header01B, body01B); + List receipts01B = Collections.emptyList(); + blockchain01.appendBlock(block01B, receipts01B); + + BlockHeader header01C = + BlockHeaderBuilder.create() + .parentHash(block01B.getHash()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(3L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body01C = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block01C = new Block(header01C, body01C); + List receipts01C = Collections.emptyList(); + blockchain01.appendBlock(block01C, receipts01C); + + assertThat(blockchain01.getChainHeadBlockNumber()).isEqualTo(3L); + + WorldStateArchive worldStateArchive00 = new WorldStateArchive(new KeyValueStorageWorldStateStorage(new InMemoryKeyValueStorage())); + + ProtocolContext protocolContext00 = new ProtocolContext(blockchain00, worldStateArchive00, null); + + int fastSyncPivotDistance = 500; + float fastSyncFullValidationRate = .1f; + SyncMode syncMode = SyncMode.FULL; + Range blockPropagationRange = Range.closed(-10L, 30L); + long downloaderChangeTargetThresholdByHeight = 20L; + UInt256 downloaderChangeTargetThresholdByTd = UInt256.of(1_000_000_000L); + int downloaderHeaderRequestSize = 10; + int downloaderCheckpointTimeoutsPermitted = 5; + int downloaderChainSegmentTimeoutsPermitted = 5; + int downloaderChainSegmentSize = 20; + long trailingPeerBlocksBehindThreshold = 3L; + int maxTrailingPeers = Integer.MAX_VALUE; + int downloaderParallelism = 2; + int transactionsParallelism = 2; + int DEFAULT_FAST_SYNC_MINIMUM_PEERS = 5; + Duration DEFAULT_FAST_SYNC_MAXIMUM_PEER_WAIT_TIME = Duration.ofMinutes(3); + + SyncMode requestedSyncMode = SyncMode.FULL; + + SynchronizerConfiguration synchronizerConfiguration00 = new SynchronizerConfiguration( + requestedSyncMode, + fastSyncPivotDistance, + fastSyncFullValidationRate, + DEFAULT_FAST_SYNC_MINIMUM_PEERS, + DEFAULT_FAST_SYNC_MAXIMUM_PEER_WAIT_TIME, + worldStateHashCountPerRequest, + worldStateRequestParallelism, + blockPropagationRange, + syncMode, + downloaderChangeTargetThresholdByHeight, + downloaderChangeTargetThresholdByTd, + downloaderHeaderRequestSize, + downloaderCheckpointTimeoutsPermitted, + downloaderChainSegmentTimeoutsPermitted, + downloaderChainSegmentSize, + trailingPeerBlocksBehindThreshold, + maxTrailingPeers, + downloaderParallelism, + transactionsParallelism, + computationParallelism); + + int DEFAULT_CHAIN_ID00 = 1; + ProtocolSchedule protocolSchedule00 = new ProtocolScheduleBuilder<>(GenesisConfigFile.mainnet().getConfigOptions(), DEFAULT_CHAIN_ID00, Function.identity(), PrivacyParameters.noPrivacy()) + .createProtocolSchedule(); + + final int networkId00 = 1; + final EthScheduler ethScheduler00 = new DeterministicEthScheduler(DeterministicEthScheduler.TimeoutPolicy.NEVER); + + EthProtocolManager ethProtocolManager00 = new EthProtocolManager( + blockchain00, + worldStateArchive00, + networkId00, + false, + EthProtocolManager.DEFAULT_REQUEST_LIMIT, + ethScheduler00); + + SyncState syncState00 = new SyncState(blockchain00, ethProtocolManager00.ethContext().getEthPeers()); + + PendingBlocks pendingBlocks00 = new PendingBlocks(); + + LabelledMetric ethTasksTimer00 = NoOpMetricsSystem.NO_OP_LABELLED_TIMER; + + BlockPropagationManager blockPropagationManager00 = new BlockPropagationManager<>( + synchronizerConfiguration00, + protocolSchedule00, + protocolContext00, + ethProtocolManager00.ethContext(), + syncState00, + pendingBlocks00, + ethTasksTimer00); + + // ... + blockPropagationManager00.start(); + + /* * */ + + // Setup peer and messages + final RespondingEthPeer respondingEthPeer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage newBlockHashesMessage = NewBlockHashesMessage.create(singletonList(new NewBlockHash(block00D.getHash(), block00D.getHeader().getNumber()))); + final Responder responder = RespondingEthPeer.blockchainResponder(blockchain01); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, respondingEthPeer, newBlockHashesMessage); + respondingEthPeer.respondWhile(responder, respondingEthPeer::hasOutstandingRequests); + + assertThat(blockchain00.contains(block00D.getHash())).isTrue(); + } } From 93c9aa86510d23067f915c338835f025647089e8 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 7 Feb 2019 11:57:56 -0500 Subject: [PATCH 02/11] add test --- .../eth/sync/BlockPropagationManagerTest.java | 1449 +++++++---------- 1 file changed, 624 insertions(+), 825 deletions(-) diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 4270ac42d1..bf338b087a 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -86,831 +86,630 @@ public class BlockPropagationManagerTest { - private static Blockchain fullBlockchain; - - private BlockchainSetupUtil blockchainUtil; - private ProtocolSchedule protocolSchedule; - private ProtocolContext protocolContext; - private MutableBlockchain blockchain; - private EthProtocolManager ethProtocolManager; - private BlockPropagationManager blockPropagationManager; - private SynchronizerConfiguration syncConfig; - private final PendingBlocks pendingBlocks = new PendingBlocks(); - private SyncState syncState; - private final LabelledMetric ethTasksTimer = - NoOpMetricsSystem.NO_OP_LABELLED_TIMER; - - @BeforeClass - public static void setupSuite() { - fullBlockchain = BlockchainSetupUtil.forTesting().importAllBlocks(); - } - - @Before - public void setup() { - blockchainUtil = BlockchainSetupUtil.forTesting(); - blockchain = spy(blockchainUtil.getBlockchain()); - protocolSchedule = blockchainUtil.getProtocolSchedule(); - final ProtocolContext tempProtocolContext = blockchainUtil.getProtocolContext(); - protocolContext = - new ProtocolContext<>( - blockchain, - tempProtocolContext.getWorldStateArchive(), - tempProtocolContext.getConsensusState()); - ethProtocolManager = - EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); - syncConfig = - SynchronizerConfiguration.builder() - .blockPropagationRange(-3, 5) - .build() - .validated(blockchain); - syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); - blockPropagationManager = - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethProtocolManager.ethContext(), - syncState, - pendingBlocks, - ethTasksTimer); - } - - @Test - public void importsAnnouncedBlocks_aheadOfChainInOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage nextAnnouncement = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockHashesMessage nextNextAnnouncement = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast second message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsAnnouncedBlocks_aheadOfChainOutOfOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage nextAnnouncement = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockHashesMessage nextNextAnnouncement = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast second message first - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsAnnouncedNewBlocks_aheadOfChainInOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage nextAnnouncement = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - final NewBlockMessage nextNextAnnouncement = - NewBlockMessage.create( - nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast second message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsAnnouncedNewBlocks_aheadOfChainOutOfOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage nextAnnouncement = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - final NewBlockMessage nextNextAnnouncement = - NewBlockMessage.create( - nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast second message first - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsMixedOutOfOrderMessages() { - blockchainUtil.importFirstBlocks(2); - final Block block1 = blockchainUtil.getBlock(2); - final Block block2 = blockchainUtil.getBlock(3); - final Block block3 = blockchainUtil.getBlock(4); - final Block block4 = blockchainUtil.getBlock(5); - - // Sanity check - assertThat(blockchain.contains(block1.getHash())).isFalse(); - assertThat(blockchain.contains(block2.getHash())).isFalse(); - assertThat(blockchain.contains(block3.getHash())).isFalse(); - assertThat(blockchain.contains(block4.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage block1Msg = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(block1.getHash(), block1.getHeader().getNumber()))); - final NewBlockMessage block2Msg = - NewBlockMessage.create( - block2, fullBlockchain.getTotalDifficultyByHash(block2.getHash()).get()); - final NewBlockHashesMessage block3Msg = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(block3.getHash(), block3.getHeader().getNumber()))); - final NewBlockMessage block4Msg = - NewBlockMessage.create( - block4, fullBlockchain.getTotalDifficultyByHash(block4.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast older blocks - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block3Msg); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block4Msg); - peer.respondWhile(responder, peer::hasOutstandingRequests); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block2Msg); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast first block - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block1Msg); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(block1.getHash())).isTrue(); - assertThat(blockchain.contains(block2.getHash())).isTrue(); - assertThat(blockchain.contains(block3.getHash())).isTrue(); - assertThat(blockchain.contains(block4.getHash())).isTrue(); - } - - @Test - public void handlesDuplicateAnnouncements() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage newBlockHash = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockMessage newBlock = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast duplicate - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast duplicate - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - verify(blockchain, times(1)).appendBlock(any(), any()); - } - - @Test - public void handlesPendingDuplicateAnnouncements() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage newBlockHash = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockMessage newBlock = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - - // Broadcast messages - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - // Respond - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - verify(blockchain, times(1)).appendBlock(any(), any()); - } - - @Test - public void ignoresFutureNewBlockHashAnnouncement() { - blockchainUtil.importFirstBlocks(2); - final Block futureBlock = blockchainUtil.getBlock(11); - - // Sanity check - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage futureAnnouncement = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(futureBlock.getHash(), futureBlock.getHeader().getNumber()))); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - } - - @Test - public void ignoresFutureNewBlockAnnouncement() { - blockchainUtil.importFirstBlocks(2); - final Block futureBlock = blockchainUtil.getBlock(11); - - // Sanity check - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage futureAnnouncement = - NewBlockMessage.create( - futureBlock, fullBlockchain.getTotalDifficultyByHash(futureBlock.getHash()).get()); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - } - - @Test - public void ignoresOldNewBlockHashAnnouncement() { - final BlockDataGenerator gen = new BlockDataGenerator(); - blockchainUtil.importFirstBlocks(10); - final Block blockOne = blockchainUtil.getBlock(1); - final Block oldBlock = gen.nextBlock(blockOne); - - // Sanity check - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - - final BlockPropagationManager propManager = spy(blockPropagationManager); - propManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage oldAnnouncement = - NewBlockHashesMessage.create( - singletonList( - new NewBlockHash(oldBlock.getHash(), oldBlock.getHeader().getNumber()))); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - verify(propManager, times(0)).importOrSavePendingBlock(any()); - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - } - - @Test - public void ignoresOldNewBlockAnnouncement() { - final BlockDataGenerator gen = new BlockDataGenerator(); - blockchainUtil.importFirstBlocks(10); - final Block blockOne = blockchainUtil.getBlock(1); - final Block oldBlock = gen.nextBlock(blockOne); - - // Sanity check - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - - final BlockPropagationManager propManager = spy(blockPropagationManager); - propManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage oldAnnouncement = NewBlockMessage.create(oldBlock, UInt256.ZERO); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - verify(propManager, times(0)).importOrSavePendingBlock(any()); - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - } - - @Test - public void purgesOldBlocks() { - final int oldBlocksToImport = 3; - syncConfig = - SynchronizerConfiguration.builder() - .blockPropagationRange(-oldBlocksToImport, 5) - .build() - .validated(blockchain); - final BlockPropagationManager blockPropagationManager = - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethProtocolManager.ethContext(), - syncState, - pendingBlocks, - ethTasksTimer); - - final BlockDataGenerator gen = new BlockDataGenerator(); - // Import some blocks - blockchainUtil.importFirstBlocks(5); - // Set up test block next to head, that should eventually be purged - final Block blockToPurge = - gen.block(BlockOptions.create().setBlockNumber(blockchain.getChainHeadBlockNumber())); - - blockPropagationManager.start(); - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage blockAnnouncementMsg = NewBlockMessage.create(blockToPurge, UInt256.ZERO); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, blockAnnouncementMsg); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - // Check that we pushed our block into the pending collection - assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); - assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); - - // Import blocks until we bury the target block far enough to be cleaned up - for (int i = 0; i < oldBlocksToImport; i++) { - blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); - - assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); - assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); + private static Blockchain fullBlockchain; + + private BlockchainSetupUtil blockchainUtil; + private ProtocolSchedule protocolSchedule; + private ProtocolContext protocolContext; + private MutableBlockchain blockchain; + private EthProtocolManager ethProtocolManager; + private BlockPropagationManager blockPropagationManager; + private SynchronizerConfiguration syncConfig; + private final PendingBlocks pendingBlocks = new PendingBlocks(); + private SyncState syncState; + private final LabelledMetric ethTasksTimer = + NoOpMetricsSystem.NO_OP_LABELLED_TIMER; + + @BeforeClass + public static void setupSuite() { + fullBlockchain = BlockchainSetupUtil.forTesting().importAllBlocks(); } - // Import again to trigger cleanup - blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); - assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); - assertThat(pendingBlocks.contains(blockToPurge.getHash())).isFalse(); - } - - @Test - public void updatesChainHeadWhenNewBlockMessageReceived() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final UInt256 totalDifficulty = - fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); - final NewBlockMessage nextAnnouncement = NewBlockMessage.create(nextBlock, totalDifficulty); - - // Broadcast message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(peer.getEthPeer().chainState().getBestBlock().getHash()) - .isEqualTo(nextBlock.getHash()); - assertThat(peer.getEthPeer().chainState().getEstimatedHeight()) - .isEqualTo(nextBlock.getHeader().getNumber()); - assertThat(peer.getEthPeer().chainState().getBestBlock().getTotalDifficulty()) - .isEqualTo(totalDifficulty); - } - - @SuppressWarnings("unchecked") - @Test - public void shouldNotImportBlocksThatAreAlreadyBeingImported() { - final EthScheduler ethScheduler = mock(EthScheduler.class); - when(ethScheduler.scheduleSyncWorkerTask(any(Supplier.class))) - .thenReturn(new CompletableFuture<>()); - final EthContext ethContext = - new EthContext("eth", new EthPeers("eth"), new EthMessages(), ethScheduler); - final BlockPropagationManager blockPropagationManager = - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethContext, - syncState, - pendingBlocks, - ethTasksTimer); - - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - blockPropagationManager.importOrSavePendingBlock(nextBlock); - blockPropagationManager.importOrSavePendingBlock(nextBlock); - - verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); - } - - @Test - public void blockPropagationManager_functionalityAssessment() { - - BlockchainStorage kvStore00 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); - BlockHeader genesisHeader00 = - BlockHeaderBuilder.create() - .parentHash(Hash.ZERO) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(0L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody genesisBody00 = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block genesisBlock00 = new Block(genesisHeader00, genesisBody00); - DefaultMutableBlockchain blockchain00 = new DefaultMutableBlockchain(genesisBlock00, kvStore00, new NoOpMetricsSystem()); - - BlockHeader header00A = - BlockHeaderBuilder.create() - .parentHash(genesisBlock00.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(1L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body00A = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block00A = new Block(header00A, body00A); - List receipts00A = Collections.emptyList(); - blockchain00.appendBlock(block00A, receipts00A); - - BlockHeader header00B = - BlockHeaderBuilder.create() - .parentHash(block00A.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(2L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body00B = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block00B = new Block(header00B, body00B); - List receipts00B = Collections.emptyList(); - blockchain00.appendBlock(block00B, receipts00B); - - BlockHeader header00C = - BlockHeaderBuilder.create() - .parentHash(block00B.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(3L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body00C = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block00C = new Block(header00C, body00C); - List receipts00C = Collections.emptyList(); - blockchain00.appendBlock(block00C, receipts00C); - - BlockHeader header00D = - BlockHeaderBuilder.create() - .parentHash(block00C.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(4L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body00D = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block00D = new Block(header00D, body00D); - List receipts00D = Collections.emptyList(); - blockchain00.appendBlock(block00D, receipts00D); - - assertThat(blockchain00.getChainHeadBlockNumber()).isEqualTo(4L); - - /* * */ - - BlockchainStorage kvStore01 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); - BlockHeader genesisHeader01 = - BlockHeaderBuilder.create() - .parentHash(Hash.ZERO) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(0L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody genesisBody01 = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block genesisBlock01 = new Block(genesisHeader01, genesisBody01); - DefaultMutableBlockchain blockchain01 = new DefaultMutableBlockchain(genesisBlock01, kvStore01, new NoOpMetricsSystem()); - - BlockHeader header01A = - BlockHeaderBuilder.create() - .parentHash(genesisBlock01.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(1L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body01A = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block01A = new Block(header01A, body01A); - List receipts01A = Collections.emptyList(); - blockchain01.appendBlock(block01A, receipts01A); - - BlockHeader header01B = - BlockHeaderBuilder.create() - .parentHash(block01A.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(2L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body01B = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block01B = new Block(header01B, body01B); - List receipts01B = Collections.emptyList(); - blockchain01.appendBlock(block01B, receipts01B); - - BlockHeader header01C = - BlockHeaderBuilder.create() - .parentHash(block01B.getHash()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(3L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body01C = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block01C = new Block(header01C, body01C); - List receipts01C = Collections.emptyList(); - blockchain01.appendBlock(block01C, receipts01C); - - assertThat(blockchain01.getChainHeadBlockNumber()).isEqualTo(3L); - - WorldStateArchive worldStateArchive00 = new WorldStateArchive(new KeyValueStorageWorldStateStorage(new InMemoryKeyValueStorage())); - - ProtocolContext protocolContext00 = new ProtocolContext(blockchain00, worldStateArchive00, null); - - int fastSyncPivotDistance = 500; - float fastSyncFullValidationRate = .1f; - SyncMode syncMode = SyncMode.FULL; - Range blockPropagationRange = Range.closed(-10L, 30L); - long downloaderChangeTargetThresholdByHeight = 20L; - UInt256 downloaderChangeTargetThresholdByTd = UInt256.of(1_000_000_000L); - int downloaderHeaderRequestSize = 10; - int downloaderCheckpointTimeoutsPermitted = 5; - int downloaderChainSegmentTimeoutsPermitted = 5; - int downloaderChainSegmentSize = 20; - long trailingPeerBlocksBehindThreshold = 3L; - int maxTrailingPeers = Integer.MAX_VALUE; - int downloaderParallelism = 2; - int transactionsParallelism = 2; - int DEFAULT_FAST_SYNC_MINIMUM_PEERS = 5; - Duration DEFAULT_FAST_SYNC_MAXIMUM_PEER_WAIT_TIME = Duration.ofMinutes(3); - - SyncMode requestedSyncMode = SyncMode.FULL; - - SynchronizerConfiguration synchronizerConfiguration00 = new SynchronizerConfiguration( - requestedSyncMode, - fastSyncPivotDistance, - fastSyncFullValidationRate, - DEFAULT_FAST_SYNC_MINIMUM_PEERS, - DEFAULT_FAST_SYNC_MAXIMUM_PEER_WAIT_TIME, - worldStateHashCountPerRequest, - worldStateRequestParallelism, - blockPropagationRange, - syncMode, - downloaderChangeTargetThresholdByHeight, - downloaderChangeTargetThresholdByTd, - downloaderHeaderRequestSize, - downloaderCheckpointTimeoutsPermitted, - downloaderChainSegmentTimeoutsPermitted, - downloaderChainSegmentSize, - trailingPeerBlocksBehindThreshold, - maxTrailingPeers, - downloaderParallelism, - transactionsParallelism, - computationParallelism); - - int DEFAULT_CHAIN_ID00 = 1; - ProtocolSchedule protocolSchedule00 = new ProtocolScheduleBuilder<>(GenesisConfigFile.mainnet().getConfigOptions(), DEFAULT_CHAIN_ID00, Function.identity(), PrivacyParameters.noPrivacy()) - .createProtocolSchedule(); - - final int networkId00 = 1; - final EthScheduler ethScheduler00 = new DeterministicEthScheduler(DeterministicEthScheduler.TimeoutPolicy.NEVER); - - EthProtocolManager ethProtocolManager00 = new EthProtocolManager( - blockchain00, - worldStateArchive00, - networkId00, - false, - EthProtocolManager.DEFAULT_REQUEST_LIMIT, - ethScheduler00); - - SyncState syncState00 = new SyncState(blockchain00, ethProtocolManager00.ethContext().getEthPeers()); - - PendingBlocks pendingBlocks00 = new PendingBlocks(); - - LabelledMetric ethTasksTimer00 = NoOpMetricsSystem.NO_OP_LABELLED_TIMER; - - BlockPropagationManager blockPropagationManager00 = new BlockPropagationManager<>( - synchronizerConfiguration00, - protocolSchedule00, - protocolContext00, - ethProtocolManager00.ethContext(), - syncState00, - pendingBlocks00, - ethTasksTimer00); - - // ... - blockPropagationManager00.start(); - - /* * */ - - // Setup peer and messages - final RespondingEthPeer respondingEthPeer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage newBlockHashesMessage = NewBlockHashesMessage.create(singletonList(new NewBlockHash(block00D.getHash(), block00D.getHeader().getNumber()))); - final Responder responder = RespondingEthPeer.blockchainResponder(blockchain01); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, respondingEthPeer, newBlockHashesMessage); - respondingEthPeer.respondWhile(responder, respondingEthPeer::hasOutstandingRequests); - - assertThat(blockchain00.contains(block00D.getHash())).isTrue(); - } + @Before + public void setup() { + blockchainUtil = BlockchainSetupUtil.forTesting(); + blockchain = spy(blockchainUtil.getBlockchain()); + protocolSchedule = blockchainUtil.getProtocolSchedule(); + final ProtocolContext tempProtocolContext = blockchainUtil.getProtocolContext(); + protocolContext = + new ProtocolContext<>( + blockchain, + tempProtocolContext.getWorldStateArchive(), + tempProtocolContext.getConsensusState()); + ethProtocolManager = + EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); + syncConfig = + SynchronizerConfiguration.builder() + .blockPropagationRange(-3, 5) + .build() + .validated(blockchain); + syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); + blockPropagationManager = + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethProtocolManager.ethContext(), + syncState, + pendingBlocks, + ethTasksTimer); + } + + @Test + public void importsAnnouncedBlocks_aheadOfChainInOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage nextAnnouncement = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockHashesMessage nextNextAnnouncement = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast second message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsAnnouncedBlocks_aheadOfChainOutOfOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage nextAnnouncement = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockHashesMessage nextNextAnnouncement = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast second message first + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsAnnouncedNewBlocks_aheadOfChainInOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage nextAnnouncement = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + final NewBlockMessage nextNextAnnouncement = + NewBlockMessage.create( + nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast second message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsAnnouncedNewBlocks_aheadOfChainOutOfOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage nextAnnouncement = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + final NewBlockMessage nextNextAnnouncement = + NewBlockMessage.create( + nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast second message first + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsMixedOutOfOrderMessages() { + blockchainUtil.importFirstBlocks(2); + final Block block1 = blockchainUtil.getBlock(2); + final Block block2 = blockchainUtil.getBlock(3); + final Block block3 = blockchainUtil.getBlock(4); + final Block block4 = blockchainUtil.getBlock(5); + + // Sanity check + assertThat(blockchain.contains(block1.getHash())).isFalse(); + assertThat(blockchain.contains(block2.getHash())).isFalse(); + assertThat(blockchain.contains(block3.getHash())).isFalse(); + assertThat(blockchain.contains(block4.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage block1Msg = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(block1.getHash(), block1.getHeader().getNumber()))); + final NewBlockMessage block2Msg = + NewBlockMessage.create( + block2, fullBlockchain.getTotalDifficultyByHash(block2.getHash()).get()); + final NewBlockHashesMessage block3Msg = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(block3.getHash(), block3.getHeader().getNumber()))); + final NewBlockMessage block4Msg = + NewBlockMessage.create( + block4, fullBlockchain.getTotalDifficultyByHash(block4.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast older blocks + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block3Msg); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block4Msg); + peer.respondWhile(responder, peer::hasOutstandingRequests); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block2Msg); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast first block + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block1Msg); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(block1.getHash())).isTrue(); + assertThat(blockchain.contains(block2.getHash())).isTrue(); + assertThat(blockchain.contains(block3.getHash())).isTrue(); + assertThat(blockchain.contains(block4.getHash())).isTrue(); + } + + @Test + public void handlesDuplicateAnnouncements() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage newBlockHash = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockMessage newBlock = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast duplicate + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast duplicate + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + verify(blockchain, times(1)).appendBlock(any(), any()); + } + + @Test + public void handlesPendingDuplicateAnnouncements() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage newBlockHash = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockMessage newBlock = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + + // Broadcast messages + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + // Respond + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + verify(blockchain, times(1)).appendBlock(any(), any()); + } + + @Test + public void ignoresFutureNewBlockHashAnnouncement() { + blockchainUtil.importFirstBlocks(2); + final Block futureBlock = blockchainUtil.getBlock(11); + + // Sanity check + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage futureAnnouncement = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(futureBlock.getHash(), futureBlock.getHeader().getNumber()))); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + } + + @Test + public void ignoresFutureNewBlockAnnouncement() { + blockchainUtil.importFirstBlocks(2); + final Block futureBlock = blockchainUtil.getBlock(11); + + // Sanity check + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage futureAnnouncement = + NewBlockMessage.create( + futureBlock, fullBlockchain.getTotalDifficultyByHash(futureBlock.getHash()).get()); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + } + + @Test + public void ignoresOldNewBlockHashAnnouncement() { + final BlockDataGenerator gen = new BlockDataGenerator(); + blockchainUtil.importFirstBlocks(10); + final Block blockOne = blockchainUtil.getBlock(1); + final Block oldBlock = gen.nextBlock(blockOne); + + // Sanity check + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + + final BlockPropagationManager propManager = spy(blockPropagationManager); + propManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage oldAnnouncement = + NewBlockHashesMessage.create( + singletonList( + new NewBlockHash(oldBlock.getHash(), oldBlock.getHeader().getNumber()))); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + verify(propManager, times(0)).importOrSavePendingBlock(any()); + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + } + + @Test + public void ignoresOldNewBlockAnnouncement() { + final BlockDataGenerator gen = new BlockDataGenerator(); + blockchainUtil.importFirstBlocks(10); + final Block blockOne = blockchainUtil.getBlock(1); + final Block oldBlock = gen.nextBlock(blockOne); + + // Sanity check + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + + final BlockPropagationManager propManager = spy(blockPropagationManager); + propManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage oldAnnouncement = NewBlockMessage.create(oldBlock, UInt256.ZERO); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + verify(propManager, times(0)).importOrSavePendingBlock(any()); + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + } + + @Test + public void purgesOldBlocks() { + final int oldBlocksToImport = 3; + syncConfig = + SynchronizerConfiguration.builder() + .blockPropagationRange(-oldBlocksToImport, 5) + .build() + .validated(blockchain); + final BlockPropagationManager blockPropagationManager = + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethProtocolManager.ethContext(), + syncState, + pendingBlocks, + ethTasksTimer); + + final BlockDataGenerator gen = new BlockDataGenerator(); + // Import some blocks + blockchainUtil.importFirstBlocks(5); + // Set up test block next to head, that should eventually be purged + final Block blockToPurge = + gen.block(BlockOptions.create().setBlockNumber(blockchain.getChainHeadBlockNumber())); + + blockPropagationManager.start(); + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage blockAnnouncementMsg = NewBlockMessage.create(blockToPurge, UInt256.ZERO); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, blockAnnouncementMsg); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + // Check that we pushed our block into the pending collection + assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); + assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); + + // Import blocks until we bury the target block far enough to be cleaned up + for (int i = 0; i < oldBlocksToImport; i++) { + blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); + + assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); + assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); + } + + // Import again to trigger cleanup + blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); + assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); + assertThat(pendingBlocks.contains(blockToPurge.getHash())).isFalse(); + } + + @Test + public void updatesChainHeadWhenNewBlockMessageReceived() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final UInt256 totalDifficulty = + fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); + final NewBlockMessage nextAnnouncement = NewBlockMessage.create(nextBlock, totalDifficulty); + + // Broadcast message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(peer.getEthPeer().chainState().getBestBlock().getHash()) + .isEqualTo(nextBlock.getHash()); + assertThat(peer.getEthPeer().chainState().getEstimatedHeight()) + .isEqualTo(nextBlock.getHeader().getNumber()); + assertThat(peer.getEthPeer().chainState().getBestBlock().getTotalDifficulty()) + .isEqualTo(totalDifficulty); + } + + @SuppressWarnings("unchecked") + @Test + public void shouldNotImportBlocksThatAreAlreadyBeingImported() { + final EthScheduler ethScheduler = mock(EthScheduler.class); + when(ethScheduler.scheduleSyncWorkerTask(any(Supplier.class))) + .thenReturn(new CompletableFuture<>()); + final EthContext ethContext = + new EthContext("eth", new EthPeers("eth"), new EthMessages(), ethScheduler); + final BlockPropagationManager blockPropagationManager = + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethContext, + syncState, + pendingBlocks, + ethTasksTimer); + + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + blockPropagationManager.importOrSavePendingBlock(nextBlock); + blockPropagationManager.importOrSavePendingBlock(nextBlock); + + verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); + } + + private MutableBlockchain generateBlockchain(BlockchainStorage blockchainStorage, Block genesisBlock, int head) { + DefaultMutableBlockchain blockchain = new DefaultMutableBlockchain(genesisBlock, blockchainStorage, new NoOpMetricsSystem()); + for (int i = 1; i <= head; i++) { + BlockHeader header = + BlockHeaderBuilder.create() + .parentHash(blockchain.getBlockHashByNumber(Long.valueOf(i - 1)).get()) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(Long.valueOf(i)) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody body = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block block = new Block(header, body); + List receipts = Collections.emptyList(); + blockchain.appendBlock(block, receipts); + } + return blockchain; + } + + @Test + public void blockPropagationManager_functionalityAssessment() { + BlockHeader genesisHeader = + BlockHeaderBuilder.create() + .parentHash(Hash.ZERO) + .ommersHash(Hash.ZERO) + .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) + .stateRoot(Hash.ZERO) + .transactionsRoot(Hash.ZERO) + .receiptsRoot(Hash.ZERO) + .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) + .difficulty(UInt256.ZERO) + .number(0L) + .gasLimit(1L) + .gasUsed(1L) + .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) + .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) + .mixHash(Hash.ZERO) + .nonce(0L) + .blockHashFunction(MainnetBlockHashFunction::createHash) + .buildBlockHeader(); + BlockBody genesisBody = new BlockBody(Collections.emptyList(), Collections.emptyList()); + Block genesisBlock = new Block(genesisHeader, genesisBody); + + BlockchainStorage blockchainStorage00 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); + MutableBlockchain blockchain00 = generateBlockchain(blockchainStorage00, genesisBlock, 4); + assertThat(blockchain00.getChainHeadBlockNumber()).isEqualTo(4L); + + BlockchainStorage blockchainStorage01 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); + MutableBlockchain blockchain01 = generateBlockchain(blockchainStorage01, genesisBlock, 3); + assertThat(blockchain01.getChainHeadBlockNumber()).isEqualTo(3L); + + SynchronizerConfiguration synchronizerConfiguration = new SynchronizerConfiguration( + SyncMode.FULL, + 500, + .1f, + 5, + Duration.ofMinutes(3), + 0, + 0, + Range.closed(-10L, 30L), + Optional.of(SyncMode.FULL), + 20L, + UInt256.of(1_000_000_000L), + 10, + 5, + 5, + 20, + 3L, + Integer.MAX_VALUE, + 2, + 2, + 0); + + WorldStateArchive worldStateArchive = new WorldStateArchive(new KeyValueStorageWorldStateStorage(new InMemoryKeyValueStorage())); + + EthProtocolManager ethProtocolManager00 = new EthProtocolManager( + blockchain00, + worldStateArchive, + 1, + false, + EthProtocolManager.DEFAULT_REQUEST_LIMIT, + new DeterministicEthScheduler(DeterministicEthScheduler.TimeoutPolicy.NEVER)); + + BlockPropagationManager blockPropagationManager00 = new BlockPropagationManager<>( + synchronizerConfiguration, + new ProtocolScheduleBuilder<>(GenesisConfigFile.mainnet().getConfigOptions(), 1, Function.identity(), PrivacyParameters.noPrivacy()).createProtocolSchedule(), + new ProtocolContext(blockchain00, worldStateArchive, null), + ethProtocolManager00.ethContext(), + new SyncState(blockchain00, ethProtocolManager00.ethContext().getEthPeers()), + new PendingBlocks(), + NoOpMetricsSystem.NO_OP_LABELLED_TIMER); + + blockPropagationManager00.start(); + + // Setup peer and messages + final RespondingEthPeer respondingEthPeer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage newBlockHashesMessage = NewBlockHashesMessage.create(singletonList(new NewBlockHash(blockchain00.getBlockHashByNumber(4L).get(), 4L))); + final Responder responder = RespondingEthPeer.blockchainResponder(blockchain01); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, respondingEthPeer, newBlockHashesMessage); + respondingEthPeer.respondWhile(responder, respondingEthPeer::hasOutstandingRequests); + assertThat(blockchain00.contains(blockchain00.getBlockHashByNumber(4L).get())).isTrue(); + } } From 610834cbb90a2d68357904c28dd18308895aacd0 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 7 Feb 2019 11:59:26 -0500 Subject: [PATCH 03/11] clean up --- .../pantheon/ethereum/eth/manager/EthProtocolManager.java | 2 +- .../pantheon/ethereum/eth/sync/SynchronizerConfiguration.java | 4 ---- .../ethereum/eth/sync/BlockPropagationManagerTest.java | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java index 06263566e5..4cc7583e40 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java @@ -44,7 +44,7 @@ import org.apache.logging.log4j.Logger; public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { - public static final int DEFAULT_REQUEST_LIMIT = 200; + static final int DEFAULT_REQUEST_LIMIT = 200; private static final Logger LOG = LogManager.getLogger(); private static final List FAST_SYNC_CAPS = Collections.singletonList(EthProtocol.ETH63); diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java index f2b43e43aa..08ef75ab6b 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java @@ -64,10 +64,6 @@ public class SynchronizerConfiguration { private final int transactionsParallelism; private final int computationParallelism; -// worldStateHashCountPerRequest -// worldStateRequestParallelism -// computationParallelism - public SynchronizerConfiguration( final SyncMode requestedSyncMode, final int fastSyncPivotDistance, diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index bf338b087a..13641d2eb7 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -688,7 +688,7 @@ public void blockPropagationManager_functionalityAssessment() { worldStateArchive, 1, false, - EthProtocolManager.DEFAULT_REQUEST_LIMIT, + 200, new DeterministicEthScheduler(DeterministicEthScheduler.TimeoutPolicy.NEVER)); BlockPropagationManager blockPropagationManager00 = new BlockPropagationManager<>( From d4468a52eed67acbd5742e705e32ce0a48a60e52 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 7 Feb 2019 12:44:55 -0500 Subject: [PATCH 04/11] scaffolding --- .../pantheon/ethereum/eth/manager/EthPeer.java | 8 ++++++++ .../eth/sync/BlockPropagationManager.java | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index a21306e4e6..2e959bd313 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -14,6 +14,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.Hash; import tech.pegasys.pantheon.ethereum.eth.manager.ChainState.EstimatedHeightListener; import tech.pegasys.pantheon.ethereum.eth.manager.RequestManager.ResponseStream; @@ -130,6 +131,13 @@ public ResponseStream send(final MessageData messageData) throws PeerNotConnecte } } + // Sends block to a peer... + public void propagateBlock(Block block, UInt256 totalDifficulty) { + + // New Block Message... + connection. + } + public ResponseStream getHeadersByHash( final Hash hash, final int maxHeaders, final int skip, final boolean reverse) throws PeerNotConnected { diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index 2157502cff..1556e26914 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -22,6 +22,7 @@ import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.manager.EthMessage; import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; +import tech.pegasys.pantheon.ethereum.eth.manager.EthPeers; import tech.pegasys.pantheon.ethereum.eth.messages.EthPV62; import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockHashesMessage; import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockHashesMessage.NewBlockHash; @@ -144,6 +145,21 @@ private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchai } } + // Predicated on the type of information requested, announce a block's availability + // or propagate it to a subset of peers: "Announced block" vs. "Propagated block". + private void broadcastBlock(Block block) { + + // Determine total difficulty of block... + + // If propagation is requested, send to a subset of peers + List availablePeers = ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); + + // Send the block... + for(EthPeer ethPeer : availablePeers) { + ethPeer.send(); + } + } + private void handleNewBlockFromNetwork(final EthMessage message) { final Blockchain blockchain = protocolContext.getBlockchain(); final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); From 96a21262e15d0f4438a643a85bed881ec99c3118 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Tue, 12 Feb 2019 14:02:38 -0500 Subject: [PATCH 05/11] update --- .../ethereum/eth/manager/EthPeer.java | 17 +- .../eth/sync/BlockPropagationManager.java | 469 +++++++++--------- .../eth/sync/BlockPropagationManagerTest.java | 171 ++----- 3 files changed, 273 insertions(+), 384 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index 2e959bd313..52c5d0f342 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -24,9 +24,11 @@ import tech.pegasys.pantheon.ethereum.eth.messages.GetBlockHeadersMessage; import tech.pegasys.pantheon.ethereum.eth.messages.GetNodeDataMessage; import tech.pegasys.pantheon.ethereum.eth.messages.GetReceiptsMessage; +import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockMessage; import tech.pegasys.pantheon.ethereum.p2p.api.MessageData; import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection; import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection.PeerNotConnected; +import tech.pegasys.pantheon.ethereum.p2p.wire.Capability; import tech.pegasys.pantheon.ethereum.p2p.wire.messages.DisconnectMessage.DisconnectReason; import tech.pegasys.pantheon.util.Subscribers; import tech.pegasys.pantheon.util.bytes.BytesValue; @@ -131,11 +133,16 @@ public ResponseStream send(final MessageData messageData) throws PeerNotConnecte } } - // Sends block to a peer... - public void propagateBlock(Block block, UInt256 totalDifficulty) { - - // New Block Message... - connection. + /** + * Sends block to a peer... + */ + public void propagateBlock(final Block block, final UInt256 totalDifficulty) { + final NewBlockMessage newBlockMessage = NewBlockMessage.create(block, totalDifficulty); + final Capability capability = Capability.create("eth", 63); + try { + connection.send(capability, newBlockMessage); + } catch (Exception ignored) { + } } public ResponseStream getHeadersByHash( diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index 1556e26914..cdbf1e70a6 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -56,261 +56,258 @@ import org.apache.logging.log4j.Logger; public class BlockPropagationManager { - private static final Logger LOG = LogManager.getLogger(); - - private final SynchronizerConfiguration config; - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final SyncState syncState; - private final LabelledMetric ethTasksTimer; - - private final AtomicBoolean started = new AtomicBoolean(false); - - private final Set requestedBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final Set importingBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final PendingBlocks pendingBlocks; - - BlockPropagationManager( - final SynchronizerConfiguration config, - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final SyncState syncState, - final PendingBlocks pendingBlocks, - final LabelledMetric ethTasksTimer) { - this.config = config; - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.ethTasksTimer = ethTasksTimer; - - this.syncState = syncState; - this.pendingBlocks = pendingBlocks; - } - - public void start() { - if (started.compareAndSet(false, true)) { - setupListeners(); - } else { - throw new IllegalStateException( - "Attempt to start an already started " + this.getClass().getSimpleName() + "."); - } - } - - private void setupListeners() { - protocolContext.getBlockchain().observeBlockAdded(this::onBlockAdded); - ethContext.getEthMessages().subscribe(EthPV62.NEW_BLOCK, this::handleNewBlockFromNetwork); - ethContext - .getEthMessages() - .subscribe(EthPV62.NEW_BLOCK_HASHES, this::handleNewBlockHashesFromNetwork); - } - - private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchain blockchain) { - // Check to see if any of our pending blocks are now ready for import - final Block newBlock = blockAddedEvent.getBlock(); - - final List readyForImport; - synchronized (pendingBlocks) { - // Remove block from pendingBlocks list - pendingBlocks.deregisterPendingBlock(newBlock); - - // Import any pending blocks that are children of the newly added block - readyForImport = pendingBlocks.childrenOf(newBlock.getHash()); + private static final Logger LOG = LogManager.getLogger(); + + private final SynchronizerConfiguration config; + private final ProtocolSchedule protocolSchedule; + private final ProtocolContext protocolContext; + private final EthContext ethContext; + private final SyncState syncState; + private final LabelledMetric ethTasksTimer; + + private final AtomicBoolean started = new AtomicBoolean(false); + + private final Set requestedBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set importingBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final PendingBlocks pendingBlocks; + + BlockPropagationManager( + final SynchronizerConfiguration config, + final ProtocolSchedule protocolSchedule, + final ProtocolContext protocolContext, + final EthContext ethContext, + final SyncState syncState, + final PendingBlocks pendingBlocks, + final LabelledMetric ethTasksTimer) { + this.config = config; + this.protocolSchedule = protocolSchedule; + this.protocolContext = protocolContext; + this.ethContext = ethContext; + this.ethTasksTimer = ethTasksTimer; + + this.syncState = syncState; + this.pendingBlocks = pendingBlocks; } - if (!readyForImport.isEmpty()) { - final Supplier>> importBlocksTask = - PersistBlockTask.forUnorderedBlocks( - protocolSchedule, - protocolContext, - readyForImport, - HeaderValidationMode.FULL, - ethTasksTimer); - ethContext - .getScheduler() - .scheduleSyncWorkerTask(importBlocksTask) - .whenComplete( - (r, t) -> { - if (r != null) { - LOG.info("Imported {} pending blocks", r.size()); - } - }); + public void start() { + if (started.compareAndSet(false, true)) { + setupListeners(); + } else { + throw new IllegalStateException( + "Attempt to start an already started " + this.getClass().getSimpleName() + "."); + } } - if (blockAddedEvent.getEventType().equals(EventType.HEAD_ADVANCED)) { - final long head = blockchain.getChainHeadBlockNumber(); - final long cutoff = head + config.blockPropagationRange().lowerEndpoint(); - pendingBlocks.purgeBlocksOlderThan(cutoff); + private void setupListeners() { + protocolContext.getBlockchain().observeBlockAdded(this::onBlockAdded); + ethContext.getEthMessages().subscribe(EthPV62.NEW_BLOCK, this::handleNewBlockFromNetwork); + ethContext + .getEthMessages() + .subscribe(EthPV62.NEW_BLOCK_HASHES, this::handleNewBlockHashesFromNetwork); } - } - - // Predicated on the type of information requested, announce a block's availability - // or propagate it to a subset of peers: "Announced block" vs. "Propagated block". - private void broadcastBlock(Block block) { - // Determine total difficulty of block... + private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchain blockchain) { + // Check to see if any of our pending blocks are now ready for import + final Block newBlock = blockAddedEvent.getBlock(); - // If propagation is requested, send to a subset of peers - List availablePeers = ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); + final List readyForImport; + synchronized (pendingBlocks) { + // Remove block from pendingBlocks list + pendingBlocks.deregisterPendingBlock(newBlock); - // Send the block... - for(EthPeer ethPeer : availablePeers) { - ethPeer.send(); - } - } - - private void handleNewBlockFromNetwork(final EthMessage message) { - final Blockchain blockchain = protocolContext.getBlockchain(); - final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); - try { - final Block block = newBlockMessage.block(protocolSchedule); - final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); - - message.getPeer().chainState().update(block.getHeader(), totalDifficulty); - - // Return early if we don't care about this block - final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); - final long bestChainHeight = syncState.bestChainHeight(localChainHeight); - if (!shouldImportBlockAtHeight( - block.getHeader().getNumber(), localChainHeight, bestChainHeight)) { - return; - } - if (pendingBlocks.contains(block.getHash())) { - return; - } - if (blockchain.contains(block.getHash())) { - return; - } - - importOrSavePendingBlock(block); - } catch (final RLPException e) { - message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); - } - } - - private void handleNewBlockHashesFromNetwork(final EthMessage message) { - final Blockchain blockchain = protocolContext.getBlockchain(); - final NewBlockHashesMessage newBlockHashesMessage = - NewBlockHashesMessage.readFrom(message.getData()); - try { - // Register announced blocks - final List announcedBlocks = - Lists.newArrayList(newBlockHashesMessage.getNewHashes()); - for (final NewBlockHash announcedBlock : announcedBlocks) { - message.getPeer().registerKnownBlock(announcedBlock.hash()); - message.getPeer().registerHeight(announcedBlock.hash(), announcedBlock.number()); - } - - // Filter announced blocks for blocks we care to import - final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); - final long bestChainHeight = syncState.bestChainHeight(localChainHeight); - final List relevantAnnouncements = - announcedBlocks - .stream() - .filter(a -> shouldImportBlockAtHeight(a.number(), localChainHeight, bestChainHeight)) - .collect(Collectors.toList()); - - // Filter for blocks we don't yet know about - final List newBlocks = new ArrayList<>(); - for (final NewBlockHash announcedBlock : relevantAnnouncements) { - if (requestedBlocks.contains(announcedBlock.hash())) { - continue; + // Import any pending blocks that are children of the newly added block + readyForImport = pendingBlocks.childrenOf(newBlock.getHash()); } - if (pendingBlocks.contains(announcedBlock.hash())) { - continue; + + if (!readyForImport.isEmpty()) { + final Supplier>> importBlocksTask = + PersistBlockTask.forUnorderedBlocks( + protocolSchedule, + protocolContext, + readyForImport, + HeaderValidationMode.FULL, + ethTasksTimer); + ethContext + .getScheduler() + .scheduleSyncWorkerTask(importBlocksTask) + .whenComplete( + (r, t) -> { + if (r != null) { + LOG.info("Imported {} pending blocks", r.size()); + } + }); } - if (importingBlocks.contains(announcedBlock.hash())) { - continue; + + if (blockAddedEvent.getEventType().equals(EventType.HEAD_ADVANCED)) { + final long head = blockchain.getChainHeadBlockNumber(); + final long cutoff = head + config.blockPropagationRange().lowerEndpoint(); + pendingBlocks.purgeBlocksOlderThan(cutoff); } - if (blockchain.contains(announcedBlock.hash())) { - continue; + } + + /** + * Predicated on the type of information requested, announce a block's availability + * or propagate it to a subset of peers: "Announced block" vs. "Propagated block". + */ + private void broadcastBlock(final Block block, final UInt256 difficulty) { + final List availablePeers = ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); + for (EthPeer ethPeer : availablePeers) { + ethPeer.propagateBlock(block, difficulty); } - if (requestedBlocks.add(announcedBlock.hash())) { - newBlocks.add(announcedBlock); + } + + private void handleNewBlockFromNetwork(final EthMessage message) { + final Blockchain blockchain = protocolContext.getBlockchain(); + final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); + try { + final Block block = newBlockMessage.block(protocolSchedule); + final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); + broadcastBlock(block, totalDifficulty); + + message.getPeer().chainState().update(block.getHeader(), totalDifficulty); + + // Return early if we don't care about this block + final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + if (!shouldImportBlockAtHeight( + block.getHeader().getNumber(), localChainHeight, bestChainHeight)) { + return; + } + if (pendingBlocks.contains(block.getHash())) { + return; + } + if (blockchain.contains(block.getHash())) { + return; + } + + importOrSavePendingBlock(block); + } catch (final RLPException e) { + message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); } - } - - // Process known blocks we care about - for (final NewBlockHash newBlock : newBlocks) { - processAnnouncedBlock(message.getPeer(), newBlock) - .whenComplete((r, t) -> requestedBlocks.remove(newBlock.hash())); - } - } catch (final RLPException e) { - message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); } - } - - private CompletableFuture processAnnouncedBlock( - final EthPeer peer, final NewBlockHash newBlock) { - final AbstractPeerTask getBlockTask = - GetBlockFromPeerTask.create(protocolSchedule, ethContext, newBlock.hash(), ethTasksTimer) - .assignPeer(peer); - - return getBlockTask.run().thenCompose((r) -> importOrSavePendingBlock(r.getResult())); - } - - @VisibleForTesting - CompletableFuture importOrSavePendingBlock(final Block block) { - // Synchronize to avoid race condition where block import event fires after the - // blockchain.contains() check and before the block is registered, causing onBlockAdded() to be - // invoked for the parent of this block before we are able to register it. - synchronized (pendingBlocks) { - if (!protocolContext.getBlockchain().contains(block.getHeader().getParentHash())) { - // Block isn't connected to local chain, save it to pending blocks collection - if (pendingBlocks.registerPendingBlock(block)) { - LOG.info( - "Saving announced block {} ({}) for future import", - block.getHeader().getNumber(), - block.getHash()); + + private void handleNewBlockHashesFromNetwork(final EthMessage message) { + final Blockchain blockchain = protocolContext.getBlockchain(); + final NewBlockHashesMessage newBlockHashesMessage = + NewBlockHashesMessage.readFrom(message.getData()); + try { + // Register announced blocks + final List announcedBlocks = + Lists.newArrayList(newBlockHashesMessage.getNewHashes()); + for (final NewBlockHash announcedBlock : announcedBlocks) { + message.getPeer().registerKnownBlock(announcedBlock.hash()); + message.getPeer().registerHeight(announcedBlock.hash(), announcedBlock.number()); + } + + // Filter announced blocks for blocks we care to import + final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + final List relevantAnnouncements = + announcedBlocks + .stream() + .filter(a -> shouldImportBlockAtHeight(a.number(), localChainHeight, bestChainHeight)) + .collect(Collectors.toList()); + + // Filter for blocks we don't yet know about + final List newBlocks = new ArrayList<>(); + for (final NewBlockHash announcedBlock : relevantAnnouncements) { + if (requestedBlocks.contains(announcedBlock.hash())) { + continue; + } + if (pendingBlocks.contains(announcedBlock.hash())) { + continue; + } + if (importingBlocks.contains(announcedBlock.hash())) { + continue; + } + if (blockchain.contains(announcedBlock.hash())) { + continue; + } + if (requestedBlocks.add(announcedBlock.hash())) { + newBlocks.add(announcedBlock); + } + } + + // Process known blocks we care about + for (final NewBlockHash newBlock : newBlocks) { + processAnnouncedBlock(message.getPeer(), newBlock) + .whenComplete((r, t) -> requestedBlocks.remove(newBlock.hash())); + } + } catch (final RLPException e) { + message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); } - return CompletableFuture.completedFuture(block); - } } - if (!importingBlocks.add(block.getHash())) { - // We're already importing this block. - return CompletableFuture.completedFuture(block); + private CompletableFuture processAnnouncedBlock( + final EthPeer peer, final NewBlockHash newBlock) { + final AbstractPeerTask getBlockTask = + GetBlockFromPeerTask.create(protocolSchedule, ethContext, newBlock.hash(), ethTasksTimer) + .assignPeer(peer); + + return getBlockTask.run().thenCompose((r) -> importOrSavePendingBlock(r.getResult())); } - if (protocolContext.getBlockchain().contains(block.getHash())) { - // We've already imported this block. - importingBlocks.remove(block.getHash()); - return CompletableFuture.completedFuture(block); + @VisibleForTesting + CompletableFuture importOrSavePendingBlock(final Block block) { + // Synchronize to avoid race condition where block import event fires after the + // blockchain.contains() check and before the block is registered, causing onBlockAdded() to be + // invoked for the parent of this block before we are able to register it. + synchronized (pendingBlocks) { + if (!protocolContext.getBlockchain().contains(block.getHeader().getParentHash())) { + // Block isn't connected to local chain, save it to pending blocks collection + if (pendingBlocks.registerPendingBlock(block)) { + LOG.info( + "Saving announced block {} ({}) for future import", + block.getHeader().getNumber(), + block.getHash()); + } + return CompletableFuture.completedFuture(block); + } + } + + if (!importingBlocks.add(block.getHash())) { + // We're already importing this block. + return CompletableFuture.completedFuture(block); + } + + if (protocolContext.getBlockchain().contains(block.getHash())) { + // We've already imported this block. + importingBlocks.remove(block.getHash()); + return CompletableFuture.completedFuture(block); + } + + // Import block + final PersistBlockTask importTask = + PersistBlockTask.create( + protocolSchedule, protocolContext, block, HeaderValidationMode.FULL, ethTasksTimer); + return ethContext + .getScheduler() + .scheduleSyncWorkerTask(importTask::run) + .whenComplete( + (r, t) -> { + importingBlocks.remove(block.getHash()); + if (t != null) { + LOG.warn( + "Failed to import announced block {} ({}).", + block.getHeader().getNumber(), + block.getHash()); + } else { + final double timeInMs = importTask.getTaskTimeInSec() * 1000; + LOG.info( + String.format( + "Successfully imported announced block %d (%s) in %01.3fms.", + block.getHeader().getNumber(), block.getHash(), timeInMs)); + } + }); } - // Import block - final PersistBlockTask importTask = - PersistBlockTask.create( - protocolSchedule, protocolContext, block, HeaderValidationMode.FULL, ethTasksTimer); - return ethContext - .getScheduler() - .scheduleSyncWorkerTask(importTask::run) - .whenComplete( - (r, t) -> { - importingBlocks.remove(block.getHash()); - if (t != null) { - LOG.warn( - "Failed to import announced block {} ({}).", - block.getHeader().getNumber(), - block.getHash()); - } else { - final double timeInMs = importTask.getTaskTimeInSec() * 1000; - LOG.info( - String.format( - "Successfully imported announced block %d (%s) in %01.3fms.", - block.getHeader().getNumber(), block.getHash(), timeInMs)); - } - }); - } - - // Only import blocks within a certain range of our head and sync target - private boolean shouldImportBlockAtHeight( - final long blockNumber, final long localHeight, final long bestChainHeight) { - final long distanceFromLocalHead = blockNumber - localHeight; - final long distanceFromBestPeer = blockNumber - bestChainHeight; - final Range importRange = config.blockPropagationRange(); - return importRange.contains(distanceFromLocalHead) - && importRange.contains(distanceFromBestPeer); - } + // Only import blocks within a certain range of our head and sync target + private boolean shouldImportBlockAtHeight( + final long blockNumber, final long localHeight, final long bestChainHeight) { + final long distanceFromLocalHead = blockNumber - localHeight; + final long distanceFromBestPeer = blockNumber - bestChainHeight; + final Range importRange = config.blockPropagationRange(); + return importRange.contains(distanceFromLocalHead) + && importRange.contains(distanceFromBestPeer); + } } diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 13641d2eb7..1459c355e9 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -12,17 +12,10 @@ */ package tech.pegasys.pantheon.ethereum.eth.sync; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static sun.security.krb5.Confounder.bytes; - import com.google.common.collect.Range; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; import tech.pegasys.pantheon.config.GenesisConfigFile; import tech.pegasys.pantheon.ethereum.ProtocolContext; import tech.pegasys.pantheon.ethereum.chain.Blockchain; @@ -80,9 +73,15 @@ import java.util.function.Function; import java.util.function.Supplier; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static sun.security.krb5.Confounder.bytes; public class BlockPropagationManagerTest { @@ -151,11 +150,11 @@ public void importsAnnouncedBlocks_aheadOfChainInOrder() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage nextAnnouncement = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockHashesMessage nextNextAnnouncement = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); @@ -186,11 +185,11 @@ public void importsAnnouncedBlocks_aheadOfChainOutOfOrder() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage nextAnnouncement = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockHashesMessage nextNextAnnouncement = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); @@ -291,14 +290,14 @@ public void importsMixedOutOfOrderMessages() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage block1Msg = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(block1.getHash(), block1.getHeader().getNumber()))); final NewBlockMessage block2Msg = NewBlockMessage.create( block2, fullBlockchain.getTotalDifficultyByHash(block2.getHash()).get()); final NewBlockHashesMessage block3Msg = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(block3.getHash(), block3.getHeader().getNumber()))); final NewBlockMessage block4Msg = NewBlockMessage.create( @@ -335,7 +334,7 @@ public void handlesDuplicateAnnouncements() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage newBlockHash = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockMessage newBlock = NewBlockMessage.create( @@ -370,7 +369,7 @@ public void handlesPendingDuplicateAnnouncements() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage newBlockHash = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); final NewBlockMessage newBlock = NewBlockMessage.create( @@ -402,7 +401,7 @@ public void ignoresFutureNewBlockHashAnnouncement() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage futureAnnouncement = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(futureBlock.getHash(), futureBlock.getHeader().getNumber()))); // Broadcast @@ -454,7 +453,7 @@ public void ignoresOldNewBlockHashAnnouncement() { final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); final NewBlockHashesMessage oldAnnouncement = NewBlockHashesMessage.create( - singletonList( + Collections.singletonList( new NewBlockHash(oldBlock.getHash(), oldBlock.getHeader().getNumber()))); // Broadcast @@ -553,6 +552,8 @@ public void updatesChainHeadWhenNewBlockMessageReceived() { // Setup peer and messages final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final UInt256 parentTotalDifficulty = + fullBlockchain.getTotalDifficultyByHash(nextBlock.getHeader().getParentHash()).get(); final UInt256 totalDifficulty = fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); final NewBlockMessage nextAnnouncement = NewBlockMessage.create(nextBlock, totalDifficulty); @@ -563,11 +564,11 @@ public void updatesChainHeadWhenNewBlockMessageReceived() { peer.respondWhile(responder, peer::hasOutstandingRequests); assertThat(peer.getEthPeer().chainState().getBestBlock().getHash()) - .isEqualTo(nextBlock.getHash()); + .isEqualTo(nextBlock.getHeader().getParentHash()); assertThat(peer.getEthPeer().chainState().getEstimatedHeight()) - .isEqualTo(nextBlock.getHeader().getNumber()); + .isEqualTo(nextBlock.getHeader().getNumber() - 1); assertThat(peer.getEthPeer().chainState().getBestBlock().getTotalDifficulty()) - .isEqualTo(totalDifficulty); + .isEqualTo(parentTotalDifficulty); } @SuppressWarnings("unchecked") @@ -596,120 +597,4 @@ public void shouldNotImportBlocksThatAreAlreadyBeingImported() { verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); } - - private MutableBlockchain generateBlockchain(BlockchainStorage blockchainStorage, Block genesisBlock, int head) { - DefaultMutableBlockchain blockchain = new DefaultMutableBlockchain(genesisBlock, blockchainStorage, new NoOpMetricsSystem()); - for (int i = 1; i <= head; i++) { - BlockHeader header = - BlockHeaderBuilder.create() - .parentHash(blockchain.getBlockHashByNumber(Long.valueOf(i - 1)).get()) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(Long.valueOf(i)) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody body = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block block = new Block(header, body); - List receipts = Collections.emptyList(); - blockchain.appendBlock(block, receipts); - } - return blockchain; - } - - @Test - public void blockPropagationManager_functionalityAssessment() { - BlockHeader genesisHeader = - BlockHeaderBuilder.create() - .parentHash(Hash.ZERO) - .ommersHash(Hash.ZERO) - .coinbase(Address.fromHexString("0x0000000000000000000000000000000000000000")) - .stateRoot(Hash.ZERO) - .transactionsRoot(Hash.ZERO) - .receiptsRoot(Hash.ZERO) - .logsBloom(new LogsBloomFilter(BytesValue.of(bytes(LogsBloomFilter.BYTE_SIZE)))) - .difficulty(UInt256.ZERO) - .number(0L) - .gasLimit(1L) - .gasUsed(1L) - .timestamp(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond()) - .extraData(Bytes32.wrap(bytes(Bytes32.SIZE))) - .mixHash(Hash.ZERO) - .nonce(0L) - .blockHashFunction(MainnetBlockHashFunction::createHash) - .buildBlockHeader(); - BlockBody genesisBody = new BlockBody(Collections.emptyList(), Collections.emptyList()); - Block genesisBlock = new Block(genesisHeader, genesisBody); - - BlockchainStorage blockchainStorage00 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); - MutableBlockchain blockchain00 = generateBlockchain(blockchainStorage00, genesisBlock, 4); - assertThat(blockchain00.getChainHeadBlockNumber()).isEqualTo(4L); - - BlockchainStorage blockchainStorage01 = new InMemoryStorageProvider().createBlockchainStorage(MainnetProtocolSchedule.create()); - MutableBlockchain blockchain01 = generateBlockchain(blockchainStorage01, genesisBlock, 3); - assertThat(blockchain01.getChainHeadBlockNumber()).isEqualTo(3L); - - SynchronizerConfiguration synchronizerConfiguration = new SynchronizerConfiguration( - SyncMode.FULL, - 500, - .1f, - 5, - Duration.ofMinutes(3), - 0, - 0, - Range.closed(-10L, 30L), - Optional.of(SyncMode.FULL), - 20L, - UInt256.of(1_000_000_000L), - 10, - 5, - 5, - 20, - 3L, - Integer.MAX_VALUE, - 2, - 2, - 0); - - WorldStateArchive worldStateArchive = new WorldStateArchive(new KeyValueStorageWorldStateStorage(new InMemoryKeyValueStorage())); - - EthProtocolManager ethProtocolManager00 = new EthProtocolManager( - blockchain00, - worldStateArchive, - 1, - false, - 200, - new DeterministicEthScheduler(DeterministicEthScheduler.TimeoutPolicy.NEVER)); - - BlockPropagationManager blockPropagationManager00 = new BlockPropagationManager<>( - synchronizerConfiguration, - new ProtocolScheduleBuilder<>(GenesisConfigFile.mainnet().getConfigOptions(), 1, Function.identity(), PrivacyParameters.noPrivacy()).createProtocolSchedule(), - new ProtocolContext(blockchain00, worldStateArchive, null), - ethProtocolManager00.ethContext(), - new SyncState(blockchain00, ethProtocolManager00.ethContext().getEthPeers()), - new PendingBlocks(), - NoOpMetricsSystem.NO_OP_LABELLED_TIMER); - - blockPropagationManager00.start(); - - // Setup peer and messages - final RespondingEthPeer respondingEthPeer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage newBlockHashesMessage = NewBlockHashesMessage.create(singletonList(new NewBlockHash(blockchain00.getBlockHashByNumber(4L).get(), 4L))); - final Responder responder = RespondingEthPeer.blockchainResponder(blockchain01); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, respondingEthPeer, newBlockHashesMessage); - respondingEthPeer.respondWhile(responder, respondingEthPeer::hasOutstandingRequests); - assertThat(blockchain00.contains(blockchain00.getBlockHashByNumber(4L).get())).isTrue(); - } -} +} \ No newline at end of file From 2c3b7e4953d26467e8e50262f987e4e34cbcc678 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Tue, 12 Feb 2019 14:27:31 -0500 Subject: [PATCH 06/11] update --- .../ethereum/eth/manager/EthPeer.java | 4 +- .../eth/manager/EthProtocolManager.java | 14 +- .../eth/sync/BlockPropagationManager.java | 474 ++++---- .../eth/sync/SynchronizerConfiguration.java | 42 +- .../eth/sync/BlockPropagationManagerTest.java | 1071 ++++++++--------- 5 files changed, 786 insertions(+), 819 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index 52c5d0f342..5397cc9ff5 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -133,9 +133,7 @@ public ResponseStream send(final MessageData messageData) throws PeerNotConnecte } } - /** - * Sends block to a peer... - */ + /** Sends block to a peer... */ public void propagateBlock(final Block block, final UInt256 totalDifficulty) { final NewBlockMessage newBlockMessage = NewBlockMessage.create(block, totalDifficulty); final Capability capability = Capability.create("eth", 63); diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java index 4cc7583e40..53373518f6 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthProtocolManager.java @@ -64,13 +64,13 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { private List supportedCapabilities; private final Blockchain blockchain; - public EthProtocolManager( - final Blockchain blockchain, - final WorldStateArchive worldStateArchive, - final int networkId, - final boolean fastSyncEnabled, - final int requestLimit, - final EthScheduler scheduler) { + EthProtocolManager( + final Blockchain blockchain, + final WorldStateArchive worldStateArchive, + final int networkId, + final boolean fastSyncEnabled, + final int requestLimit, + final EthScheduler scheduler) { this.networkId = networkId; this.scheduler = scheduler; diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index cdbf1e70a6..82e107dd6e 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -22,7 +22,6 @@ import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.manager.EthMessage; import tech.pegasys.pantheon.ethereum.eth.manager.EthPeer; -import tech.pegasys.pantheon.ethereum.eth.manager.EthPeers; import tech.pegasys.pantheon.ethereum.eth.messages.EthPV62; import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockHashesMessage; import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockHashesMessage.NewBlockHash; @@ -56,258 +55,259 @@ import org.apache.logging.log4j.Logger; public class BlockPropagationManager { - private static final Logger LOG = LogManager.getLogger(); - - private final SynchronizerConfiguration config; - private final ProtocolSchedule protocolSchedule; - private final ProtocolContext protocolContext; - private final EthContext ethContext; - private final SyncState syncState; - private final LabelledMetric ethTasksTimer; - - private final AtomicBoolean started = new AtomicBoolean(false); - - private final Set requestedBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final Set importingBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final PendingBlocks pendingBlocks; - - BlockPropagationManager( - final SynchronizerConfiguration config, - final ProtocolSchedule protocolSchedule, - final ProtocolContext protocolContext, - final EthContext ethContext, - final SyncState syncState, - final PendingBlocks pendingBlocks, - final LabelledMetric ethTasksTimer) { - this.config = config; - this.protocolSchedule = protocolSchedule; - this.protocolContext = protocolContext; - this.ethContext = ethContext; - this.ethTasksTimer = ethTasksTimer; - - this.syncState = syncState; - this.pendingBlocks = pendingBlocks; + private static final Logger LOG = LogManager.getLogger(); + + private final SynchronizerConfiguration config; + private final ProtocolSchedule protocolSchedule; + private final ProtocolContext protocolContext; + private final EthContext ethContext; + private final SyncState syncState; + private final LabelledMetric ethTasksTimer; + + private final AtomicBoolean started = new AtomicBoolean(false); + + private final Set requestedBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set importingBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final PendingBlocks pendingBlocks; + + BlockPropagationManager( + final SynchronizerConfiguration config, + final ProtocolSchedule protocolSchedule, + final ProtocolContext protocolContext, + final EthContext ethContext, + final SyncState syncState, + final PendingBlocks pendingBlocks, + final LabelledMetric ethTasksTimer) { + this.config = config; + this.protocolSchedule = protocolSchedule; + this.protocolContext = protocolContext; + this.ethContext = ethContext; + this.ethTasksTimer = ethTasksTimer; + + this.syncState = syncState; + this.pendingBlocks = pendingBlocks; + } + + public void start() { + if (started.compareAndSet(false, true)) { + setupListeners(); + } else { + throw new IllegalStateException( + "Attempt to start an already started " + this.getClass().getSimpleName() + "."); } - - public void start() { - if (started.compareAndSet(false, true)) { - setupListeners(); - } else { - throw new IllegalStateException( - "Attempt to start an already started " + this.getClass().getSimpleName() + "."); - } + } + + private void setupListeners() { + protocolContext.getBlockchain().observeBlockAdded(this::onBlockAdded); + ethContext.getEthMessages().subscribe(EthPV62.NEW_BLOCK, this::handleNewBlockFromNetwork); + ethContext + .getEthMessages() + .subscribe(EthPV62.NEW_BLOCK_HASHES, this::handleNewBlockHashesFromNetwork); + } + + private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchain blockchain) { + // Check to see if any of our pending blocks are now ready for import + final Block newBlock = blockAddedEvent.getBlock(); + + final List readyForImport; + synchronized (pendingBlocks) { + // Remove block from pendingBlocks list + pendingBlocks.deregisterPendingBlock(newBlock); + + // Import any pending blocks that are children of the newly added block + readyForImport = pendingBlocks.childrenOf(newBlock.getHash()); } - private void setupListeners() { - protocolContext.getBlockchain().observeBlockAdded(this::onBlockAdded); - ethContext.getEthMessages().subscribe(EthPV62.NEW_BLOCK, this::handleNewBlockFromNetwork); - ethContext - .getEthMessages() - .subscribe(EthPV62.NEW_BLOCK_HASHES, this::handleNewBlockHashesFromNetwork); + if (!readyForImport.isEmpty()) { + final Supplier>> importBlocksTask = + PersistBlockTask.forUnorderedBlocks( + protocolSchedule, + protocolContext, + readyForImport, + HeaderValidationMode.FULL, + ethTasksTimer); + ethContext + .getScheduler() + .scheduleSyncWorkerTask(importBlocksTask) + .whenComplete( + (r, t) -> { + if (r != null) { + LOG.info("Imported {} pending blocks", r.size()); + } + }); } - private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchain blockchain) { - // Check to see if any of our pending blocks are now ready for import - final Block newBlock = blockAddedEvent.getBlock(); - - final List readyForImport; - synchronized (pendingBlocks) { - // Remove block from pendingBlocks list - pendingBlocks.deregisterPendingBlock(newBlock); - - // Import any pending blocks that are children of the newly added block - readyForImport = pendingBlocks.childrenOf(newBlock.getHash()); + if (blockAddedEvent.getEventType().equals(EventType.HEAD_ADVANCED)) { + final long head = blockchain.getChainHeadBlockNumber(); + final long cutoff = head + config.blockPropagationRange().lowerEndpoint(); + pendingBlocks.purgeBlocksOlderThan(cutoff); + } + } + + /** + * Predicated on the type of information requested, announce a block's availability or propagate + * it to a subset of peers: "Announced block" vs. "Propagated block". + */ + private void broadcastBlock(final Block block, final UInt256 difficulty) { + final List availablePeers = + ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); + for (EthPeer ethPeer : availablePeers) { + ethPeer.propagateBlock(block, difficulty); + } + } + + private void handleNewBlockFromNetwork(final EthMessage message) { + final Blockchain blockchain = protocolContext.getBlockchain(); + final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); + try { + final Block block = newBlockMessage.block(protocolSchedule); + final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); + broadcastBlock(block, totalDifficulty); + + message.getPeer().chainState().update(block.getHeader(), totalDifficulty); + + // Return early if we don't care about this block + final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + if (!shouldImportBlockAtHeight( + block.getHeader().getNumber(), localChainHeight, bestChainHeight)) { + return; + } + if (pendingBlocks.contains(block.getHash())) { + return; + } + if (blockchain.contains(block.getHash())) { + return; + } + + importOrSavePendingBlock(block); + } catch (final RLPException e) { + message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); + } + } + + private void handleNewBlockHashesFromNetwork(final EthMessage message) { + final Blockchain blockchain = protocolContext.getBlockchain(); + final NewBlockHashesMessage newBlockHashesMessage = + NewBlockHashesMessage.readFrom(message.getData()); + try { + // Register announced blocks + final List announcedBlocks = + Lists.newArrayList(newBlockHashesMessage.getNewHashes()); + for (final NewBlockHash announcedBlock : announcedBlocks) { + message.getPeer().registerKnownBlock(announcedBlock.hash()); + message.getPeer().registerHeight(announcedBlock.hash(), announcedBlock.number()); + } + + // Filter announced blocks for blocks we care to import + final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + final List relevantAnnouncements = + announcedBlocks + .stream() + .filter(a -> shouldImportBlockAtHeight(a.number(), localChainHeight, bestChainHeight)) + .collect(Collectors.toList()); + + // Filter for blocks we don't yet know about + final List newBlocks = new ArrayList<>(); + for (final NewBlockHash announcedBlock : relevantAnnouncements) { + if (requestedBlocks.contains(announcedBlock.hash())) { + continue; } - - if (!readyForImport.isEmpty()) { - final Supplier>> importBlocksTask = - PersistBlockTask.forUnorderedBlocks( - protocolSchedule, - protocolContext, - readyForImport, - HeaderValidationMode.FULL, - ethTasksTimer); - ethContext - .getScheduler() - .scheduleSyncWorkerTask(importBlocksTask) - .whenComplete( - (r, t) -> { - if (r != null) { - LOG.info("Imported {} pending blocks", r.size()); - } - }); + if (pendingBlocks.contains(announcedBlock.hash())) { + continue; } - - if (blockAddedEvent.getEventType().equals(EventType.HEAD_ADVANCED)) { - final long head = blockchain.getChainHeadBlockNumber(); - final long cutoff = head + config.blockPropagationRange().lowerEndpoint(); - pendingBlocks.purgeBlocksOlderThan(cutoff); + if (importingBlocks.contains(announcedBlock.hash())) { + continue; } - } - - /** - * Predicated on the type of information requested, announce a block's availability - * or propagate it to a subset of peers: "Announced block" vs. "Propagated block". - */ - private void broadcastBlock(final Block block, final UInt256 difficulty) { - final List availablePeers = ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); - for (EthPeer ethPeer : availablePeers) { - ethPeer.propagateBlock(block, difficulty); + if (blockchain.contains(announcedBlock.hash())) { + continue; } - } - - private void handleNewBlockFromNetwork(final EthMessage message) { - final Blockchain blockchain = protocolContext.getBlockchain(); - final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); - try { - final Block block = newBlockMessage.block(protocolSchedule); - final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); - broadcastBlock(block, totalDifficulty); - - message.getPeer().chainState().update(block.getHeader(), totalDifficulty); - - // Return early if we don't care about this block - final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); - final long bestChainHeight = syncState.bestChainHeight(localChainHeight); - if (!shouldImportBlockAtHeight( - block.getHeader().getNumber(), localChainHeight, bestChainHeight)) { - return; - } - if (pendingBlocks.contains(block.getHash())) { - return; - } - if (blockchain.contains(block.getHash())) { - return; - } - - importOrSavePendingBlock(block); - } catch (final RLPException e) { - message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); + if (requestedBlocks.add(announcedBlock.hash())) { + newBlocks.add(announcedBlock); } + } + + // Process known blocks we care about + for (final NewBlockHash newBlock : newBlocks) { + processAnnouncedBlock(message.getPeer(), newBlock) + .whenComplete((r, t) -> requestedBlocks.remove(newBlock.hash())); + } + } catch (final RLPException e) { + message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); } - - private void handleNewBlockHashesFromNetwork(final EthMessage message) { - final Blockchain blockchain = protocolContext.getBlockchain(); - final NewBlockHashesMessage newBlockHashesMessage = - NewBlockHashesMessage.readFrom(message.getData()); - try { - // Register announced blocks - final List announcedBlocks = - Lists.newArrayList(newBlockHashesMessage.getNewHashes()); - for (final NewBlockHash announcedBlock : announcedBlocks) { - message.getPeer().registerKnownBlock(announcedBlock.hash()); - message.getPeer().registerHeight(announcedBlock.hash(), announcedBlock.number()); - } - - // Filter announced blocks for blocks we care to import - final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); - final long bestChainHeight = syncState.bestChainHeight(localChainHeight); - final List relevantAnnouncements = - announcedBlocks - .stream() - .filter(a -> shouldImportBlockAtHeight(a.number(), localChainHeight, bestChainHeight)) - .collect(Collectors.toList()); - - // Filter for blocks we don't yet know about - final List newBlocks = new ArrayList<>(); - for (final NewBlockHash announcedBlock : relevantAnnouncements) { - if (requestedBlocks.contains(announcedBlock.hash())) { - continue; - } - if (pendingBlocks.contains(announcedBlock.hash())) { - continue; - } - if (importingBlocks.contains(announcedBlock.hash())) { - continue; - } - if (blockchain.contains(announcedBlock.hash())) { - continue; - } - if (requestedBlocks.add(announcedBlock.hash())) { - newBlocks.add(announcedBlock); - } - } - - // Process known blocks we care about - for (final NewBlockHash newBlock : newBlocks) { - processAnnouncedBlock(message.getPeer(), newBlock) - .whenComplete((r, t) -> requestedBlocks.remove(newBlock.hash())); - } - } catch (final RLPException e) { - message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL); + } + + private CompletableFuture processAnnouncedBlock( + final EthPeer peer, final NewBlockHash newBlock) { + final AbstractPeerTask getBlockTask = + GetBlockFromPeerTask.create(protocolSchedule, ethContext, newBlock.hash(), ethTasksTimer) + .assignPeer(peer); + + return getBlockTask.run().thenCompose((r) -> importOrSavePendingBlock(r.getResult())); + } + + @VisibleForTesting + CompletableFuture importOrSavePendingBlock(final Block block) { + // Synchronize to avoid race condition where block import event fires after the + // blockchain.contains() check and before the block is registered, causing onBlockAdded() to be + // invoked for the parent of this block before we are able to register it. + synchronized (pendingBlocks) { + if (!protocolContext.getBlockchain().contains(block.getHeader().getParentHash())) { + // Block isn't connected to local chain, save it to pending blocks collection + if (pendingBlocks.registerPendingBlock(block)) { + LOG.info( + "Saving announced block {} ({}) for future import", + block.getHeader().getNumber(), + block.getHash()); } + return CompletableFuture.completedFuture(block); + } } - private CompletableFuture processAnnouncedBlock( - final EthPeer peer, final NewBlockHash newBlock) { - final AbstractPeerTask getBlockTask = - GetBlockFromPeerTask.create(protocolSchedule, ethContext, newBlock.hash(), ethTasksTimer) - .assignPeer(peer); - - return getBlockTask.run().thenCompose((r) -> importOrSavePendingBlock(r.getResult())); + if (!importingBlocks.add(block.getHash())) { + // We're already importing this block. + return CompletableFuture.completedFuture(block); } - @VisibleForTesting - CompletableFuture importOrSavePendingBlock(final Block block) { - // Synchronize to avoid race condition where block import event fires after the - // blockchain.contains() check and before the block is registered, causing onBlockAdded() to be - // invoked for the parent of this block before we are able to register it. - synchronized (pendingBlocks) { - if (!protocolContext.getBlockchain().contains(block.getHeader().getParentHash())) { - // Block isn't connected to local chain, save it to pending blocks collection - if (pendingBlocks.registerPendingBlock(block)) { - LOG.info( - "Saving announced block {} ({}) for future import", - block.getHeader().getNumber(), - block.getHash()); - } - return CompletableFuture.completedFuture(block); - } - } - - if (!importingBlocks.add(block.getHash())) { - // We're already importing this block. - return CompletableFuture.completedFuture(block); - } - - if (protocolContext.getBlockchain().contains(block.getHash())) { - // We've already imported this block. - importingBlocks.remove(block.getHash()); - return CompletableFuture.completedFuture(block); - } - - // Import block - final PersistBlockTask importTask = - PersistBlockTask.create( - protocolSchedule, protocolContext, block, HeaderValidationMode.FULL, ethTasksTimer); - return ethContext - .getScheduler() - .scheduleSyncWorkerTask(importTask::run) - .whenComplete( - (r, t) -> { - importingBlocks.remove(block.getHash()); - if (t != null) { - LOG.warn( - "Failed to import announced block {} ({}).", - block.getHeader().getNumber(), - block.getHash()); - } else { - final double timeInMs = importTask.getTaskTimeInSec() * 1000; - LOG.info( - String.format( - "Successfully imported announced block %d (%s) in %01.3fms.", - block.getHeader().getNumber(), block.getHash(), timeInMs)); - } - }); + if (protocolContext.getBlockchain().contains(block.getHash())) { + // We've already imported this block. + importingBlocks.remove(block.getHash()); + return CompletableFuture.completedFuture(block); } - // Only import blocks within a certain range of our head and sync target - private boolean shouldImportBlockAtHeight( - final long blockNumber, final long localHeight, final long bestChainHeight) { - final long distanceFromLocalHead = blockNumber - localHeight; - final long distanceFromBestPeer = blockNumber - bestChainHeight; - final Range importRange = config.blockPropagationRange(); - return importRange.contains(distanceFromLocalHead) - && importRange.contains(distanceFromBestPeer); - } + // Import block + final PersistBlockTask importTask = + PersistBlockTask.create( + protocolSchedule, protocolContext, block, HeaderValidationMode.FULL, ethTasksTimer); + return ethContext + .getScheduler() + .scheduleSyncWorkerTask(importTask::run) + .whenComplete( + (r, t) -> { + importingBlocks.remove(block.getHash()); + if (t != null) { + LOG.warn( + "Failed to import announced block {} ({}).", + block.getHeader().getNumber(), + block.getHash()); + } else { + final double timeInMs = importTask.getTaskTimeInSec() * 1000; + LOG.info( + String.format( + "Successfully imported announced block %d (%s) in %01.3fms.", + block.getHeader().getNumber(), block.getHash(), timeInMs)); + } + }); + } + + // Only import blocks within a certain range of our head and sync target + private boolean shouldImportBlockAtHeight( + final long blockNumber, final long localHeight, final long bestChainHeight) { + final long distanceFromLocalHead = blockNumber - localHeight; + final long distanceFromBestPeer = blockNumber - bestChainHeight; + final Range importRange = config.blockPropagationRange(); + return importRange.contains(distanceFromLocalHead) + && importRange.contains(distanceFromBestPeer); + } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java index 08ef75ab6b..f8a4a3cc95 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/SynchronizerConfiguration.java @@ -64,27 +64,27 @@ public class SynchronizerConfiguration { private final int transactionsParallelism; private final int computationParallelism; - public SynchronizerConfiguration( - final SyncMode requestedSyncMode, - final int fastSyncPivotDistance, - final float fastSyncFullValidationRate, - final int fastSyncMinimumPeerCount, - final Duration fastSyncMaximumPeerWaitTime, - final int worldStateHashCountPerRequest, - final int worldStateRequestParallelism, - final Range blockPropagationRange, - final Optional syncMode, - final long downloaderChangeTargetThresholdByHeight, - final UInt256 downloaderChangeTargetThresholdByTd, - final int downloaderHeaderRequestSize, - final int downloaderCheckpointTimeoutsPermitted, - final int downloaderChainSegmentTimeoutsPermitted, - final int downloaderChainSegmentSize, - final long trailingPeerBlocksBehindThreshold, - final int maxTrailingPeers, - final int downloaderParallelism, - final int transactionsParallelism, - final int computationParallelism) { + private SynchronizerConfiguration( + final SyncMode requestedSyncMode, + final int fastSyncPivotDistance, + final float fastSyncFullValidationRate, + final int fastSyncMinimumPeerCount, + final Duration fastSyncMaximumPeerWaitTime, + final int worldStateHashCountPerRequest, + final int worldStateRequestParallelism, + final Range blockPropagationRange, + final Optional syncMode, + final long downloaderChangeTargetThresholdByHeight, + final UInt256 downloaderChangeTargetThresholdByTd, + final int downloaderHeaderRequestSize, + final int downloaderCheckpointTimeoutsPermitted, + final int downloaderChainSegmentTimeoutsPermitted, + final int downloaderChainSegmentSize, + final long trailingPeerBlocksBehindThreshold, + final int maxTrailingPeers, + final int downloaderParallelism, + final int transactionsParallelism, + final int computationParallelism) { this.requestedSyncMode = requestedSyncMode; this.fastSyncPivotDistance = fastSyncPivotDistance; this.fastSyncFullValidationRate = fastSyncFullValidationRate; diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 1459c355e9..0c411d043f 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -12,29 +12,20 @@ */ package tech.pegasys.pantheon.ethereum.eth.sync; -import com.google.common.collect.Range; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import tech.pegasys.pantheon.config.GenesisConfigFile; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import tech.pegasys.pantheon.ethereum.ProtocolContext; import tech.pegasys.pantheon.ethereum.chain.Blockchain; -import tech.pegasys.pantheon.ethereum.chain.BlockchainStorage; -import tech.pegasys.pantheon.ethereum.chain.DefaultMutableBlockchain; import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Address; import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockBody; import tech.pegasys.pantheon.ethereum.core.BlockDataGenerator; import tech.pegasys.pantheon.ethereum.core.BlockDataGenerator.BlockOptions; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.BlockHeaderBuilder; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; -import tech.pegasys.pantheon.ethereum.core.LogsBloomFilter; -import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; -import tech.pegasys.pantheon.ethereum.core.TransactionReceipt; -import tech.pegasys.pantheon.ethereum.eth.manager.DeterministicEthScheduler; import tech.pegasys.pantheon.ethereum.eth.manager.EthContext; import tech.pegasys.pantheon.ethereum.eth.manager.EthMessages; import tech.pegasys.pantheon.ethereum.eth.manager.EthPeers; @@ -49,552 +40,530 @@ import tech.pegasys.pantheon.ethereum.eth.messages.NewBlockMessage; import tech.pegasys.pantheon.ethereum.eth.sync.state.PendingBlocks; import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState; -import tech.pegasys.pantheon.ethereum.mainnet.MainnetBlockHashFunction; -import tech.pegasys.pantheon.ethereum.mainnet.MainnetProtocolSchedule; import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolScheduleBuilder; -import tech.pegasys.pantheon.ethereum.storage.keyvalue.KeyValueStorageWorldStateStorage; -import tech.pegasys.pantheon.ethereum.worldstate.WorldStateArchive; import tech.pegasys.pantheon.metrics.LabelledMetric; import tech.pegasys.pantheon.metrics.OperationTimer; import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; -import tech.pegasys.pantheon.services.kvstore.InMemoryKeyValueStorage; -import tech.pegasys.pantheon.util.bytes.Bytes32; -import tech.pegasys.pantheon.util.bytes.BytesValue; import tech.pegasys.pantheon.util.uint.UInt256; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Collections; -import java.util.List; -import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.function.Function; import java.util.function.Supplier; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static sun.security.krb5.Confounder.bytes; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; public class BlockPropagationManagerTest { - private static Blockchain fullBlockchain; - - private BlockchainSetupUtil blockchainUtil; - private ProtocolSchedule protocolSchedule; - private ProtocolContext protocolContext; - private MutableBlockchain blockchain; - private EthProtocolManager ethProtocolManager; - private BlockPropagationManager blockPropagationManager; - private SynchronizerConfiguration syncConfig; - private final PendingBlocks pendingBlocks = new PendingBlocks(); - private SyncState syncState; - private final LabelledMetric ethTasksTimer = - NoOpMetricsSystem.NO_OP_LABELLED_TIMER; - - @BeforeClass - public static void setupSuite() { - fullBlockchain = BlockchainSetupUtil.forTesting().importAllBlocks(); - } - - @Before - public void setup() { - blockchainUtil = BlockchainSetupUtil.forTesting(); - blockchain = spy(blockchainUtil.getBlockchain()); - protocolSchedule = blockchainUtil.getProtocolSchedule(); - final ProtocolContext tempProtocolContext = blockchainUtil.getProtocolContext(); - protocolContext = - new ProtocolContext<>( - blockchain, - tempProtocolContext.getWorldStateArchive(), - tempProtocolContext.getConsensusState()); - ethProtocolManager = - EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); - syncConfig = - SynchronizerConfiguration.builder() - .blockPropagationRange(-3, 5) - .build() - .validated(blockchain); - syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); - blockPropagationManager = - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethProtocolManager.ethContext(), - syncState, - pendingBlocks, - ethTasksTimer); - } - - @Test - public void importsAnnouncedBlocks_aheadOfChainInOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage nextAnnouncement = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockHashesMessage nextNextAnnouncement = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast second message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsAnnouncedBlocks_aheadOfChainOutOfOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage nextAnnouncement = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockHashesMessage nextNextAnnouncement = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast second message first - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsAnnouncedNewBlocks_aheadOfChainInOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage nextAnnouncement = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - final NewBlockMessage nextNextAnnouncement = - NewBlockMessage.create( - nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast second message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsAnnouncedNewBlocks_aheadOfChainOutOfOrder() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - final Block nextNextBlock = blockchainUtil.getBlock(3); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage nextAnnouncement = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - final NewBlockMessage nextNextAnnouncement = - NewBlockMessage.create( - nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast second message first - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); - } - - @Test - public void importsMixedOutOfOrderMessages() { - blockchainUtil.importFirstBlocks(2); - final Block block1 = blockchainUtil.getBlock(2); - final Block block2 = blockchainUtil.getBlock(3); - final Block block3 = blockchainUtil.getBlock(4); - final Block block4 = blockchainUtil.getBlock(5); - - // Sanity check - assertThat(blockchain.contains(block1.getHash())).isFalse(); - assertThat(blockchain.contains(block2.getHash())).isFalse(); - assertThat(blockchain.contains(block3.getHash())).isFalse(); - assertThat(blockchain.contains(block4.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage block1Msg = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(block1.getHash(), block1.getHeader().getNumber()))); - final NewBlockMessage block2Msg = - NewBlockMessage.create( - block2, fullBlockchain.getTotalDifficultyByHash(block2.getHash()).get()); - final NewBlockHashesMessage block3Msg = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(block3.getHash(), block3.getHeader().getNumber()))); - final NewBlockMessage block4Msg = - NewBlockMessage.create( - block4, fullBlockchain.getTotalDifficultyByHash(block4.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast older blocks - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block3Msg); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block4Msg); - peer.respondWhile(responder, peer::hasOutstandingRequests); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block2Msg); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast first block - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block1Msg); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(block1.getHash())).isTrue(); - assertThat(blockchain.contains(block2.getHash())).isTrue(); - assertThat(blockchain.contains(block3.getHash())).isTrue(); - assertThat(blockchain.contains(block4.getHash())).isTrue(); + private static Blockchain fullBlockchain; + + private BlockchainSetupUtil blockchainUtil; + private ProtocolSchedule protocolSchedule; + private ProtocolContext protocolContext; + private MutableBlockchain blockchain; + private EthProtocolManager ethProtocolManager; + private BlockPropagationManager blockPropagationManager; + private SynchronizerConfiguration syncConfig; + private final PendingBlocks pendingBlocks = new PendingBlocks(); + private SyncState syncState; + private final LabelledMetric ethTasksTimer = + NoOpMetricsSystem.NO_OP_LABELLED_TIMER; + + @BeforeClass + public static void setupSuite() { + fullBlockchain = BlockchainSetupUtil.forTesting().importAllBlocks(); + } + + @Before + public void setup() { + blockchainUtil = BlockchainSetupUtil.forTesting(); + blockchain = spy(blockchainUtil.getBlockchain()); + protocolSchedule = blockchainUtil.getProtocolSchedule(); + final ProtocolContext tempProtocolContext = blockchainUtil.getProtocolContext(); + protocolContext = + new ProtocolContext<>( + blockchain, + tempProtocolContext.getWorldStateArchive(), + tempProtocolContext.getConsensusState()); + ethProtocolManager = + EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); + syncConfig = + SynchronizerConfiguration.builder() + .blockPropagationRange(-3, 5) + .build() + .validated(blockchain); + syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); + blockPropagationManager = + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethProtocolManager.ethContext(), + syncState, + pendingBlocks, + ethTasksTimer); + } + + @Test + public void importsAnnouncedBlocks_aheadOfChainInOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage nextAnnouncement = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockHashesMessage nextNextAnnouncement = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast second message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsAnnouncedBlocks_aheadOfChainOutOfOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage nextAnnouncement = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockHashesMessage nextNextAnnouncement = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(nextNextBlock.getHash(), nextNextBlock.getHeader().getNumber()))); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast second message first + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsAnnouncedNewBlocks_aheadOfChainInOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage nextAnnouncement = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + final NewBlockMessage nextNextAnnouncement = + NewBlockMessage.create( + nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast second message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsAnnouncedNewBlocks_aheadOfChainOutOfOrder() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + final Block nextNextBlock = blockchainUtil.getBlock(3); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage nextAnnouncement = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + final NewBlockMessage nextNextAnnouncement = + NewBlockMessage.create( + nextNextBlock, fullBlockchain.getTotalDifficultyByHash(nextNextBlock.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast second message first + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextNextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + assertThat(blockchain.contains(nextNextBlock.getHash())).isTrue(); + } + + @Test + public void importsMixedOutOfOrderMessages() { + blockchainUtil.importFirstBlocks(2); + final Block block1 = blockchainUtil.getBlock(2); + final Block block2 = blockchainUtil.getBlock(3); + final Block block3 = blockchainUtil.getBlock(4); + final Block block4 = blockchainUtil.getBlock(5); + + // Sanity check + assertThat(blockchain.contains(block1.getHash())).isFalse(); + assertThat(blockchain.contains(block2.getHash())).isFalse(); + assertThat(blockchain.contains(block3.getHash())).isFalse(); + assertThat(blockchain.contains(block4.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage block1Msg = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(block1.getHash(), block1.getHeader().getNumber()))); + final NewBlockMessage block2Msg = + NewBlockMessage.create( + block2, fullBlockchain.getTotalDifficultyByHash(block2.getHash()).get()); + final NewBlockHashesMessage block3Msg = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(block3.getHash(), block3.getHeader().getNumber()))); + final NewBlockMessage block4Msg = + NewBlockMessage.create( + block4, fullBlockchain.getTotalDifficultyByHash(block4.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast older blocks + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block3Msg); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block4Msg); + peer.respondWhile(responder, peer::hasOutstandingRequests); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block2Msg); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast first block + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, block1Msg); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(block1.getHash())).isTrue(); + assertThat(blockchain.contains(block2.getHash())).isTrue(); + assertThat(blockchain.contains(block3.getHash())).isTrue(); + assertThat(blockchain.contains(block4.getHash())).isTrue(); + } + + @Test + public void handlesDuplicateAnnouncements() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage newBlockHash = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockMessage newBlock = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + + // Broadcast first message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast duplicate + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); + peer.respondWhile(responder, peer::hasOutstandingRequests); + // Broadcast duplicate + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + verify(blockchain, times(1)).appendBlock(any(), any()); + } + + @Test + public void handlesPendingDuplicateAnnouncements() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + // Sanity check + assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage newBlockHash = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); + final NewBlockMessage newBlock = + NewBlockMessage.create( + nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); + + // Broadcast messages + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); + // Respond + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); + verify(blockchain, times(1)).appendBlock(any(), any()); + } + + @Test + public void ignoresFutureNewBlockHashAnnouncement() { + blockchainUtil.importFirstBlocks(2); + final Block futureBlock = blockchainUtil.getBlock(11); + + // Sanity check + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage futureAnnouncement = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(futureBlock.getHash(), futureBlock.getHeader().getNumber()))); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + } + + @Test + public void ignoresFutureNewBlockAnnouncement() { + blockchainUtil.importFirstBlocks(2); + final Block futureBlock = blockchainUtil.getBlock(11); + + // Sanity check + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage futureAnnouncement = + NewBlockMessage.create( + futureBlock, fullBlockchain.getTotalDifficultyByHash(futureBlock.getHash()).get()); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); + } + + @Test + public void ignoresOldNewBlockHashAnnouncement() { + final BlockDataGenerator gen = new BlockDataGenerator(); + blockchainUtil.importFirstBlocks(10); + final Block blockOne = blockchainUtil.getBlock(1); + final Block oldBlock = gen.nextBlock(blockOne); + + // Sanity check + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + + final BlockPropagationManager propManager = spy(blockPropagationManager); + propManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockHashesMessage oldAnnouncement = + NewBlockHashesMessage.create( + Collections.singletonList( + new NewBlockHash(oldBlock.getHash(), oldBlock.getHeader().getNumber()))); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + verify(propManager, times(0)).importOrSavePendingBlock(any()); + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + } + + @Test + public void ignoresOldNewBlockAnnouncement() { + final BlockDataGenerator gen = new BlockDataGenerator(); + blockchainUtil.importFirstBlocks(10); + final Block blockOne = blockchainUtil.getBlock(1); + final Block oldBlock = gen.nextBlock(blockOne); + + // Sanity check + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + + final BlockPropagationManager propManager = spy(blockPropagationManager); + propManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage oldAnnouncement = NewBlockMessage.create(oldBlock, UInt256.ZERO); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + verify(propManager, times(0)).importOrSavePendingBlock(any()); + assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); + } + + @Test + public void purgesOldBlocks() { + final int oldBlocksToImport = 3; + syncConfig = + SynchronizerConfiguration.builder() + .blockPropagationRange(-oldBlocksToImport, 5) + .build() + .validated(blockchain); + final BlockPropagationManager blockPropagationManager = + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethProtocolManager.ethContext(), + syncState, + pendingBlocks, + ethTasksTimer); + + final BlockDataGenerator gen = new BlockDataGenerator(); + // Import some blocks + blockchainUtil.importFirstBlocks(5); + // Set up test block next to head, that should eventually be purged + final Block blockToPurge = + gen.block(BlockOptions.create().setBlockNumber(blockchain.getChainHeadBlockNumber())); + + blockPropagationManager.start(); + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final NewBlockMessage blockAnnouncementMsg = NewBlockMessage.create(blockToPurge, UInt256.ZERO); + + // Broadcast + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, blockAnnouncementMsg); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + // Check that we pushed our block into the pending collection + assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); + assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); + + // Import blocks until we bury the target block far enough to be cleaned up + for (int i = 0; i < oldBlocksToImport; i++) { + blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); + + assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); + assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); } - @Test - public void handlesDuplicateAnnouncements() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage newBlockHash = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockMessage newBlock = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - - // Broadcast first message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast duplicate - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); - peer.respondWhile(responder, peer::hasOutstandingRequests); - // Broadcast duplicate - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - verify(blockchain, times(1)).appendBlock(any(), any()); - } - - @Test - public void handlesPendingDuplicateAnnouncements() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - // Sanity check - assertThat(blockchain.contains(nextBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage newBlockHash = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(nextBlock.getHash(), nextBlock.getHeader().getNumber()))); - final NewBlockMessage newBlock = - NewBlockMessage.create( - nextBlock, fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get()); - - // Broadcast messages - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlockHash); - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, newBlock); - // Respond - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(nextBlock.getHash())).isTrue(); - verify(blockchain, times(1)).appendBlock(any(), any()); - } - - @Test - public void ignoresFutureNewBlockHashAnnouncement() { - blockchainUtil.importFirstBlocks(2); - final Block futureBlock = blockchainUtil.getBlock(11); - - // Sanity check - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage futureAnnouncement = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(futureBlock.getHash(), futureBlock.getHeader().getNumber()))); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - } - - @Test - public void ignoresFutureNewBlockAnnouncement() { - blockchainUtil.importFirstBlocks(2); - final Block futureBlock = blockchainUtil.getBlock(11); - - // Sanity check - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage futureAnnouncement = - NewBlockMessage.create( - futureBlock, fullBlockchain.getTotalDifficultyByHash(futureBlock.getHash()).get()); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, futureAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(blockchain.contains(futureBlock.getHash())).isFalse(); - } - - @Test - public void ignoresOldNewBlockHashAnnouncement() { - final BlockDataGenerator gen = new BlockDataGenerator(); - blockchainUtil.importFirstBlocks(10); - final Block blockOne = blockchainUtil.getBlock(1); - final Block oldBlock = gen.nextBlock(blockOne); - - // Sanity check - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - - final BlockPropagationManager propManager = spy(blockPropagationManager); - propManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockHashesMessage oldAnnouncement = - NewBlockHashesMessage.create( - Collections.singletonList( - new NewBlockHash(oldBlock.getHash(), oldBlock.getHeader().getNumber()))); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - verify(propManager, times(0)).importOrSavePendingBlock(any()); - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - } - - @Test - public void ignoresOldNewBlockAnnouncement() { - final BlockDataGenerator gen = new BlockDataGenerator(); - blockchainUtil.importFirstBlocks(10); - final Block blockOne = blockchainUtil.getBlock(1); - final Block oldBlock = gen.nextBlock(blockOne); - - // Sanity check - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - - final BlockPropagationManager propManager = spy(blockPropagationManager); - propManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage oldAnnouncement = NewBlockMessage.create(oldBlock, UInt256.ZERO); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, oldAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - verify(propManager, times(0)).importOrSavePendingBlock(any()); - assertThat(blockchain.contains(oldBlock.getHash())).isFalse(); - } - - @Test - public void purgesOldBlocks() { - final int oldBlocksToImport = 3; - syncConfig = - SynchronizerConfiguration.builder() - .blockPropagationRange(-oldBlocksToImport, 5) - .build() - .validated(blockchain); - final BlockPropagationManager blockPropagationManager = - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethProtocolManager.ethContext(), - syncState, - pendingBlocks, - ethTasksTimer); - - final BlockDataGenerator gen = new BlockDataGenerator(); - // Import some blocks - blockchainUtil.importFirstBlocks(5); - // Set up test block next to head, that should eventually be purged - final Block blockToPurge = - gen.block(BlockOptions.create().setBlockNumber(blockchain.getChainHeadBlockNumber())); - - blockPropagationManager.start(); - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final NewBlockMessage blockAnnouncementMsg = NewBlockMessage.create(blockToPurge, UInt256.ZERO); - - // Broadcast - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, blockAnnouncementMsg); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - // Check that we pushed our block into the pending collection - assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); - assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); - - // Import blocks until we bury the target block far enough to be cleaned up - for (int i = 0; i < oldBlocksToImport; i++) { - blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); - - assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); - assertThat(pendingBlocks.contains(blockToPurge.getHash())).isTrue(); - } - - // Import again to trigger cleanup - blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); - assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); - assertThat(pendingBlocks.contains(blockToPurge.getHash())).isFalse(); - } - - @Test - public void updatesChainHeadWhenNewBlockMessageReceived() { - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final UInt256 parentTotalDifficulty = - fullBlockchain.getTotalDifficultyByHash(nextBlock.getHeader().getParentHash()).get(); - final UInt256 totalDifficulty = - fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); - final NewBlockMessage nextAnnouncement = NewBlockMessage.create(nextBlock, totalDifficulty); - - // Broadcast message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer.respondWhile(responder, peer::hasOutstandingRequests); - - assertThat(peer.getEthPeer().chainState().getBestBlock().getHash()) - .isEqualTo(nextBlock.getHeader().getParentHash()); - assertThat(peer.getEthPeer().chainState().getEstimatedHeight()) - .isEqualTo(nextBlock.getHeader().getNumber() - 1); - assertThat(peer.getEthPeer().chainState().getBestBlock().getTotalDifficulty()) - .isEqualTo(parentTotalDifficulty); - } - - @SuppressWarnings("unchecked") - @Test - public void shouldNotImportBlocksThatAreAlreadyBeingImported() { - final EthScheduler ethScheduler = mock(EthScheduler.class); - when(ethScheduler.scheduleSyncWorkerTask(any(Supplier.class))) - .thenReturn(new CompletableFuture<>()); - final EthContext ethContext = - new EthContext("eth", new EthPeers("eth"), new EthMessages(), ethScheduler); - final BlockPropagationManager blockPropagationManager = - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethContext, - syncState, - pendingBlocks, - ethTasksTimer); - - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - blockPropagationManager.importOrSavePendingBlock(nextBlock); - blockPropagationManager.importOrSavePendingBlock(nextBlock); - - verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); - } -} \ No newline at end of file + // Import again to trigger cleanup + blockchainUtil.importBlockAtIndex((int) blockchain.getChainHeadBlockNumber() + 1); + assertThat(blockchain.contains(blockToPurge.getHash())).isFalse(); + assertThat(pendingBlocks.contains(blockToPurge.getHash())).isFalse(); + } + + @Test + public void updatesChainHeadWhenNewBlockMessageReceived() { + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final UInt256 totalDifficulty = + fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); + final NewBlockMessage nextAnnouncement = NewBlockMessage.create(nextBlock, totalDifficulty); + + // Broadcast message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer, nextAnnouncement); + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer.respondWhile(responder, peer::hasOutstandingRequests); + + assertThat(peer.getEthPeer().chainState().getBestBlock().getHash()) + .isEqualTo(nextBlock.getHash()); + assertThat(peer.getEthPeer().chainState().getEstimatedHeight()) + .isEqualTo(nextBlock.getHeader().getNumber()); + assertThat(peer.getEthPeer().chainState().getBestBlock().getTotalDifficulty()) + .isEqualTo(totalDifficulty); + } + + @SuppressWarnings("unchecked") + @Test + public void shouldNotImportBlocksThatAreAlreadyBeingImported() { + final EthScheduler ethScheduler = mock(EthScheduler.class); + when(ethScheduler.scheduleSyncWorkerTask(any(Supplier.class))) + .thenReturn(new CompletableFuture<>()); + final EthContext ethContext = + new EthContext("eth", new EthPeers("eth"), new EthMessages(), ethScheduler); + final BlockPropagationManager blockPropagationManager = + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethContext, + syncState, + pendingBlocks, + ethTasksTimer); + + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + blockPropagationManager.importOrSavePendingBlock(nextBlock); + blockPropagationManager.importOrSavePendingBlock(nextBlock); + + verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); + } +} From 1523c184cdc93ecd0971b19454c4fa997e036143 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Tue, 12 Feb 2019 14:38:39 -0500 Subject: [PATCH 07/11] comments --- .../tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java | 1 - .../pantheon/ethereum/eth/sync/BlockPropagationManager.java | 4 ---- 2 files changed, 5 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index 5397cc9ff5..5c31002a05 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -133,7 +133,6 @@ public ResponseStream send(final MessageData messageData) throws PeerNotConnecte } } - /** Sends block to a peer... */ public void propagateBlock(final Block block, final UInt256 totalDifficulty) { final NewBlockMessage newBlockMessage = NewBlockMessage.create(block, totalDifficulty); final Capability capability = Capability.create("eth", 63); diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index 82e107dd6e..c2a6a729f1 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -144,10 +144,6 @@ private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchai } } - /** - * Predicated on the type of information requested, announce a block's availability or propagate - * it to a subset of peers: "Announced block" vs. "Propagated block". - */ private void broadcastBlock(final Block block, final UInt256 difficulty) { final List availablePeers = ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); From e3d12b657dbd4c2c9f7a1a749de75ed574d9068c Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Tue, 12 Feb 2019 15:38:08 -0500 Subject: [PATCH 08/11] add test --- .../eth/sync/BlockPropagationManager.java | 4 +- .../eth/sync/BlockPropagationManagerTest.java | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index c2a6a729f1..357cece6a8 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -144,7 +144,7 @@ private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchai } } - private void broadcastBlock(final Block block, final UInt256 difficulty) { + void broadcastBlock(final Block block, final UInt256 difficulty) { final List availablePeers = ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); for (EthPeer ethPeer : availablePeers) { @@ -152,7 +152,7 @@ private void broadcastBlock(final Block block, final UInt256 difficulty) { } } - private void handleNewBlockFromNetwork(final EthMessage message) { + void handleNewBlockFromNetwork(final EthMessage message) { final Blockchain blockchain = protocolContext.getBlockchain(); final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); try { diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 0c411d043f..9506adb31e 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -566,4 +566,61 @@ public void shouldNotImportBlocksThatAreAlreadyBeingImported() { verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); } + + @SuppressWarnings("unchecked") + @Test + public void broadcastBlockTest() { + BlockchainSetupUtil blockchainUtil = BlockchainSetupUtil.forTesting(); + + MutableBlockchain blockchain = spy(blockchainUtil.getBlockchain()); + ProtocolSchedule protocolSchedule = blockchainUtil.getProtocolSchedule(); + ProtocolContext tempProtocolContext = blockchainUtil.getProtocolContext(); + ProtocolContext protocolContext = + new ProtocolContext<>( + blockchain, + tempProtocolContext.getWorldStateArchive(), + tempProtocolContext.getConsensusState()); + + EthProtocolManager ethProtocolManager = + EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); + + SynchronizerConfiguration syncConfig = + SynchronizerConfiguration.builder() + .blockPropagationRange(-3, 5) + .build() + .validated(blockchain); + SyncState syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); + BlockPropagationManager blockPropagationManager = + spy( + new BlockPropagationManager<>( + syncConfig, + protocolSchedule, + protocolContext, + ethProtocolManager.ethContext(), + syncState, + pendingBlocks, + ethTasksTimer)); + + blockchainUtil.importFirstBlocks(2); + final Block nextBlock = blockchainUtil.getBlock(2); + + blockPropagationManager.start(); + + // Setup peer and messages + final RespondingEthPeer peer0 = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final RespondingEthPeer peer1 = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + final RespondingEthPeer peer2 = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); + + final UInt256 totalDifficulty = + fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); + final NewBlockMessage newBlockMessage = NewBlockMessage.create(nextBlock, totalDifficulty); + + // Broadcast message + EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer0, newBlockMessage); + + final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); + peer0.respondWhile(responder, peer0::hasOutstandingRequests); + + verify(blockPropagationManager, times(1)).broadcastBlock(any(), any()); + } } From b00ce22a20d8dcbf40e5a67b271b0b1bfdfa568b Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 14 Feb 2019 00:45:49 -0500 Subject: [PATCH 09/11] update --- .../tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java | 3 +-- .../pantheon/ethereum/eth/sync/BlockPropagationManager.java | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index 5c31002a05..3a08f55091 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -135,9 +135,8 @@ public ResponseStream send(final MessageData messageData) throws PeerNotConnecte public void propagateBlock(final Block block, final UInt256 totalDifficulty) { final NewBlockMessage newBlockMessage = NewBlockMessage.create(block, totalDifficulty); - final Capability capability = Capability.create("eth", 63); try { - connection.send(capability, newBlockMessage); + connection.sendForProtocol(protocolName, newBlockMessage); } catch (Exception ignored) { } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index 6d30803bd0..a7b69bfe65 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -158,7 +158,9 @@ void handleNewBlockFromNetwork(final EthMessage message) { try { final Block block = newBlockMessage.block(protocolSchedule); final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); - broadcastBlock(block, totalDifficulty); + + // TODO: Extract broadcast functionality to independent class. + // broadcastBlock(block, totalDifficulty); message.getPeer().chainState().updateForAnnouncedBlock(block.getHeader(), totalDifficulty); From 63cc6e0cb3c113644acc287f58fa69339ab0b543 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 14 Feb 2019 01:04:16 -0500 Subject: [PATCH 10/11] update --- .../tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java | 4 ++-- .../pantheon/ethereum/eth/sync/BlockPropagationManager.java | 2 +- .../ethereum/eth/sync/BlockPropagationManagerTest.java | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java index 3a08f55091..5fc4a56c80 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/manager/EthPeer.java @@ -28,7 +28,6 @@ import tech.pegasys.pantheon.ethereum.p2p.api.MessageData; import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection; import tech.pegasys.pantheon.ethereum.p2p.api.PeerConnection.PeerNotConnected; -import tech.pegasys.pantheon.ethereum.p2p.wire.Capability; import tech.pegasys.pantheon.ethereum.p2p.wire.messages.DisconnectMessage.DisconnectReason; import tech.pegasys.pantheon.util.Subscribers; import tech.pegasys.pantheon.util.bytes.BytesValue; @@ -137,7 +136,8 @@ public void propagateBlock(final Block block, final UInt256 totalDifficulty) { final NewBlockMessage newBlockMessage = NewBlockMessage.create(block, totalDifficulty); try { connection.sendForProtocol(protocolName, newBlockMessage); - } catch (Exception ignored) { + } catch (PeerNotConnected e) { + LOG.trace("Failed to broadcast new block to peer", e); } } diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index a7b69bfe65..e2c5ee86ae 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -160,7 +160,7 @@ void handleNewBlockFromNetwork(final EthMessage message) { final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); // TODO: Extract broadcast functionality to independent class. - // broadcastBlock(block, totalDifficulty); + broadcastBlock(block, totalDifficulty); message.getPeer().chainState().updateForAnnouncedBlock(block.getHeader(), totalDifficulty); diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 161dd2eb1f..1f98dc9649 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -580,10 +580,7 @@ public void broadcastBlockTest() { EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); SynchronizerConfiguration syncConfig = - SynchronizerConfiguration.builder() - .blockPropagationRange(-3, 5) - .build() - .validated(blockchain); + SynchronizerConfiguration.builder().blockPropagationRange(-3, 5).build(); SyncState syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); BlockPropagationManager blockPropagationManager = spy( From 53dad62ab1d23f93cc7a51e32daeb1ee857b3b48 Mon Sep 17 00:00:00 2001 From: "S. Matthew English" Date: Thu, 14 Feb 2019 01:17:41 -0500 Subject: [PATCH 11/11] update --- .../eth/sync/BlockPropagationManager.java | 11 ++-- .../eth/sync/BlockPropagationManagerTest.java | 54 ------------------- 2 files changed, 5 insertions(+), 60 deletions(-) diff --git a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java index e2c5ee86ae..dca15d6539 100644 --- a/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java +++ b/ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManager.java @@ -145,11 +145,10 @@ private void onBlockAdded(final BlockAddedEvent blockAddedEvent, final Blockchai } void broadcastBlock(final Block block, final UInt256 difficulty) { - final List availablePeers = - ethContext.getEthPeers().availablePeers().collect(Collectors.toList()); - for (EthPeer ethPeer : availablePeers) { - ethPeer.propagateBlock(block, difficulty); - } + ethContext + .getEthPeers() + .availablePeers() + .forEach(ethPeer -> ethPeer.propagateBlock(block, difficulty)); } void handleNewBlockFromNetwork(final EthMessage message) { @@ -160,7 +159,7 @@ void handleNewBlockFromNetwork(final EthMessage message) { final UInt256 totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); // TODO: Extract broadcast functionality to independent class. - broadcastBlock(block, totalDifficulty); + // broadcastBlock(block, totalDifficulty); message.getPeer().chainState().updateForAnnouncedBlock(block.getHeader(), totalDifficulty); diff --git a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java index 1f98dc9649..71603e0a02 100644 --- a/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java +++ b/ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/BlockPropagationManagerTest.java @@ -561,58 +561,4 @@ public void shouldNotImportBlocksThatAreAlreadyBeingImported() { verify(ethScheduler, times(1)).scheduleSyncWorkerTask(any(Supplier.class)); } - - @SuppressWarnings("unchecked") - @Test - public void broadcastBlockTest() { - BlockchainSetupUtil blockchainUtil = BlockchainSetupUtil.forTesting(); - - MutableBlockchain blockchain = spy(blockchainUtil.getBlockchain()); - ProtocolSchedule protocolSchedule = blockchainUtil.getProtocolSchedule(); - ProtocolContext tempProtocolContext = blockchainUtil.getProtocolContext(); - ProtocolContext protocolContext = - new ProtocolContext<>( - blockchain, - tempProtocolContext.getWorldStateArchive(), - tempProtocolContext.getConsensusState()); - - EthProtocolManager ethProtocolManager = - EthProtocolManagerTestUtil.create(blockchain, blockchainUtil.getWorldArchive()); - - SynchronizerConfiguration syncConfig = - SynchronizerConfiguration.builder().blockPropagationRange(-3, 5).build(); - SyncState syncState = new SyncState(blockchain, ethProtocolManager.ethContext().getEthPeers()); - BlockPropagationManager blockPropagationManager = - spy( - new BlockPropagationManager<>( - syncConfig, - protocolSchedule, - protocolContext, - ethProtocolManager.ethContext(), - syncState, - pendingBlocks, - ethTasksTimer)); - - blockchainUtil.importFirstBlocks(2); - final Block nextBlock = blockchainUtil.getBlock(2); - - blockPropagationManager.start(); - - // Setup peer and messages - final RespondingEthPeer peer0 = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final RespondingEthPeer peer1 = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - final RespondingEthPeer peer2 = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0); - - final UInt256 totalDifficulty = - fullBlockchain.getTotalDifficultyByHash(nextBlock.getHash()).get(); - final NewBlockMessage newBlockMessage = NewBlockMessage.create(nextBlock, totalDifficulty); - - // Broadcast message - EthProtocolManagerTestUtil.broadcastMessage(ethProtocolManager, peer0, newBlockMessage); - - final Responder responder = RespondingEthPeer.blockchainResponder(fullBlockchain); - peer0.respondWhile(responder, peer0::hasOutstandingRequests); - - verify(blockPropagationManager, times(1)).broadcastBlock(any(), any()); - } }