From 1f432999f190d6f67cba59a56b9898dc046e282d Mon Sep 17 00:00:00 2001 From: Meredith Baxter Date: Mon, 12 Aug 2019 17:52:18 -0400 Subject: [PATCH 1/6] Rearrange import / export utilities for symmetry, export rlp format --- .../java/tech/pegasys/pantheon/Pantheon.java | 12 +- .../pantheon/chainexport/BlockExporter.java | 90 ++++++++++ .../chainexport/RlpBlockExporter.java | 35 ++++ ...inImporter.java => JsonBlockImporter.java} | 11 +- .../RlpBlockImporter.java} | 11 +- .../chainimport/{ => internal}/BlockData.java | 4 +- .../chainimport/{ => internal}/ChainData.java | 2 +- .../{ => internal}/TransactionData.java | 2 +- .../pegasys/pantheon/cli/PantheonCommand.java | 29 ++-- .../subcommands/blocks/BlockExportFormat.java | 17 ++ ...lockFormat.java => BlockImportFormat.java} | 2 +- .../subcommands/blocks/BlocksSubCommand.java | 162 +++++++++++------- .../pegasys/pantheon/util/BlockExporter.java | 88 ---------- .../RlpBlockExporterTest.java} | 94 +++++----- ...erTest.java => JsonBlockImporterTest.java} | 22 +-- .../RlpBlockImporterTest.java} | 15 +- .../pantheon/cli/CommandTestAbstract.java | 36 ++-- .../blocks/BlocksSubCommandTest.java | 13 +- 18 files changed, 373 insertions(+), 272 deletions(-) create mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java create mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ChainImporter.java => JsonBlockImporter.java} (96%) rename pantheon/src/main/java/tech/pegasys/pantheon/{util/BlockImporter.java => chainimport/RlpBlockImporter.java} (96%) rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ => internal}/BlockData.java (96%) rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ => internal}/ChainData.java (94%) rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ => internal}/TransactionData.java (98%) create mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java rename pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/{BlockFormat.java => BlockImportFormat.java} (95%) delete mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java rename pantheon/src/test/java/tech/pegasys/pantheon/{util/BlockExporterTest.java => chainexport/RlpBlockExporterTest.java} (61%) rename pantheon/src/test/java/tech/pegasys/pantheon/chainimport/{ChainImporterTest.java => JsonBlockImporterTest.java} (95%) rename pantheon/src/test/java/tech/pegasys/pantheon/{util/BlockImporterTest.java => chainimport/RlpBlockImporterTest.java} (91%) diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java b/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java index 638de717c2..fe0e549ee9 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java @@ -14,12 +14,12 @@ import static org.apache.logging.log4j.LogManager.getLogger; -import tech.pegasys.pantheon.chainimport.ChainImporter; +import tech.pegasys.pantheon.chainexport.RlpBlockExporter; +import tech.pegasys.pantheon.chainimport.JsonBlockImporter; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.PantheonCommand; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.services.PantheonPluginContextImpl; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import org.apache.logging.log4j.Logger; import picocli.CommandLine.RunLast; @@ -36,9 +36,9 @@ public static void main(final String... args) { final PantheonCommand pantheonCommand = new PantheonCommand( logger, - new BlockImporter(), - new BlockExporter(), - ChainImporter::new, + new RlpBlockImporter(), + JsonBlockImporter::new, + RlpBlockExporter::new, new RunnerBuilder(), new PantheonController.Builder(), new PantheonPluginContextImpl(), diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java new file mode 100644 index 0000000000..a5e2f32130 --- /dev/null +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.chainexport; + +import tech.pegasys.pantheon.ethereum.ProtocolContext; +import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; +import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.core.BlockHeader; +import tech.pegasys.pantheon.ethereum.core.Hash; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Optional; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** Pantheon Block Export Util. */ +public abstract class BlockExporter { + private static final Logger LOG = LogManager.getLogger(); + private final ProtocolContext context; + + protected BlockExporter(final ProtocolContext context) { + this.context = context; + } + + /** + * Export blocks that are stored in Pantheon's block storage. + * + * @param outputFile the path at which to save the exported block data + * @param maybeStartBlock the starting index of the block list to export (inclusive) + * @param maybeEndBlock the ending index of the block list to export (exclusive), if not specified + * a single block will be export + * @throws IOException if an I/O error occurs while writing data to disk + */ + public void exportBlocks( + final File outputFile, + final Optional maybeStartBlock, + final Optional maybeEndBlock) + throws IOException { + + // Get range to export + final MutableBlockchain blockchain = context.getBlockchain(); + final long startBlock = maybeStartBlock.orElse(BlockHeader.GENESIS_BLOCK_NUMBER); + final long endBlock = + maybeEndBlock.orElse( + maybeStartBlock + // If only start is specified, just export one block + .map(start -> start + 1L) + // Otherwise, export everything + .orElse(context.getBlockchain().getChainHeadBlockNumber() + 1L)); + + // Append to file if a range is specified + final boolean append = maybeStartBlock.isPresent(); + FileOutputStream outputStream = new FileOutputStream(outputFile, append); + + LOG.info( + "Exporting blocks [{},{}) to file {} (appending: {})", + startBlock, + endBlock, + outputFile.toString(), + Boolean.toString(append)); + for (long i = startBlock; i < endBlock; i++) { + Optional blockHash = blockchain.getBlockHashByNumber(i); + if (!blockHash.isPresent()) { + LOG.warn("Unable to export blocks [{} - {}). Blocks not found.", i, endBlock); + break; + } + final Block block = blockchain.getBlockByHash(blockHash.get()); + exportBlock(outputStream, block); + } + + outputStream.close(); + LOG.info("Export complete."); + } + + protected abstract void exportBlock(final FileOutputStream outputStream, final Block block) + throws IOException; +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java new file mode 100644 index 0000000000..1388296160 --- /dev/null +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.chainexport; + +import tech.pegasys.pantheon.ethereum.ProtocolContext; +import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.rlp.RLP; +import tech.pegasys.pantheon.util.bytes.BytesValue; + +import java.io.FileOutputStream; +import java.io.IOException; + +public class RlpBlockExporter extends BlockExporter { + + public RlpBlockExporter(final ProtocolContext context) { + super(context); + } + + @Override + protected void exportBlock(final FileOutputStream outputStream, final Block block) + throws IOException { + final BytesValue rlp = RLP.encode(block::writeTo); + outputStream.write(rlp.getArrayUnsafe()); + } +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java similarity index 96% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java index 6f1088f0cb..9aabb91729 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java @@ -12,6 +12,8 @@ */ package tech.pegasys.pantheon.chainimport; +import tech.pegasys.pantheon.chainimport.internal.BlockData; +import tech.pegasys.pantheon.chainimport.internal.ChainData; import tech.pegasys.pantheon.config.GenesisConfigOptions; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.ethereum.blockcreation.MiningCoordinator; @@ -39,13 +41,18 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class ChainImporter { +/** + * Tool for importing blocks with transactions from human-readable json. + * + * @param The consensus algorithm context + */ +public class JsonBlockImporter { private static final Logger LOG = LogManager.getLogger(); private final ObjectMapper mapper; private final PantheonController controller; - public ChainImporter(final PantheonController controller) { + public JsonBlockImporter(final PantheonController controller) { this.controller = controller; mapper = new ObjectMapper(); // Jdk8Module allows us to easily parse {@code Optional} values from json diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java similarity index 96% rename from pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java index 0b3d476999..a9a7cca3a4 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.util; +package tech.pegasys.pantheon.chainimport; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.logging.log4j.LogManager.getLogger; @@ -41,8 +41,8 @@ import com.google.common.base.MoreObjects; import org.apache.logging.log4j.Logger; -/** Pantheon Block Import Util. */ -public class BlockImporter { +/** Tool for importing rlp-encoded block data from files. */ +public class RlpBlockImporter { private static final Logger LOG = getLogger(); private final Semaphore blockBacklog = new Semaphore(2); @@ -60,7 +60,7 @@ public class BlockImporter { * @return the import result * @throws IOException On Failure */ - public BlockImporter.ImportResult importBlockchain( + public RlpBlockImporter.ImportResult importBlockchain( final Path blocks, final PantheonController pantheonController) throws IOException { final ProtocolSchedule protocolSchedule = pantheonController.getProtocolSchedule(); final ProtocolContext context = pantheonController.getProtocolContext(); @@ -125,7 +125,8 @@ public BlockImporter.ImportResult importBlockchain( if (previousBlockFuture != null) { previousBlockFuture.join(); } - return new BlockImporter.ImportResult(blockchain.getChainHead().getTotalDifficulty(), count); + return new RlpBlockImporter.ImportResult( + blockchain.getChainHead().getTotalDifficulty(), count); } finally { validationExecutor.shutdownNow(); try { diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java similarity index 96% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java index 06b20562d3..ef8743aadd 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java @@ -10,9 +10,9 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.chainimport; +package tech.pegasys.pantheon.chainimport.internal; -import tech.pegasys.pantheon.chainimport.TransactionData.NonceProvider; +import tech.pegasys.pantheon.chainimport.internal.TransactionData.NonceProvider; import tech.pegasys.pantheon.ethereum.core.Account; import tech.pegasys.pantheon.ethereum.core.Address; import tech.pegasys.pantheon.ethereum.core.Hash; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java similarity index 94% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java index 7c3127fadc..6b7cd38fca 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.chainimport; +package tech.pegasys.pantheon.chainimport.internal; import java.util.List; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java similarity index 98% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java index fb133ea481..cbcf02e221 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.chainimport; +package tech.pegasys.pantheon.chainimport.internal; import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; import tech.pegasys.pantheon.crypto.SECP256K1.PrivateKey; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java index 9d63456bd2..1428765a25 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java @@ -31,6 +31,7 @@ import tech.pegasys.pantheon.PantheonInfo; import tech.pegasys.pantheon.Runner; import tech.pegasys.pantheon.RunnerBuilder; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.config.EthNetworkConfig; import tech.pegasys.pantheon.cli.config.NetworkName; import tech.pegasys.pantheon.cli.converter.MetricCategoryConverter; @@ -51,7 +52,8 @@ import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader; import tech.pegasys.pantheon.cli.subcommands.RetestethSubCommand; import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand; -import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory; import tech.pegasys.pantheon.cli.subcommands.operator.OperatorSubCommand; import tech.pegasys.pantheon.cli.subcommands.rlp.RLPSubCommand; import tech.pegasys.pantheon.cli.util.ConfigOptionSearchAndRunHandler; @@ -93,8 +95,6 @@ import tech.pegasys.pantheon.services.PantheonPluginContextImpl; import tech.pegasys.pantheon.services.PicoCLIOptionsImpl; import tech.pegasys.pantheon.services.kvstore.RocksDbConfiguration; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.PermissioningConfigurationValidator; import tech.pegasys.pantheon.util.bytes.BytesValue; import tech.pegasys.pantheon.util.number.Fraction; @@ -153,12 +153,12 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { private final Logger logger; - private final ChainImporterFactory chainImporterFactory; private CommandLine commandLine; - private final BlockImporter blockImporter; - private final BlockExporter blockExporter; + private final RlpBlockImporter rlpBlockImporter; + private final JsonBlockImporterFactory jsonBlockImporterFactory; + private final RlpBlockExporterFactory rlpBlockExporterFactory; final NetworkingOptions networkingOptions = NetworkingOptions.create(); final SynchronizerOptions synchronizerOptions = SynchronizerOptions.create(); @@ -633,17 +633,17 @@ void setBannedNodeIds(final List values) { public PantheonCommand( final Logger logger, - final BlockImporter blockImporter, - final BlockExporter blockExporter, - final ChainImporterFactory chainImporterFactory, + final RlpBlockImporter rlpBlockImporter, + final JsonBlockImporterFactory jsonBlockImporterFactory, + final RlpBlockExporterFactory rlpBlockExporterFactory, final RunnerBuilder runnerBuilder, final PantheonController.Builder controllerBuilderFactory, final PantheonPluginContextImpl pantheonPluginContext, final Map environment) { this.logger = logger; - this.blockImporter = blockImporter; - this.blockExporter = blockExporter; - this.chainImporterFactory = chainImporterFactory; + this.rlpBlockImporter = rlpBlockImporter; + this.rlpBlockExporterFactory = rlpBlockExporterFactory; + this.jsonBlockImporterFactory = jsonBlockImporterFactory; this.runnerBuilder = runnerBuilder; this.controllerBuilderFactory = controllerBuilderFactory; this.pantheonPluginContext = pantheonPluginContext; @@ -690,7 +690,10 @@ private PantheonCommand addSubCommands( commandLine.addSubcommand( BlocksSubCommand.COMMAND_NAME, new BlocksSubCommand( - blockImporter, blockExporter, chainImporterFactory, resultHandler.out())); + rlpBlockImporter, + jsonBlockImporterFactory, + rlpBlockExporterFactory, + resultHandler.out())); commandLine.addSubcommand( PublicKeySubCommand.COMMAND_NAME, new PublicKeySubCommand(resultHandler.out(), getKeyLoader())); diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java new file mode 100644 index 0000000000..2fda6eb6bf --- /dev/null +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java @@ -0,0 +1,17 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.cli.subcommands.blocks; + +public enum BlockExportFormat { + RLP +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java similarity index 95% rename from pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java rename to pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java index e7207986c9..45a6575ab1 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java @@ -12,7 +12,7 @@ */ package tech.pegasys.pantheon.cli.subcommands.blocks; -public enum BlockFormat { +public enum BlockImportFormat { RLP, JSON } diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java index 13d48baac3..4a69e53c09 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java @@ -13,34 +13,31 @@ package tech.pegasys.pantheon.cli.subcommands.blocks; import static com.google.common.base.Preconditions.checkNotNull; -import static java.nio.charset.StandardCharsets.UTF_8; import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP; import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP; import static tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.COMMAND_NAME; -import tech.pegasys.pantheon.chainimport.ChainImporter; +import tech.pegasys.pantheon.chainexport.RlpBlockExporter; +import tech.pegasys.pantheon.chainimport.JsonBlockImporter; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.PantheonCommand; import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ExportSubCommand; import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ImportSubCommand; import tech.pegasys.pantheon.controller.PantheonController; +import tech.pegasys.pantheon.ethereum.ProtocolContext; import tech.pegasys.pantheon.ethereum.core.Address; -import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.MiningParameters; import tech.pegasys.pantheon.ethereum.core.Wei; import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration; import tech.pegasys.pantheon.metrics.prometheus.MetricsService; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.bytes.BytesValue; -import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Optional; import io.vertx.core.Vertx; @@ -75,20 +72,21 @@ public class BlocksSubCommand implements Runnable { @Spec private CommandSpec spec; // Picocli injects reference to command spec - private final BlockImporter blockImporter; - private final BlockExporter blockExporter; - private final ChainImporterFactory chainImporterFactory; + private final RlpBlockImporter rlpBlockImporter; + private final JsonBlockImporterFactory jsonBlockImporterFactory; + + private final RlpBlockExporterFactory rlpBlockExporterFactory; private final PrintStream out; public BlocksSubCommand( - final BlockImporter blockImporter, - final BlockExporter blockExporter, - final ChainImporterFactory chainImporterFactory, + final RlpBlockImporter rlpBlockImporter, + final JsonBlockImporterFactory jsonBlockImporterFactory, + final RlpBlockExporterFactory rlpBlockExporterFactory, final PrintStream out) { - this.blockImporter = blockImporter; - this.blockExporter = blockExporter; - this.chainImporterFactory = chainImporterFactory; + this.rlpBlockImporter = rlpBlockImporter; + this.rlpBlockExporterFactory = rlpBlockExporterFactory; + this.jsonBlockImporterFactory = jsonBlockImporterFactory; this.out = out; } @@ -125,7 +123,7 @@ static class ImportSubCommand implements Runnable { description = "The type of data to be imported, possible values are: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", arity = "1..1") - private final BlockFormat format = BlockFormat.RLP; + private final BlockImportFormat format = BlockImportFormat.RLP; @SuppressWarnings("unused") @Spec @@ -136,7 +134,8 @@ public void run() { LOG.info("Import {} block data from {}", format, blocksImportFile); checkCommand(parentCommand); - checkNotNull(parentCommand.blockImporter); + checkNotNull(parentCommand.rlpBlockImporter); + checkNotNull(parentCommand.jsonBlockImporterFactory); Optional metricsService = initMetrics(parentCommand); @@ -199,14 +198,14 @@ private MiningParameters getMiningParameters() { private void importJsonBlocks(final PantheonController controller, final Path path) throws IOException { - ChainImporter importer = parentCommand.chainImporterFactory.get(controller); + JsonBlockImporter importer = parentCommand.jsonBlockImporterFactory.get(controller); final String jsonData = Files.readString(path); importer.importChain(jsonData); } private void importRlpBlocks(final PantheonController controller, final Path path) throws IOException { - parentCommand.blockImporter.importBlockchain(path, controller); + parentCommand.rlpBlockImporter.importBlockchain(path, controller); } } @@ -226,7 +225,6 @@ static class ExportSubCommand implements Runnable { @Option( names = "--start-block", - required = true, paramLabel = MANDATORY_LONG_FORMAT_HELP, description = "the starting index of the block list to export (inclusive)", arity = "1..1") @@ -241,72 +239,107 @@ static class ExportSubCommand implements Runnable { arity = "1..1") private final Long endBlock = null; + @Option( + names = "--format", + hidden = true, + description = + "The format to export, possible values are: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", + arity = "1..1") + private final BlockExportFormat format = BlockExportFormat.RLP; + @Option( names = "--to", + required = true, paramLabel = MANDATORY_FILE_FORMAT_HELP, description = "File to write the block list instead of standard output", arity = "1..1") private File blocksExportFile = null; + @SuppressWarnings("unused") + @Spec + private CommandSpec spec; + @Override public void run() { LOG.info("Runs export sub command"); checkCommand(this, startBlock, endBlock); + final Optional metricsService = initMetrics(parentCommand); - Optional metricsService = initMetrics(parentCommand); - + final PantheonController controller = createPantheonController(); try { - - final BlockExporter.ExportResult exportResult = - parentCommand.blockExporter.exportBlockchain( - parentCommand.parentCommand.buildController(), startBlock, endBlock); - - outputBlock(exportResult.blocks); - - if (exportResult.blocks.isEmpty()) { - throw new ExecutionException(new CommandLine(this), "No block found at the given index"); - } else if (!exportResult.allBlocksAreFound) { - throw new ExecutionException( - new CommandLine(this), - "Partial export due to inability to recover all requested blocks"); + switch (format) { + case RLP: + exportRlpFormat(controller); + break; + default: + throw new ParameterException( + spec.commandLine(), "Unsupported format: " + format.toString()); } - + } catch (IOException e) { + throw new ExecutionException( + spec.commandLine(), "An error occurred while exporting blocks.", e); } finally { metricsService.ifPresent(MetricsService::stop); } } - private static void checkCommand( + private PantheonController createPantheonController() { + return parentCommand.parentCommand.buildController(); + } + + private void exportRlpFormat(final PantheonController controller) throws IOException { + final ProtocolContext context = controller.getProtocolContext(); + RlpBlockExporter exporter = parentCommand.rlpBlockExporterFactory.get(context); + exporter.exportBlocks(blocksExportFile, getStartBlock(), getEndBlock()); + } + + private void checkCommand( final ExportSubCommand exportSubCommand, final Long startBlock, final Long endBlock) { checkNotNull(exportSubCommand.parentCommand); - checkNotNull(exportSubCommand.parentCommand.blockExporter); - checkNotNull(startBlock); - if (startBlock < 0) { - throw new CommandLine.ParameterException( - new CommandLine(exportSubCommand), - "--start-block must be greater than or equal to zero"); - } - if (endBlock != null && startBlock >= endBlock) { - throw new CommandLine.ParameterException( - new CommandLine(exportSubCommand), "--end-block must be greater than --start-block"); - } - } - private void outputBlock(final List blocks) { - if (blocksExportFile != null) { - final Path path = blocksExportFile.toPath(); - try (final BufferedWriter fileWriter = Files.newBufferedWriter(path, UTF_8)) { - fileWriter.write(blocks.toString()); - } catch (final IOException e) { - throw new ExecutionException( - new CommandLine(this), "An error occurred while trying to write the exported blocks"); + final Optional maybeStartBlock = getStartBlock(); + final Optional maybeEndBlock = getEndBlock(); + + maybeStartBlock + .filter(blockNum -> blockNum < 0) + .ifPresent( + (blockNum) -> { + throw new CommandLine.ParameterException( + spec.commandLine(), + "Parameter --start-block must be greater than or equal to zero: " + blockNum); + }); + + maybeEndBlock + .filter(blockNum -> blockNum < 0) + .ifPresent( + (blockNum) -> { + throw new CommandLine.ParameterException( + spec.commandLine(), + "Parameter --end-block must be greater than or equal to zero: " + blockNum); + }); + + if (maybeStartBlock.isPresent() && maybeEndBlock.isPresent()) { + if (endBlock <= startBlock) { + throw new CommandLine.ParameterException( + spec.commandLine(), + "Parameter --end-block (" + + endBlock + + ") must be greater start block (" + + startBlock + + ")."); } - } else { - parentCommand.out.println(blocks.toString()); } } + + private Optional getStartBlock() { + return Optional.ofNullable(startBlock); + } + + private Optional getEndBlock() { + return Optional.ofNullable(endBlock); + } } private static Optional initMetrics(final BlocksSubCommand parentCommand) { @@ -326,7 +359,12 @@ private static Optional initMetrics(final BlocksSubCommand paren } @FunctionalInterface - public interface ChainImporterFactory { - ChainImporter get(PantheonController controller); + public interface JsonBlockImporterFactory { + JsonBlockImporter get(PantheonController controller); + } + + @FunctionalInterface + public interface RlpBlockExporterFactory { + RlpBlockExporter get(ProtocolContext context); } } diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java deleted file mode 100644 index d3300535af..0000000000 --- a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.util; - -import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.Hash; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import com.google.common.base.MoreObjects; - -/** Pantheon Block Export Util. */ -public class BlockExporter { - - /** - * Export blocks that are stored in Pantheon's block storage. - * - * @param pantheonController the PantheonController that defines blockchain behavior - * @param the consensus context type - * @param startBlock the starting index of the block list to export (inclusive) - * @param endBlock the ending index of the block list to export (exclusive), if not specified a - * single block will be export - * @return the export result - */ - public ExportResult exportBlockchain( - final PantheonController pantheonController, final Long startBlock, final Long endBlock) { - - final ProtocolContext context = pantheonController.getProtocolContext(); - final MutableBlockchain blockchain = context.getBlockchain(); - - final Long sanitizedEndBlock = sanitizedEndBlockIndex(startBlock, endBlock); - - final List blocks = new ArrayList<>(); - for (long currentBlockIndex = startBlock; - currentBlockIndex < sanitizedEndBlock; - currentBlockIndex += 1) { - Optional blockHashByNumber = blockchain.getBlockHashByNumber(currentBlockIndex); - blockHashByNumber.ifPresent(hash -> blocks.add(blockchain.getBlockByHash(hash))); - } - - final boolean allBlocksAreFound = blocks.size() == (sanitizedEndBlock - startBlock); - - return new ExportResult(blocks, allBlocksAreFound); - } - - private Long sanitizedEndBlockIndex(final Long startBlock, final Long endBlock) { - if (endBlock == null) { - return startBlock + 1; - } else { - return endBlock; - } - } - - public static final class ExportResult { - - public final List blocks; - - public final boolean allBlocksAreFound; - - ExportResult(final List blocks, final boolean allBlocksAreFound) { - this.blocks = blocks; - this.allBlocksAreFound = allBlocksAreFound; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("blocks", blocks) - .add("allBlocksAreFound", allBlocksAreFound) - .toString(); - } - } -} diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java similarity index 61% rename from pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java rename to pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java index 9f9afd870e..c289c8fe79 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java @@ -10,15 +10,12 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.util; - -import static org.assertj.core.api.Assertions.assertThat; +package tech.pegasys.pantheon.chainexport; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.config.GenesisConfigFile; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.crypto.SECP256K1; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; @@ -34,21 +31,18 @@ import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Test; import org.junit.rules.TemporaryFolder; /** Tests for {@link BlockExporter}. */ -public final class BlockExporterTest { +public final class RlpBlockExporterTest { @ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); private static PantheonController targetController; - private final BlockExporter blockExporter = new BlockExporter(); - @BeforeClass public static void initPantheonController() throws IOException { - final BlockImporter blockImporter = new BlockImporter(); + final RlpBlockImporter blockImporter = new RlpBlockImporter(); final Path dataDir = folder.newFolder().toPath(); final Path source = dataDir.resolve("1000.blocks"); BlockTestUtil.write1000Blocks(source); @@ -70,42 +64,46 @@ public static void initPantheonController() throws IOException { blockImporter.importBlockchain(source, targetController); } - @Test - public void callingBlockExporterWithOnlyStartBlockShouldReturnOneBlock() throws Exception { - - final long startBlock = 0L; - - final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); - - final Block blockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); - - BlockExporter.ExportResult exportResult = - blockExporter.exportBlockchain(targetController, startBlock, null); - - assertThat(exportResult.blocks).contains(blockFromBlockchain); - } - - @Test - public void callingBlockExporterWithStartBlockAndBlockShouldReturnSeveralBlocks() - throws Exception { - - final long startBlock = 0L; - final long endBlock = 1L; - - final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); - - final Block blockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); - - final Block secondBlockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get()); - - BlockExporter.ExportResult exportResult = - blockExporter.exportBlockchain(targetController, startBlock, endBlock); - - assertThat(exportResult.blocks) - .contains(blockFromBlockchain) - .contains(secondBlockFromBlockchain); - } + // protected BlockExporter createBlockExporter() { + // return new RlpBlockExporter(targetController.getProtocolContext()); + // } + + // @Test + // public void callingBlockExporterWithOnlyStartBlockShouldReturnOneBlock() throws Exception { + // + // final long startBlock = 0L; + // + // final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); + // + // final Block blockFromBlockchain = + // blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); + // + // BlockExporter.ExportResult exportResult = + // blockExporter.exportBlocks(targetController, startBlock, null); + // + // assertThat(exportResult.blocks).contains(blockFromBlockchain); + // } + // + // @Test + // public void callingBlockExporterWithStartBlockAndBlockShouldReturnSeveralBlocks() + // throws Exception { + // + // final long startBlock = 0L; + // final long endBlock = 1L; + // + // final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); + // + // final Block blockFromBlockchain = + // blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); + // + // final Block secondBlockFromBlockchain = + // blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get()); + // + // BlockExporter.ExportResult exportResult = + // blockExporter.exportBlocks(targetController, startBlock, endBlock); + // + // assertThat(exportResult.blocks) + // .contains(blockFromBlockchain) + // .contains(secondBlockFromBlockchain); + // } } diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/ChainImporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java similarity index 95% rename from pantheon/src/test/java/tech/pegasys/pantheon/chainimport/ChainImporterTest.java rename to pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java index 595dd20302..0889499793 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/ChainImporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java @@ -54,7 +54,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -public abstract class ChainImporterTest { +public abstract class JsonBlockImporterTest { @Rule public final TemporaryFolder folder = new TemporaryFolder(); @@ -62,14 +62,14 @@ public abstract class ChainImporterTest { protected final GenesisConfigFile genesisConfigFile; protected final boolean isEthash; - public ChainImporterTest(final String consensusEngine) throws IOException { + public JsonBlockImporterTest(final String consensusEngine) throws IOException { this.consensusEngine = consensusEngine; final String genesisData = getFileContents("genesis.json"); this.genesisConfigFile = GenesisConfigFile.fromConfig(genesisData); this.isEthash = genesisConfigFile.getConfigOptions().isEthHash(); } - public static class SingletonTests extends ChainImporterTest { + public static class SingletonTests extends JsonBlockImporterTest { public SingletonTests() throws IOException { super("unsupported"); } @@ -77,7 +77,7 @@ public SingletonTests() throws IOException { @Test public void importChain_unsupportedConsensusAlgorithm() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("clique", "blocks-import-valid.json"); @@ -90,7 +90,7 @@ public void importChain_unsupportedConsensusAlgorithm() throws IOException { } @RunWith(Parameterized.class) - public static class ParameterizedTests extends ChainImporterTest { + public static class ParameterizedTests extends JsonBlockImporterTest { public ParameterizedTests(final String consensusEngine) throws IOException { super(consensusEngine); @@ -105,7 +105,7 @@ public static Collection getParameters() { @Test public void importChain_validJson_withBlockNumbers() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-valid.json"); importer.importChain(jsonData); @@ -196,7 +196,7 @@ public void importChain_validJson_withBlockNumbers() throws IOException { @Test public void importChain_validJson_noBlockIdentifiers() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-valid-no-block-identifiers.json"); importer.importChain(jsonData); @@ -287,7 +287,7 @@ public void importChain_validJson_noBlockIdentifiers() throws IOException { @Test public void importChain_validJson_withParentHashes() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); String jsonData = getFileContents("blocks-import-valid.json"); @@ -338,7 +338,7 @@ public void importChain_validJson_withParentHashes() throws IOException { @Test public void importChain_invalidParent() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-invalid-bad-parent.json"); @@ -350,7 +350,7 @@ public void importChain_invalidParent() throws IOException { @Test public void importChain_invalidTransaction() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-invalid-bad-tx.json"); @@ -363,7 +363,7 @@ public void importChain_invalidTransaction() throws IOException { @Test public void importChain_specialFields() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-special-fields.json"); diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockImporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java similarity index 91% rename from pantheon/src/test/java/tech/pegasys/pantheon/util/BlockImporterTest.java rename to pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java index 56935f2891..47c54cee4b 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockImporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.util; +package tech.pegasys.pantheon.chainimport; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; @@ -39,12 +39,12 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -/** Tests for {@link BlockImporter}. */ -public final class BlockImporterTest { +/** Tests for {@link RlpBlockImporter}. */ +public final class RlpBlockImporterTest { @Rule public final TemporaryFolder folder = new TemporaryFolder(); - BlockImporter blockImporter = new BlockImporter(); + private RlpBlockImporter rlpBlockImporter = new RlpBlockImporter(); @Test public void blockImport() throws IOException { @@ -66,8 +66,8 @@ public void blockImport() throws IOException { .clock(TestClock.fixed()) .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) .build(); - final BlockImporter.ImportResult result = - blockImporter.importBlockchain(source, targetController); + final RlpBlockImporter.ImportResult result = + rlpBlockImporter.importBlockchain(source, targetController); // Don't count the Genesis block assertThat(result.count).isEqualTo(999); assertThat(result.td).isEqualTo(UInt256.of(21991996248790L)); @@ -105,7 +105,8 @@ public void ibftImport() throws IOException { .clock(TestClock.fixed()) .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) .build(); - final BlockImporter.ImportResult result = blockImporter.importBlockchain(source, controller); + final RlpBlockImporter.ImportResult result = rlpBlockImporter + .importBlockchain(source, controller); // Don't count the Genesis block assertThat(result.count).isEqualTo(958); diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java index 39b0edc6d7..9b12b98301 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java @@ -23,7 +23,9 @@ import tech.pegasys.pantheon.Runner; import tech.pegasys.pantheon.RunnerBuilder; -import tech.pegasys.pantheon.chainimport.ChainImporter; +import tech.pegasys.pantheon.chainexport.RlpBlockExporter; +import tech.pegasys.pantheon.chainimport.JsonBlockImporter; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.config.EthNetworkConfig; import tech.pegasys.pantheon.cli.options.EthProtocolOptions; import tech.pegasys.pantheon.cli.options.MetricsCLIOptions; @@ -32,7 +34,8 @@ import tech.pegasys.pantheon.cli.options.SynchronizerOptions; import tech.pegasys.pantheon.cli.options.TransactionPoolOptions; import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader; -import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.controller.PantheonControllerBuilder; import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; @@ -48,8 +51,6 @@ import tech.pegasys.pantheon.ethereum.permissioning.PermissioningConfiguration; import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration; import tech.pegasys.pantheon.services.PantheonPluginContextImpl; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.bytes.BytesValue; import java.io.ByteArrayOutputStream; @@ -101,9 +102,9 @@ public abstract class CommandTestAbstract { @Mock protected ProtocolContext mockProtocolContext; @Mock protected BlockBroadcaster mockBlockBroadcaster; @Mock protected PantheonController mockController; - @Mock protected BlockImporter mockBlockImporter; - @Mock protected BlockExporter mockBlockExporter; - @Mock protected ChainImporter chainImporter; + @Mock protected RlpBlockExporter mockRlpBlockExporter; + @Mock protected JsonBlockImporter jsonBlockImporter; + @Mock protected RlpBlockImporter mockRlpBlockImporter; @Mock protected Logger mockLogger; @Mock protected PantheonPluginContextImpl mockPantheonPluginContext; @@ -215,8 +216,9 @@ protected TestPantheonCommand parseCommand(final InputStream in, final String... } @SuppressWarnings("unchecked") - private ChainImporter chainImporterFactory(final PantheonController controller) { - return (ChainImporter) chainImporter; + private JsonBlockImporter jsonBlockImporterFactory( + final PantheonController controller) { + return (JsonBlockImporter) jsonBlockImporter; } private TestPantheonCommand parseCommand( @@ -227,9 +229,9 @@ private TestPantheonCommand parseCommand( final TestPantheonCommand pantheonCommand = new TestPantheonCommand( mockLogger, - mockBlockImporter, - mockBlockExporter, - this::chainImporterFactory, + mockRlpBlockImporter, + this::jsonBlockImporterFactory, + (context) -> mockRlpBlockExporter, mockRunnerBuilder, mockControllerBuilderFactory, keyLoader, @@ -257,9 +259,9 @@ protected KeyLoader getKeyLoader() { TestPantheonCommand( final Logger mockLogger, - final BlockImporter mockBlockImporter, - final BlockExporter mockBlockExporter, - final ChainImporterFactory chainImporterFactory, + final RlpBlockImporter mockBlockImporter, + final JsonBlockImporterFactory jsonBlockImporterFactory, + final RlpBlockExporterFactory rlpBlockExporterFactory, final RunnerBuilder mockRunnerBuilder, final PantheonController.Builder controllerBuilderFactory, final KeyLoader keyLoader, @@ -268,8 +270,8 @@ protected KeyLoader getKeyLoader() { super( mockLogger, mockBlockImporter, - mockBlockExporter, - chainImporterFactory, + jsonBlockImporterFactory, + rlpBlockExporterFactory, mockRunnerBuilder, controllerBuilderFactory, pantheonPluginContext, diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java index 2c993bc109..9162d70f26 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.CommandTestAbstract; import tech.pegasys.pantheon.config.GenesisConfigFile; import tech.pegasys.pantheon.controller.PantheonController; @@ -36,8 +37,6 @@ import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; import tech.pegasys.pantheon.testutil.BlockTestUtil; import tech.pegasys.pantheon.testutil.TestClock; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import java.io.File; import java.io.IOException; @@ -161,7 +160,7 @@ public void callingBlockImportSubCommandWithPathMustImportBlocksWithThisPath() t parseCommand( BLOCK_SUBCOMMAND_NAME, BLOCK_IMPORT_SUBCOMMAND_NAME, "--from", fileToImport.getPath()); - verify(mockBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); + verify(mockRlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath()); @@ -180,7 +179,7 @@ public void blocksImport_rlpFormat() throws Exception { "--from", fileToImport.getPath()); - verify(mockBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); + verify(mockRlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath()); @@ -207,7 +206,7 @@ public void blocksImport_jsonFormat() throws Exception { assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).isEmpty(); - verify(chainImporter, times(1)).importChain(stringArgumentCaptor.capture()); + verify(jsonBlockImporter, times(1)).importChain(stringArgumentCaptor.capture()); assertThat(stringArgumentCaptor.getValue()).isEqualTo(fileContent); } @@ -263,8 +262,6 @@ public void callingExportSubCommandWithFilePathMustWriteBlockInThisFile() throws final MutableBlockchain blockchain = pantheonController.getProtocolContext().getBlockchain(); final File outputFile = File.createTempFile("export", "store"); - mockBlockExporter = new BlockExporter(); - doReturn(pantheonController).when(mockControllerBuilder).build(); parseCommand( @@ -288,7 +285,7 @@ public void callingExportSubCommandWithFilePathMustWriteBlockInThisFile() throws } private PantheonController initPantheonController() throws IOException { - final BlockImporter blockImporter = new BlockImporter(); + final RlpBlockImporter blockImporter = new RlpBlockImporter(); final Path dataDir = folder.newFolder().toPath(); final Path source = dataDir.resolve("1000.blocks"); BlockTestUtil.write1000Blocks(source); From 4391172d291f74b67e0021c5c674898e91039b21 Mon Sep 17 00:00:00 2001 From: Meredith Baxter Date: Tue, 13 Aug 2019 10:30:36 -0400 Subject: [PATCH 2/6] Update subcommand tests --- .../subcommands/blocks/BlocksSubCommand.java | 10 +- .../chainimport/RlpBlockImporterTest.java | 4 +- .../pantheon/cli/CommandTestAbstract.java | 8 +- .../blocks/BlocksSubCommandTest.java | 185 ++++++++++-------- 4 files changed, 117 insertions(+), 90 deletions(-) diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java index 4a69e53c09..13093d3e01 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java @@ -262,7 +262,7 @@ static class ExportSubCommand implements Runnable { @Override public void run() { - LOG.info("Runs export sub command"); + LOG.info("Export {} block data to {}", format, blocksExportFile.toPath()); checkCommand(this, startBlock, endBlock); final Optional metricsService = initMetrics(parentCommand); @@ -308,7 +308,9 @@ private void checkCommand( (blockNum) -> { throw new CommandLine.ParameterException( spec.commandLine(), - "Parameter --start-block must be greater than or equal to zero: " + blockNum); + "Parameter --start-block (" + + blockNum + + ") must be greater than or equal to zero."); }); maybeEndBlock @@ -317,7 +319,9 @@ private void checkCommand( (blockNum) -> { throw new CommandLine.ParameterException( spec.commandLine(), - "Parameter --end-block must be greater than or equal to zero: " + blockNum); + "Parameter --end-block (" + + blockNum + + ") must be greater than or equal to zero."); }); if (maybeStartBlock.isPresent() && maybeEndBlock.isPresent()) { diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java index 47c54cee4b..d51282b431 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java @@ -105,8 +105,8 @@ public void ibftImport() throws IOException { .clock(TestClock.fixed()) .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) .build(); - final RlpBlockImporter.ImportResult result = rlpBlockImporter - .importBlockchain(source, controller); + final RlpBlockImporter.ImportResult result = + rlpBlockImporter.importBlockchain(source, controller); // Don't count the Genesis block assertThat(result.count).isEqualTo(958); diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java index 9b12b98301..574ec62987 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java @@ -102,9 +102,9 @@ public abstract class CommandTestAbstract { @Mock protected ProtocolContext mockProtocolContext; @Mock protected BlockBroadcaster mockBlockBroadcaster; @Mock protected PantheonController mockController; - @Mock protected RlpBlockExporter mockRlpBlockExporter; + @Mock protected RlpBlockExporter rlpBlockExporter; @Mock protected JsonBlockImporter jsonBlockImporter; - @Mock protected RlpBlockImporter mockRlpBlockImporter; + @Mock protected RlpBlockImporter rlpBlockImporter; @Mock protected Logger mockLogger; @Mock protected PantheonPluginContextImpl mockPantheonPluginContext; @@ -229,9 +229,9 @@ private TestPantheonCommand parseCommand( final TestPantheonCommand pantheonCommand = new TestPantheonCommand( mockLogger, - mockRlpBlockImporter, + rlpBlockImporter, this::jsonBlockImporterFactory, - (context) -> mockRlpBlockExporter, + (context) -> rlpBlockExporter, mockRunnerBuilder, mockControllerBuilderFactory, keyLoader, diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java index 9162d70f26..3d99c78f3c 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java @@ -14,35 +14,18 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.contentOf; -import static org.assertj.core.util.Arrays.asList; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.CommandTestAbstract; -import tech.pegasys.pantheon.config.GenesisConfigFile; -import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.crypto.SECP256K1; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; -import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; -import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; -import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; -import tech.pegasys.pantheon.testutil.BlockTestUtil; -import tech.pegasys.pantheon.testutil.TestClock; import java.io.File; import java.io.IOException; import java.io.Writer; import java.nio.file.Files; -import java.nio.file.Path; +import java.util.Optional; import org.junit.Rule; import org.junit.Test; @@ -86,9 +69,9 @@ public class BlocksSubCommandTest extends CommandTestAbstract { + System.lineSeparator(); private static final String EXPECTED_BLOCK_EXPORT_USAGE = - "Usage: pantheon blocks export [-hV] [--end-block=] --start-block=" + "Usage: pantheon blocks export [-hV] [--end-block=] [--start-block=]" + System.lineSeparator() - + " [--to=]" + + " --to=" + System.lineSeparator() + "This command export a specific block from storage" + System.lineSeparator() @@ -160,7 +143,7 @@ public void callingBlockImportSubCommandWithPathMustImportBlocksWithThisPath() t parseCommand( BLOCK_SUBCOMMAND_NAME, BLOCK_IMPORT_SUBCOMMAND_NAME, "--from", fileToImport.getPath()); - verify(mockRlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); + verify(rlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath()); @@ -179,7 +162,7 @@ public void blocksImport_rlpFormat() throws Exception { "--from", fileToImport.getPath()); - verify(mockRlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); + verify(rlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath()); @@ -212,99 +195,139 @@ public void blocksImport_jsonFormat() throws Exception { // Export sub-sub-command @Test - public void callingBlockExportSubCommandWithoutStartBlockMustDisplayErrorAndUsage() { + public void blocksExport_missingFileParam() { parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME); - final String expectedErrorOutputStart = "Missing required option '--start-block='"; + final String expectedErrorOutputStart = "Missing required option '--to='"; assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); } @Test - public void callingBlockExportSubCommandHelpMustDisplayUsage() { - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--help"); - assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE); + public void blocksExport_noStartOrEnd() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", outputFile.getPath()); + assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.empty(), Optional.empty()); } @Test - public void callingBlockExportSubCommandWithNegativeStartBlockMustDisplayErrorAndUsage() { - final String expectedErrorOutputStart = "--start-block must be greater than or equal to zero"; - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=-1"); + public void blocksExport_withStartAndNoEnd() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=1"); assertThat(commandOutput.toString()).isEmpty(); - assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.of(1L), Optional.empty()); } @Test - public void - callingBlockExportSubCommandWithEndBlockGreaterThanStartBlockMustDisplayErrorMessage() { - final String expectedErrorOutputStart = "--end-block must be greater than --start-block"; + public void blocksExport_withEndAndNoStart() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); parseCommand( - BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=2", "--end-block=1"); + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--end-block=10"); assertThat(commandOutput.toString()).isEmpty(); - assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.empty(), Optional.of(10L)); } @Test - public void callingBlockExportSubCommandWithEndBlockEqualToStartBlockMustDisplayErrorMessage() { - final String expectedErrorOutputStart = "--end-block must be greater than --start-block"; + public void blocksExport_withStartAndEnd() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); parseCommand( - BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=2", "--end-block=2"); + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=1", + "--end-block=10"); assertThat(commandOutput.toString()).isEmpty(); - assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.of(1L), Optional.of(10L)); } @Test - public void callingExportSubCommandWithFilePathMustWriteBlockInThisFile() throws Exception { - - final long startBlock = 0L; - final long endBlock = 2L; - final PantheonController pantheonController = initPantheonController(); - final MutableBlockchain blockchain = pantheonController.getProtocolContext().getBlockchain(); - final File outputFile = File.createTempFile("export", "store"); + public void blocksExport_withOutOfOrderStartAndEnd() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=10", + "--end-block=1"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --end-block (1) must be greater start block (10)"); + assertThat(commandOutput.toString()).isEmpty(); - doReturn(pantheonController).when(mockControllerBuilder).build(); + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + @Test + public void blocksExport_withEmptyRange() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); parseCommand( BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, - "--start-block=" + startBlock, - "--end-block=" + endBlock, - "--to=" + outputFile.getPath()); + "--to", + outputFile.getPath(), + "--start-block=10", + "--end-block=10"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --end-block (10) must be greater start block (10)"); + assertThat(commandOutput.toString()).isEmpty(); - final Block blockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } - final Block secondBlockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get()); + @Test + public void blocksExport_withInvalidStart() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=-1"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --start-block (-1) must be greater than or equal to zero"); + assertThat(commandOutput.toString()).isEmpty(); - assertThat(contentOf(outputFile)) - .isEqualTo(asList(new Block[] {blockFromBlockchain, secondBlockFromBlockchain}).toString()); + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + @Test + public void blocksExport_withInvalidEnd() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--end-block=-1"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --end-block (-1) must be greater than or equal to zero"); assertThat(commandOutput.toString()).isEmpty(); - assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); } - private PantheonController initPantheonController() throws IOException { - final RlpBlockImporter blockImporter = new RlpBlockImporter(); - final Path dataDir = folder.newFolder().toPath(); - final Path source = dataDir.resolve("1000.blocks"); - BlockTestUtil.write1000Blocks(source); - final PantheonController targetController = - new PantheonController.Builder() - .fromGenesisConfig(GenesisConfigFile.mainnet()) - .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) - .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) - .storageProvider(new InMemoryStorageProvider()) - .networkId(1) - .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) - .nodeKeys(SECP256K1.KeyPair.generate()) - .metricsSystem(new NoOpMetricsSystem()) - .privacyParameters(PrivacyParameters.DEFAULT) - .dataDirectory(dataDir) - .clock(TestClock.fixed()) - .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) - .build(); - blockImporter.importBlockchain(source, targetController); - return targetController; + @Test + public void callingBlockExportSubCommandHelpMustDisplayUsage() { + parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--help"); + assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE); + assertThat(commandErrorOutput.toString()).isEmpty(); } } From 9614d6cf2480d516340d73234aa8e37d00b3f950 Mon Sep 17 00:00:00 2001 From: Meredith Baxter Date: Tue, 13 Aug 2019 11:26:34 -0400 Subject: [PATCH 3/6] Error out if there is no data to export --- .../pegasys/pantheon/cli/PantheonCommand.java | 2 +- .../subcommands/blocks/BlocksSubCommand.java | 19 ++++- .../blocks/BlocksSubCommandTest.java | 85 ++++++++++++++++++- 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java index 1428765a25..743424f621 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java @@ -1366,7 +1366,7 @@ private File genesisFile() { } } - private Path dataDir() { + public Path dataDir() { if (isFullInstantiation()) { return standaloneCommands.dataPath.toAbsolutePath(); } else if (isDocker) { diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java index 13093d3e01..47ed03c06a 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java @@ -38,6 +38,7 @@ import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Optional; import io.vertx.core.Vertx; @@ -262,7 +263,7 @@ static class ExportSubCommand implements Runnable { @Override public void run() { - LOG.info("Export {} block data to {}", format, blocksExportFile.toPath()); + LOG.info("Export {} block data to file {}", format, blocksExportFile.toPath()); checkCommand(this, startBlock, endBlock); final Optional metricsService = initMetrics(parentCommand); @@ -335,6 +336,22 @@ private void checkCommand( + ")."); } } + + // Error if data directory is empty + Path databasePath = + Paths.get( + parentCommand.parentCommand.dataDir().toAbsolutePath().toString(), + PantheonController.DATABASE_PATH); + File databaseDirectory = new File(databasePath.toString()); + if (!databaseDirectory.exists() + || !databaseDirectory.isDirectory() + || databaseDirectory.list().length == 0) { + // Empty data directory, nothing to export + throw new CommandLine.ParameterException( + spec.commandLine(), + "Chain is empty. Unable to export blocks from specified data directory: " + + databaseDirectory.toString()); + } } private Optional getStartBlock() { diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java index 3d99c78f3c..6f745283e2 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java @@ -20,11 +20,14 @@ import static org.mockito.Mockito.verify; import tech.pegasys.pantheon.cli.CommandTestAbstract; +import tech.pegasys.pantheon.controller.PantheonController; import java.io.File; import java.io.IOException; import java.io.Writer; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Optional; import org.junit.Rule; @@ -195,17 +198,70 @@ public void blocksImport_jsonFormat() throws Exception { // Export sub-sub-command @Test - public void blocksExport_missingFileParam() { - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME); + public void blocksExport_missingFileParam() throws IOException { + createDbDirectory(true); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME); final String expectedErrorOutputStart = "Missing required option '--to='"; assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + + @Test + public void blocksExport_noDbDirectory() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath()); + final String expectedErrorOutputStart = + "Chain is empty. Unable to export blocks from specified data directory: " + + folder.getRoot().getAbsolutePath() + + "/" + + PantheonController.DATABASE_PATH; + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + + @Test + public void blocksExport_emptyDbDirectory() throws IOException { + createDbDirectory(false); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath()); + final String expectedErrorOutputStart = + "Chain is empty. Unable to export blocks from specified data directory: " + + folder.getRoot().getAbsolutePath() + + "/" + + PantheonController.DATABASE_PATH; + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); } @Test public void blocksExport_noStartOrEnd() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", outputFile.getPath()); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath()); assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).isEmpty(); @@ -214,8 +270,10 @@ public void blocksExport_noStartOrEnd() throws IOException { @Test public void blocksExport_withStartAndNoEnd() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -229,8 +287,10 @@ public void blocksExport_withStartAndNoEnd() throws IOException { @Test public void blocksExport_withEndAndNoStart() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -244,8 +304,10 @@ public void blocksExport_withEndAndNoStart() throws IOException { @Test public void blocksExport_withStartAndEnd() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -260,8 +322,10 @@ public void blocksExport_withStartAndEnd() throws IOException { @Test public void blocksExport_withOutOfOrderStartAndEnd() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -277,8 +341,10 @@ public void blocksExport_withOutOfOrderStartAndEnd() throws IOException { @Test public void blocksExport_withEmptyRange() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -294,8 +360,10 @@ public void blocksExport_withEmptyRange() throws IOException { @Test public void blocksExport_withInvalidStart() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -310,8 +378,10 @@ public void blocksExport_withInvalidStart() throws IOException { @Test public void blocksExport_withInvalidEnd() throws IOException { + createDbDirectory(true); final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--to", @@ -330,4 +400,13 @@ public void callingBlockExportSubCommandHelpMustDisplayUsage() { assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE); assertThat(commandErrorOutput.toString()).isEmpty(); } + + private void createDbDirectory(final boolean createDataFiles) throws IOException { + File dbDir = folder.newFolder(PantheonController.DATABASE_PATH); + if (createDataFiles) { + Path dataFilePath = Paths.get(dbDir.getAbsolutePath(), "0000001.sst"); + final boolean success = new File(dataFilePath.toString()).createNewFile(); + assertThat(success).isTrue(); + } + } } From eb840cfe5ab8ad3ffacc3aa93e0f27fc5670024d Mon Sep 17 00:00:00 2001 From: Meredith Baxter Date: Wed, 14 Aug 2019 10:24:05 -0400 Subject: [PATCH 4/6] Pass blockchain to BlockExporter utility --- .../pantheon/chainexport/BlockExporter.java | 20 +++++++++++-------- .../chainexport/RlpBlockExporter.java | 6 +++--- .../subcommands/blocks/BlocksSubCommand.java | 6 ++++-- .../pantheon/cli/CommandTestAbstract.java | 2 +- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java index a5e2f32130..d3f4830cd1 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java @@ -12,8 +12,7 @@ */ package tech.pegasys.pantheon.chainexport; -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; +import tech.pegasys.pantheon.ethereum.chain.Blockchain; import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.BlockHeader; import tech.pegasys.pantheon.ethereum.core.Hash; @@ -29,10 +28,10 @@ /** Pantheon Block Export Util. */ public abstract class BlockExporter { private static final Logger LOG = LogManager.getLogger(); - private final ProtocolContext context; + private final Blockchain blockchain; - protected BlockExporter(final ProtocolContext context) { - this.context = context; + protected BlockExporter(final Blockchain blockchain) { + this.blockchain = blockchain; } /** @@ -51,7 +50,6 @@ public void exportBlocks( throws IOException { // Get range to export - final MutableBlockchain blockchain = context.getBlockchain(); final long startBlock = maybeStartBlock.orElse(BlockHeader.GENESIS_BLOCK_NUMBER); final long endBlock = maybeEndBlock.orElse( @@ -59,7 +57,7 @@ public void exportBlocks( // If only start is specified, just export one block .map(start -> start + 1L) // Otherwise, export everything - .orElse(context.getBlockchain().getChainHeadBlockNumber() + 1L)); + .orElse(blockchain.getChainHeadBlockNumber() + 1L)); // Append to file if a range is specified final boolean append = maybeStartBlock.isPresent(); @@ -73,11 +71,17 @@ public void exportBlocks( Boolean.toString(append)); for (long i = startBlock; i < endBlock; i++) { Optional blockHash = blockchain.getBlockHashByNumber(i); - if (!blockHash.isPresent()) { + if (blockHash.isEmpty()) { LOG.warn("Unable to export blocks [{} - {}). Blocks not found.", i, endBlock); break; } + final Block block = blockchain.getBlockByHash(blockHash.get()); + final long blockNumber = block.getHeader().getNumber(); + if (blockNumber % 100 == 0) { + LOG.info("Export at block {}", blockNumber); + } + exportBlock(outputStream, block); } diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java index 1388296160..63b348f7a6 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java @@ -12,7 +12,7 @@ */ package tech.pegasys.pantheon.chainexport; -import tech.pegasys.pantheon.ethereum.ProtocolContext; +import tech.pegasys.pantheon.ethereum.chain.Blockchain; import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.rlp.RLP; import tech.pegasys.pantheon.util.bytes.BytesValue; @@ -22,8 +22,8 @@ public class RlpBlockExporter extends BlockExporter { - public RlpBlockExporter(final ProtocolContext context) { - super(context); + public RlpBlockExporter(final Blockchain blockchain) { + super(blockchain); } @Override diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java index 47ed03c06a..648829c0dc 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java @@ -25,6 +25,7 @@ import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ImportSubCommand; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.ethereum.ProtocolContext; +import tech.pegasys.pantheon.ethereum.chain.Blockchain; import tech.pegasys.pantheon.ethereum.core.Address; import tech.pegasys.pantheon.ethereum.core.MiningParameters; import tech.pegasys.pantheon.ethereum.core.Wei; @@ -292,7 +293,8 @@ private PantheonController createPantheonController() { private void exportRlpFormat(final PantheonController controller) throws IOException { final ProtocolContext context = controller.getProtocolContext(); - RlpBlockExporter exporter = parentCommand.rlpBlockExporterFactory.get(context); + RlpBlockExporter exporter = + parentCommand.rlpBlockExporterFactory.get(context.getBlockchain()); exporter.exportBlocks(blocksExportFile, getStartBlock(), getEndBlock()); } @@ -386,6 +388,6 @@ public interface JsonBlockImporterFactory { @FunctionalInterface public interface RlpBlockExporterFactory { - RlpBlockExporter get(ProtocolContext context); + RlpBlockExporter get(Blockchain blockchain); } } diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java index 574ec62987..f974cee15c 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java @@ -231,7 +231,7 @@ private TestPantheonCommand parseCommand( mockLogger, rlpBlockImporter, this::jsonBlockImporterFactory, - (context) -> rlpBlockExporter, + (blockchain) -> rlpBlockExporter, mockRunnerBuilder, mockControllerBuilderFactory, keyLoader, From 3208f464c8eef161329d7869269f53f3f3354a8e Mon Sep 17 00:00:00 2001 From: Meredith Baxter Date: Wed, 14 Aug 2019 11:26:54 -0400 Subject: [PATCH 5/6] Add tests for rlp block exporter --- .../pantheon/chainexport/BlockExporter.java | 18 +- .../chainexport/RlpBlockExporterTest.java | 264 +++++++++++++----- 2 files changed, 209 insertions(+), 73 deletions(-) diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java index d3f4830cd1..3281343862 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java @@ -12,6 +12,8 @@ */ package tech.pegasys.pantheon.chainexport; +import static com.google.common.base.Preconditions.checkArgument; + import tech.pegasys.pantheon.ethereum.chain.Blockchain; import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.BlockHeader; @@ -51,13 +53,9 @@ public void exportBlocks( // Get range to export final long startBlock = maybeStartBlock.orElse(BlockHeader.GENESIS_BLOCK_NUMBER); - final long endBlock = - maybeEndBlock.orElse( - maybeStartBlock - // If only start is specified, just export one block - .map(start -> start + 1L) - // Otherwise, export everything - .orElse(blockchain.getChainHeadBlockNumber() + 1L)); + final long endBlock = maybeEndBlock.orElse(blockchain.getChainHeadBlockNumber() + 1L); + checkArgument(startBlock >= 0 && endBlock >= 0, "Start and end blocks must be greater than 0."); + checkArgument(startBlock < endBlock, "Start block must be less than end block"); // Append to file if a range is specified final boolean append = maybeStartBlock.isPresent(); @@ -69,6 +67,8 @@ public void exportBlocks( endBlock, outputFile.toString(), Boolean.toString(append)); + + long blockNumber = 0L; for (long i = startBlock; i < endBlock; i++) { Optional blockHash = blockchain.getBlockHashByNumber(i); if (blockHash.isEmpty()) { @@ -77,7 +77,7 @@ public void exportBlocks( } final Block block = blockchain.getBlockByHash(blockHash.get()); - final long blockNumber = block.getHeader().getNumber(); + blockNumber = block.getHeader().getNumber(); if (blockNumber % 100 == 0) { LOG.info("Export at block {}", blockNumber); } @@ -86,7 +86,7 @@ public void exportBlocks( } outputStream.close(); - LOG.info("Export complete."); + LOG.info("Export complete at block {}", blockNumber); } protected abstract void exportBlock(final FileOutputStream outputStream, final Block block) diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java index c289c8fe79..4ed40097cf 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java @@ -12,98 +12,234 @@ */ package tech.pegasys.pantheon.chainexport; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.config.GenesisConfigFile; import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.crypto.SECP256K1; +import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; +import tech.pegasys.pantheon.ethereum.chain.Blockchain; +import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.core.BlockBody; +import tech.pegasys.pantheon.ethereum.core.BlockHeader; import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration; import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration; +import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; +import tech.pegasys.pantheon.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; +import tech.pegasys.pantheon.ethereum.util.RawBlockIterator; import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; import tech.pegasys.pantheon.testutil.BlockTestUtil; import tech.pegasys.pantheon.testutil.TestClock; +import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.Optional; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Test; import org.junit.rules.TemporaryFolder; /** Tests for {@link BlockExporter}. */ public final class RlpBlockExporterTest { @ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); - - private static PantheonController targetController; + private static Blockchain blockchain; + private static long chainHead; + private static ProtocolSchedule protocolSchedule; @BeforeClass - public static void initPantheonController() throws IOException { + public static void setupBlockchain() throws IOException { + final PantheonController controller = createController(); + final Path blocks = folder.newFile("1000.blocks").toPath(); + BlockTestUtil.write1000Blocks(blocks); + blockchain = importBlocks(controller, blocks); + chainHead = blockchain.getChainHeadBlockNumber(); + protocolSchedule = controller.getProtocolSchedule(); + } + + private static Blockchain importBlocks( + final PantheonController controller, final Path blocksFile) throws IOException { final RlpBlockImporter blockImporter = new RlpBlockImporter(); + + blockImporter.importBlockchain(blocksFile, controller); + return controller.getProtocolContext().getBlockchain(); + } + + private static PantheonController createController() throws IOException { final Path dataDir = folder.newFolder().toPath(); - final Path source = dataDir.resolve("1000.blocks"); - BlockTestUtil.write1000Blocks(source); - targetController = - new PantheonController.Builder() - .fromGenesisConfig(GenesisConfigFile.mainnet()) - .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) - .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) - .storageProvider(new InMemoryStorageProvider()) - .networkId(1) - .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) - .nodeKeys(SECP256K1.KeyPair.generate()) - .metricsSystem(new NoOpMetricsSystem()) - .privacyParameters(PrivacyParameters.DEFAULT) - .dataDirectory(dataDir) - .clock(TestClock.fixed()) - .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) - .build(); - blockImporter.importBlockchain(source, targetController); + return new PantheonController.Builder() + .fromGenesisConfig(GenesisConfigFile.mainnet()) + .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) + .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) + .storageProvider(new InMemoryStorageProvider()) + .networkId(1) + .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) + .nodeKeys(KeyPair.generate()) + .metricsSystem(new NoOpMetricsSystem()) + .privacyParameters(PrivacyParameters.DEFAULT) + .dataDirectory(dataDir) + .clock(TestClock.fixed()) + .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) + .build(); + } + + @Test + public void exportBlocks_noBounds() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + exporter.exportBlocks(outputPath, Optional.empty(), Optional.empty()); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = 0; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(chainHead + 1L); } - // protected BlockExporter createBlockExporter() { - // return new RlpBlockExporter(targetController.getProtocolContext()); - // } - - // @Test - // public void callingBlockExporterWithOnlyStartBlockShouldReturnOneBlock() throws Exception { - // - // final long startBlock = 0L; - // - // final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); - // - // final Block blockFromBlockchain = - // blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); - // - // BlockExporter.ExportResult exportResult = - // blockExporter.exportBlocks(targetController, startBlock, null); - // - // assertThat(exportResult.blocks).contains(blockFromBlockchain); - // } - // - // @Test - // public void callingBlockExporterWithStartBlockAndBlockShouldReturnSeveralBlocks() - // throws Exception { - // - // final long startBlock = 0L; - // final long endBlock = 1L; - // - // final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); - // - // final Block blockFromBlockchain = - // blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); - // - // final Block secondBlockFromBlockchain = - // blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get()); - // - // BlockExporter.ExportResult exportResult = - // blockExporter.exportBlocks(targetController, startBlock, endBlock); - // - // assertThat(exportResult.blocks) - // .contains(blockFromBlockchain) - // .contains(secondBlockFromBlockchain); - // } + @Test + public void exportBlocks_withLowerBound() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long lowerBound = 990; + exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.empty()); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = lowerBound; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(chainHead + 1L); + } + + @Test + public void exportBlocks_withUpperBound() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long upperBound = 10; + exporter.exportBlocks(outputPath, Optional.empty(), Optional.of(upperBound)); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = 0; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(upperBound); + } + + @Test + public void exportBlocks_withUpperAndLowerBounds() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long lowerBound = 5; + final long upperBound = 10; + exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.of(upperBound)); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = lowerBound; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(upperBound); + } + + @Test + public void exportBlocks_withRangeBeyondChainHead() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long lowerBound = chainHead - 10; + final long upperBound = chainHead + 10; + exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.of(upperBound)); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = lowerBound; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(chainHead + 1L); + } + + @Test + public void exportBlocks_negativeStartNumber() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.of(-1L), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("greater than 0"); + } + + @Test + public void exportBlocks_negativeEndNumber() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.empty(), Optional.of(-1L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("greater than 0"); + } + + @Test + public void exportBlocks_outOfOrderBounds() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.of(10L), Optional.of(2L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Start block must be less than end block"); + } + + private RawBlockIterator getBlockIterator(final Path blocks) throws IOException { + return new RawBlockIterator( + blocks, + rlp -> + BlockHeader.readFrom(rlp, ScheduleBasedBlockHeaderFunctions.create(protocolSchedule))); + } + + private Block getBlock(final Blockchain blockchain, final long blockNumber) { + final BlockHeader header = blockchain.getBlockHeader(blockNumber).get(); + final BlockBody body = blockchain.getBlockBody(header.getHash()).get(); + return new Block(header, body); + } } From 38c4361c48b8be136e620d6b6047682a3dd0acbc Mon Sep 17 00:00:00 2001 From: Meredith Baxter Date: Wed, 14 Aug 2019 12:31:30 -0400 Subject: [PATCH 6/6] Remove redundant check --- .../pantheon/cli/subcommands/blocks/BlocksSubCommand.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java index 648829c0dc..685214381f 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java @@ -345,9 +345,7 @@ private void checkCommand( parentCommand.parentCommand.dataDir().toAbsolutePath().toString(), PantheonController.DATABASE_PATH); File databaseDirectory = new File(databasePath.toString()); - if (!databaseDirectory.exists() - || !databaseDirectory.isDirectory() - || databaseDirectory.list().length == 0) { + if (!databaseDirectory.isDirectory() || databaseDirectory.list().length == 0) { // Empty data directory, nothing to export throw new CommandLine.ParameterException( spec.commandLine(),