[PIE-1810] Update export subcommand to export blocks in rlp format

pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java

@@ -14,12 +14,12 @@
 
 import static org.apache.logging.log4j.LogManager.getLogger;
 
-import tech.pegasys.pantheon.chainimport.ChainImporter;
+import tech.pegasys.pantheon.chainexport.RlpBlockExporter;
+import tech.pegasys.pantheon.chainimport.JsonBlockImporter;
+import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
 import tech.pegasys.pantheon.cli.PantheonCommand;
 import tech.pegasys.pantheon.controller.PantheonController;
 import tech.pegasys.pantheon.services.PantheonPluginContextImpl;
-import tech.pegasys.pantheon.util.BlockExporter;
-import tech.pegasys.pantheon.util.BlockImporter;
 
 import org.apache.logging.log4j.Logger;
 import picocli.CommandLine.RunLast;
@@ -36,9 +36,9 @@ public static void main(final String... args) {
     final PantheonCommand pantheonCommand =
         new PantheonCommand(
             logger,
-            new BlockImporter(),
-            new BlockExporter(),
-            ChainImporter::new,
+            new RlpBlockImporter(),
+            JsonBlockImporter::new,
+            RlpBlockExporter::new,
             new RunnerBuilder(),
             new PantheonController.Builder(),
             new PantheonPluginContextImpl(),

pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.chainexport;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import tech.pegasys.pantheon.ethereum.chain.Blockchain;
+import tech.pegasys.pantheon.ethereum.core.Block;
+import tech.pegasys.pantheon.ethereum.core.BlockHeader;
+import tech.pegasys.pantheon.ethereum.core.Hash;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/** Pantheon Block Export Util. */
+public abstract class BlockExporter {
+  private static final Logger LOG = LogManager.getLogger();
+  private final Blockchain blockchain;
+
+  protected BlockExporter(final Blockchain blockchain) {
+    this.blockchain = blockchain;
+  }
+
+  /**
+   * Export blocks that are stored in Pantheon's block storage.
+   *
+   * @param outputFile the path at which to save the exported block data
+   * @param maybeStartBlock the starting index of the block list to export (inclusive)
+   * @param maybeEndBlock the ending index of the block list to export (exclusive), if not specified
+   *     a single block will be export
+   * @throws IOException if an I/O error occurs while writing data to disk
+   */
+  public void exportBlocks(
+      final File outputFile,
+      final Optional<Long> maybeStartBlock,
+      final Optional<Long> maybeEndBlock)
+      throws IOException {
+
+    // Get range to export
+    final long startBlock = maybeStartBlock.orElse(BlockHeader.GENESIS_BLOCK_NUMBER);
+    final long endBlock = maybeEndBlock.orElse(blockchain.getChainHeadBlockNumber() + 1L);
+    checkArgument(startBlock >= 0 && endBlock >= 0, "Start and end blocks must be greater than 0.");
+    checkArgument(startBlock < endBlock, "Start block must be less than end block");
+
+    // Append to file if a range is specified
+    final boolean append = maybeStartBlock.isPresent();
+    FileOutputStream outputStream = new FileOutputStream(outputFile, append);
+
+    LOG.info(
+        "Exporting blocks [{},{}) to file {} (appending: {})",
+        startBlock,
+        endBlock,
+        outputFile.toString(),
+        Boolean.toString(append));
+
+    long blockNumber = 0L;
+    for (long i = startBlock; i < endBlock; i++) {
+      Optional<Hash> blockHash = blockchain.getBlockHashByNumber(i);
+      if (blockHash.isEmpty()) {
+        LOG.warn("Unable to export blocks [{} - {}).  Blocks not found.", i, endBlock);
+        break;
+      }
+
+      final Block block = blockchain.getBlockByHash(blockHash.get());
+      blockNumber = block.getHeader().getNumber();
+      if (blockNumber % 100 == 0) {
+        LOG.info("Export at block {}", blockNumber);
+      }
+
+      exportBlock(outputStream, block);
+    }
+
+    outputStream.close();
+    LOG.info("Export complete at block {}", blockNumber);
+  }
+
+  protected abstract void exportBlock(final FileOutputStream outputStream, final Block block)
+      throws IOException;
+}

pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.chainexport;
+
+import tech.pegasys.pantheon.ethereum.chain.Blockchain;
+import tech.pegasys.pantheon.ethereum.core.Block;
+import tech.pegasys.pantheon.ethereum.rlp.RLP;
+import tech.pegasys.pantheon.util.bytes.BytesValue;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+public class RlpBlockExporter extends BlockExporter {
+
+  public RlpBlockExporter(final Blockchain blockchain) {
+    super(blockchain);
+  }
+
+  @Override
+  protected void exportBlock(final FileOutputStream outputStream, final Block block)
+      throws IOException {
+    final BytesValue rlp = RLP.encode(block::writeTo);
+    outputStream.write(rlp.getArrayUnsafe());
+  }
+}

pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java → pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java

@@ -12,6 +12,8 @@
  */
 package tech.pegasys.pantheon.chainimport;
 
+import tech.pegasys.pantheon.chainimport.internal.BlockData;
+import tech.pegasys.pantheon.chainimport.internal.ChainData;
 import tech.pegasys.pantheon.config.GenesisConfigOptions;
 import tech.pegasys.pantheon.controller.PantheonController;
 import tech.pegasys.pantheon.ethereum.blockcreation.MiningCoordinator;
@@ -39,13 +41,18 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-public class ChainImporter<C> {
+/**
+ * Tool for importing blocks with transactions from human-readable json.
+ *
+ * @param <C> The consensus algorithm context
+ */
+public class JsonBlockImporter<C> {
   private static final Logger LOG = LogManager.getLogger();
 
   private final ObjectMapper mapper;
   private final PantheonController<C> controller;
 
-  public ChainImporter(final PantheonController<C> controller) {
+  public JsonBlockImporter(final PantheonController<C> controller) {
     this.controller = controller;
     mapper = new ObjectMapper();
     // Jdk8Module allows us to easily parse {@code Optional} values from json

pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java → pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java

@@ -10,7 +10,7 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package tech.pegasys.pantheon.util;
+package tech.pegasys.pantheon.chainimport;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.logging.log4j.LogManager.getLogger;
@@ -41,8 +41,8 @@
 import com.google.common.base.MoreObjects;
 import org.apache.logging.log4j.Logger;
 
-/** Pantheon Block Import Util. */
-public class BlockImporter {
+/** Tool for importing rlp-encoded block data from files. */
+public class RlpBlockImporter {
   private static final Logger LOG = getLogger();
 
   private final Semaphore blockBacklog = new Semaphore(2);
@@ -60,7 +60,7 @@ public class BlockImporter {
    * @return the import result
    * @throws IOException On Failure
    */
-  public <C> BlockImporter.ImportResult importBlockchain(
+  public <C> RlpBlockImporter.ImportResult importBlockchain(
       final Path blocks, final PantheonController<C> pantheonController) throws IOException {
     final ProtocolSchedule<C> protocolSchedule = pantheonController.getProtocolSchedule();
     final ProtocolContext<C> context = pantheonController.getProtocolContext();
@@ -125,7 +125,8 @@ public <C> BlockImporter.ImportResult importBlockchain(
       if (previousBlockFuture != null) {
         previousBlockFuture.join();
       }
-      return new BlockImporter.ImportResult(blockchain.getChainHead().getTotalDifficulty(), count);
+      return new RlpBlockImporter.ImportResult(
+          blockchain.getChainHead().getTotalDifficulty(), count);
     } finally {
       validationExecutor.shutdownNow();
       try {

pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java → pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java

@@ -10,9 +10,9 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package tech.pegasys.pantheon.chainimport;
+package tech.pegasys.pantheon.chainimport.internal;
 
-import tech.pegasys.pantheon.chainimport.TransactionData.NonceProvider;
+import tech.pegasys.pantheon.chainimport.internal.TransactionData.NonceProvider;
 import tech.pegasys.pantheon.ethereum.core.Account;
 import tech.pegasys.pantheon.ethereum.core.Address;
 import tech.pegasys.pantheon.ethereum.core.Hash;

pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java → pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java

@@ -10,7 +10,7 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package tech.pegasys.pantheon.chainimport;
+package tech.pegasys.pantheon.chainimport.internal;
 
 import java.util.List;
 

pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java → pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java

@@ -10,7 +10,7 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package tech.pegasys.pantheon.chainimport;
+package tech.pegasys.pantheon.chainimport.internal;
 
 import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair;
 import tech.pegasys.pantheon.crypto.SECP256K1.PrivateKey;

pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java

@@ -31,6 +31,7 @@
 import tech.pegasys.pantheon.PantheonInfo;
 import tech.pegasys.pantheon.Runner;
 import tech.pegasys.pantheon.RunnerBuilder;
+import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
 import tech.pegasys.pantheon.cli.config.EthNetworkConfig;
 import tech.pegasys.pantheon.cli.config.NetworkName;
 import tech.pegasys.pantheon.cli.converter.MetricCategoryConverter;
@@ -51,7 +52,8 @@
 import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader;
 import tech.pegasys.pantheon.cli.subcommands.RetestethSubCommand;
 import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand;
-import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory;
+import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory;
+import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory;
 import tech.pegasys.pantheon.cli.subcommands.operator.OperatorSubCommand;
 import tech.pegasys.pantheon.cli.subcommands.rlp.RLPSubCommand;
 import tech.pegasys.pantheon.cli.util.ConfigOptionSearchAndRunHandler;
@@ -93,8 +95,6 @@
 import tech.pegasys.pantheon.services.PantheonPluginContextImpl;
 import tech.pegasys.pantheon.services.PicoCLIOptionsImpl;
 import tech.pegasys.pantheon.services.kvstore.RocksDbConfiguration;
-import tech.pegasys.pantheon.util.BlockExporter;
-import tech.pegasys.pantheon.util.BlockImporter;
 import tech.pegasys.pantheon.util.PermissioningConfigurationValidator;
 import tech.pegasys.pantheon.util.bytes.BytesValue;
 import tech.pegasys.pantheon.util.number.Fraction;
@@ -153,12 +153,12 @@
 public class PantheonCommand implements DefaultCommandValues, Runnable {
 
   private final Logger logger;
-  private final ChainImporterFactory chainImporterFactory;
 
   private CommandLine commandLine;
 
-  private final BlockImporter blockImporter;
-  private final BlockExporter blockExporter;
+  private final RlpBlockImporter rlpBlockImporter;
+  private final JsonBlockImporterFactory jsonBlockImporterFactory;
+  private final RlpBlockExporterFactory rlpBlockExporterFactory;
 
   final NetworkingOptions networkingOptions = NetworkingOptions.create();
   final SynchronizerOptions synchronizerOptions = SynchronizerOptions.create();
@@ -633,17 +633,17 @@ void setBannedNodeIds(final List<String> values) {
 
   public PantheonCommand(
       final Logger logger,
-      final BlockImporter blockImporter,
-      final BlockExporter blockExporter,
-      final ChainImporterFactory chainImporterFactory,
+      final RlpBlockImporter rlpBlockImporter,
+      final JsonBlockImporterFactory jsonBlockImporterFactory,
+      final RlpBlockExporterFactory rlpBlockExporterFactory,
       final RunnerBuilder runnerBuilder,
       final PantheonController.Builder controllerBuilderFactory,
       final PantheonPluginContextImpl pantheonPluginContext,
       final Map<String, String> environment) {
     this.logger = logger;
-    this.blockImporter = blockImporter;
-    this.blockExporter = blockExporter;
-    this.chainImporterFactory = chainImporterFactory;
+    this.rlpBlockImporter = rlpBlockImporter;
+    this.rlpBlockExporterFactory = rlpBlockExporterFactory;
+    this.jsonBlockImporterFactory = jsonBlockImporterFactory;
     this.runnerBuilder = runnerBuilder;
     this.controllerBuilderFactory = controllerBuilderFactory;
     this.pantheonPluginContext = pantheonPluginContext;
@@ -690,7 +690,10 @@ private PantheonCommand addSubCommands(
     commandLine.addSubcommand(
         BlocksSubCommand.COMMAND_NAME,
         new BlocksSubCommand(
-            blockImporter, blockExporter, chainImporterFactory, resultHandler.out()));
+            rlpBlockImporter,
+            jsonBlockImporterFactory,
+            rlpBlockExporterFactory,
+            resultHandler.out()));
     commandLine.addSubcommand(
         PublicKeySubCommand.COMMAND_NAME,
         new PublicKeySubCommand(resultHandler.out(), getKeyLoader()));
@@ -1363,7 +1366,7 @@ private File genesisFile() {
     }
   }
 
-  private Path dataDir() {
+  public Path dataDir() {
     if (isFullInstantiation()) {
       return standaloneCommands.dataPath.toAbsolutePath();
     } else if (isDocker) {

pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.cli.subcommands.blocks;
+
+public enum BlockExportFormat {
+  RLP
+}

pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java → pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java

@@ -12,7 +12,7 @@
  */
 package tech.pegasys.pantheon.cli.subcommands.blocks;
 
-public enum BlockFormat {
+public enum BlockImportFormat {
   RLP,
   JSON
 }