[PIE-1602] Implement operator tool. (blockchain network configuration for permisionned networks)

gradle/versions.gradle

@@ -26,7 +26,7 @@ dependencyManagement {
     dependency 'com.google.errorprone:error_prone_test_helpers:2.3.3'
 
     dependency 'com.graphql-java:graphql-java:12.0'
-    
+
     dependency 'com.google.guava:guava:27.1-jre'
 
     dependency 'com.squareup.okhttp3:okhttp:3.14.2'
@@ -78,12 +78,14 @@ dependencyManagement {
     dependency 'org.rocksdb:rocksdbjni:5.15.10'
 
     dependency 'org.springframework.security:spring-security-crypto:5.1.5.RELEASE'
-    
+
     dependency 'org.web3j:abi:4.3.0'
     dependency 'org.web3j:core:4.3.0'
     dependency 'org.web3j:crypto:4.3.0'
 
     dependency 'org.xerial.snappy:snappy-java:1.1.7.3'
+    dependency 'com.jayway.jsonpath:json-path:2.4.0'
+
 
     dependency "tech.pegasys.pantheon:plugin-api:${rootProject.version}"
   }

pantheon/build.gradle

@@ -54,6 +54,9 @@ dependencies {
   implementation 'org.springframework.security:spring-security-crypto'
   implementation 'tech.pegasys.pantheon:plugin-api'
 
+  implementation 'com.google.code.gson:gson:2.8.5'
+
+
   runtime 'org.apache.logging.log4j:log4j-core'
   runtime 'org.apache.logging.log4j:log4j-slf4j-impl'
 
@@ -67,6 +70,7 @@ dependencies {
   testImplementation 'org.awaitility:awaitility'
   testImplementation 'org.mockito:mockito-core'
   testImplementation 'org.apache.commons:commons-text'
+  testImplementation 'com.jayway.jsonpath:json-path'
 }
 
 task writeInfoFile(type: ProjectPropertiesFile) {

pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java

@@ -36,6 +36,7 @@
 import tech.pegasys.pantheon.cli.custom.CorsAllowedOriginsProperty;
 import tech.pegasys.pantheon.cli.custom.JsonRPCWhitelistHostsProperty;
 import tech.pegasys.pantheon.cli.custom.RpcAuthFileValidator;
+import tech.pegasys.pantheon.cli.operator.OperatorSubCommand;
 import tech.pegasys.pantheon.cli.rlp.RLPSubCommand;
 import tech.pegasys.pantheon.config.GenesisConfigFile;
 import tech.pegasys.pantheon.controller.KeyPairUtil;
@@ -627,6 +628,8 @@ public void parse(
         PasswordSubCommand.COMMAND_NAME, new PasswordSubCommand(resultHandler.out()));
     commandLine.addSubcommand(
         RLPSubCommand.COMMAND_NAME, new RLPSubCommand(resultHandler.out(), in));
+    commandLine.addSubcommand(
+        OperatorSubCommand.COMMAND_NAME, new OperatorSubCommand(resultHandler.out()));
 
     commandLine.registerConverter(Address.class, Address::fromHexStringStrict);
     commandLine.registerConverter(BytesValue.class, BytesValue::fromHexString);

pantheon/src/main/java/tech/pegasys/pantheon/cli/operator/OperatorSubCommand.java

@@ -0,0 +1,300 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.cli.operator;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.deleteIfExists;
+import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP;
+import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_PATH_FORMAT_HELP;
+import static tech.pegasys.pantheon.cli.operator.OperatorSubCommand.COMMAND_NAME;
+
+import tech.pegasys.pantheon.cli.PantheonCommand;
+import tech.pegasys.pantheon.cli.rlp.RLPType;
+import tech.pegasys.pantheon.crypto.SECP256K1;
+import tech.pegasys.pantheon.ethereum.core.Address;
+import tech.pegasys.pantheon.ethereum.core.Util;
+import tech.pegasys.pantheon.util.bytes.BytesValue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import com.google.common.io.Resources;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import io.vertx.core.json.JsonArray;
+import io.vertx.core.json.JsonObject;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.ParentCommand;
+import picocli.CommandLine.Spec;
+
+/** Operator related sub-command */
+@Command(
+    name = COMMAND_NAME,
+    description = "This command provides operator related actions.",
+    mixinStandardHelpOptions = true,
+    subcommands = {OperatorSubCommand.GenerateNetworkConfigSubCommand.class})
+public class OperatorSubCommand implements Runnable {
+  private static final Logger LOG = LogManager.getLogger();
+
+  public static final String COMMAND_NAME = "operator";
+
+  @SuppressWarnings("unused")
+  @ParentCommand
+  private PantheonCommand parentCommand; // Picocli injects reference to parent command
+
+  @SuppressWarnings("unused")
+  @Spec
+  private CommandSpec spec; // Picocli injects reference to command spec
+
+  private final PrintStream out;
+
+  public OperatorSubCommand(final PrintStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void run() {
+    spec.commandLine().usage(out);
+  }
+
+  @Command(
+      name = "generate-blockchain-config",
+      description = "This command generates blockchain network configuration files.",
+      mixinStandardHelpOptions = true)
+  static class GenerateNetworkConfigSubCommand implements Runnable {
+    @Option(
+        required = true,
+        names = "--config-file",
+        paramLabel = MANDATORY_FILE_FORMAT_HELP,
+        description = "Configuration file.",
+        arity = "1..1")
+    private File configurationFile = null;
+
+    @Option(
+        required = true,
+        names = "--to",
+        paramLabel = MANDATORY_FILE_FORMAT_HELP,
+        description = "Directory to write output files to.",
+        arity = "1..1")
+    private File outputDirectory = null;
+
+    @Option(
+        names = "--genesis-file-name",
+        paramLabel = MANDATORY_PATH_FORMAT_HELP,
+        description = "Name of the genesis file.",
+        arity = "1..1")
+    private String genesisFileName = "genesis.json";
+
+    @Option(
+        names = "--private-key-file-name",
+        paramLabel = MANDATORY_PATH_FORMAT_HELP,
+        description = "Name of the private key file.",
+        arity = "1..1")
+    private String privateKeyFileName = "key.priv";
+
+    @Option(
+        names = "--public-key-file-name",
+        paramLabel = MANDATORY_PATH_FORMAT_HELP,
+        description = "Name of the public key file.",
+        arity = "1..1")
+    private String publicKeyFileName = "key.pub";
+
+    @SuppressWarnings("unused")
+    @ParentCommand
+    private OperatorSubCommand parentCommand; // Picocli injects reference to parent command
+
+    private JsonObject operatorConfig;
+    private JsonObject genesisConfig;
+    private JsonObject blockchainConfig;
+    private JsonObject nodesConfig;
+    private boolean generateNodesKeys;
+    private List<String> addressesForGenesisExtraData = new ArrayList<>();
+    private Path keysDirectory;
+
+    @Override
+    public void run() {
+      checkNotNull(parentCommand);
+      checkNotNull(parentCommand.parentCommand);
+      generateBlockchainConfig();
+    }
+
+    /** Generates output directory with all required configuration files. */
+    private void generateBlockchainConfig() {
+      try {
+        createOutputDirectory();
+        parseConfig();
+        if (generateNodesKeys) {
+          generateNodesKeys();
+        } else {
+          importPublicKeysFromConfig();
+        }
+        processExtraData();
+        writeGenesisFile(outputDirectory, genesisFileName, genesisConfig);
+      } catch (IOException e) {
+        LOG.error("An error occurred while trying to generate network configuration.", e);
+      }
+    }
+
+    /** Imports public keys from input configuration. */
+    private void importPublicKeysFromConfig() {
+      LOG.info("Importing public keys from configuration.");
+      JsonArray keys = nodesConfig.getJsonArray("keys");
+      keys.stream().forEach(this::importPublicKey);
+    }
+
+    /**
+     * Imports a single public key.
+     *
+     * @param publicKeyObject The public key.
+     */
+    private void importPublicKey(final Object publicKeyObject) {
+      try {
+        final SECP256K1.PublicKey publicKey =
+            SECP256K1.PublicKey.create(BytesValue.fromHexString((String) publicKeyObject));
+        writeKeypair(publicKey, null);
+        LOG.info("Public key imported from configuration.({})", publicKey.toString());
+      } catch (IOException e) {
+        LOG.error("An error occurred while trying to import node public key.", e);
+      }
+    }
+
+    /** Generates nodes keypairs. */
+    private void generateNodesKeys() {
+      final int nodesCount = nodesConfig.getInteger("count");
+      LOG.info("Generating {} nodes keys.", nodesCount);
+      IntStream.range(0, nodesCount).forEach(this::generateNodeKeypair);
+    }
+
+    /**
+     * Generate a keypair for a node.
+     *
+     * @param node The number of the node.
+     */
+    private void generateNodeKeypair(final int node) {
+      try {
+        LOG.info("Generating keypair for node {}.", node);
+        final SECP256K1.KeyPair keyPair = SECP256K1.KeyPair.generate();
+        writeKeypair(keyPair.getPublicKey(), keyPair.getPrivateKey());
+
+      } catch (IOException e) {
+        LOG.error("An error occurred while trying to generate node keypair.", e);
+      }
+    }
+
+    /**
+     * Writes public and private keys in separate files. Both are written in the same directory
+     * named with the address derived from the public key.
+     *
+     * @param publicKey The public key.
+     * @param privateKey The private key. No file is created if privateKey is NULL.
+     * @throws IOException
+     */
+    private void writeKeypair(
+        final SECP256K1.PublicKey publicKey, final SECP256K1.PrivateKey privateKey)
+        throws IOException {
+      final Address nodeAddress = Util.publicKeyToAddress(publicKey);
+      addressesForGenesisExtraData.add(nodeAddress.toUnprefixedString());
+      final Path nodeDirectoryPath = keysDirectory.resolve(nodeAddress.toString());
+      Files.createDirectory(nodeDirectoryPath);
+      createFileAndWrite(nodeDirectoryPath, publicKeyFileName, publicKey.toString());
+      if (privateKey != null) {
+        createFileAndWrite(nodeDirectoryPath, privateKeyFileName, privateKey.toString());
+      }
+    }
+
+    /**
+     * Computes RLP encoded exta data from pre filled list of addresses.
+     *
+     * @throws IOException
+     */
+    private void processExtraData() throws IOException {
+      LOG.info("Generating genesis extra data.");
+      final Gson gson = new GsonBuilder().setPrettyPrinting().create();
+      final String jsonAddresses = gson.toJson(addressesForGenesisExtraData);
+      final String extraData =
+          RLPType.IBFT_EXTRA_DATA.getAdapter().encode(jsonAddresses).toString();
+      genesisConfig.put("extraData", extraData);
+    }
+
+    private void createFileAndWrite(
+        final Path directory, final String fileName, final String content) throws IOException {
+      final Path filePath = directory.resolve(fileName);
+      Files.createFile(filePath);
+      Files.write(filePath, content.getBytes(UTF_8));
+    }
+
+    /**
+     * Parses the root configuration file and related sub elements.
+     *
+     * @throws IOException
+     */
+    private void parseConfig() throws IOException {
+      final String configString =
+          Resources.toString(configurationFile.toPath().toUri().toURL(), UTF_8);
+      operatorConfig = new JsonObject(configString);
+      genesisConfig = operatorConfig.getJsonObject("genesis");
+      blockchainConfig = operatorConfig.getJsonObject("blockchain");
+      nodesConfig = blockchainConfig.getJsonObject("nodes");
+      generateNodesKeys = nodesConfig.getBoolean("generate", false);
+    }
+
+    /**
+     * Creates the output directory if not exists.
+     *
+     * @throws IOException
+     */
+    private void createOutputDirectory() throws IOException {
+      final Path outputDirectoryPath = outputDirectory.toPath();
+
+      if (outputDirectory.exists() && outputDirectory.isDirectory()) {
+        try (Stream<Path> stream = Files.walk(outputDirectoryPath)) {
+          stream.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+          deleteIfExists(outputDirectoryPath);
+        }
+      }
+      Files.createDirectory(outputDirectoryPath);
+      keysDirectory = outputDirectoryPath.resolve("keys");
+      Files.createDirectory(keysDirectory);
+    }
+
+    /**
+     * Write the content of the genesis to the output file.
+     *
+     * @param directory The directory to write the file to.
+     * @param fileName The name of the output file.
+     * @param genesis The genesis content.
+     * @throws IOException
+     */
+    private void writeGenesisFile(
+        final File directory, final String fileName, final JsonObject genesis) throws IOException {
+      final Path genesisPath = directory.toPath().resolve(fileName);
+      deleteIfExists(genesisPath);
+      LOG.info("Writing genesis file.");
+      Files.write(genesisPath, genesis.encodePrettily().getBytes(UTF_8));
+    }
+  }
+}

pantheon/src/main/java/tech/pegasys/pantheon/cli/rlp/JSONToRLP.java

@@ -17,7 +17,7 @@
 import java.io.IOException;
 
 /** Behaviour of objects that can be encoded from JSON to RLP */
-interface JSONToRLP {
+public interface JSONToRLP {
 
   /**
    * Encodes the object into an RLP value.