functionality to rebuild the blockchain based on the current stored b…

…locks

modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java

@@ -538,9 +538,10 @@ public synchronized void compactState() {
 
     // TEMPORARY: here to support the ConsensusTest
     public Pair<ImportResult, AionBlockSummary> tryToConnectAndFetchSummary(
-            AionBlock block, long currTimeSeconds) {
+            AionBlock block, long currTimeSeconds, boolean doExistCheck) {
         // Check block exists before processing more rules
-        if (getBlockStore().getMaxNumber() >= block.getNumber()
+        if (doExistCheck // skipped when re-importing
+                && getBlockStore().getMaxNumber() >= block.getNumber()
                 && getBlockStore().isBlockExist(block.getHash())) {
 
             if (LOG.isDebugEnabled()) {
@@ -646,7 +647,7 @@ && getBlockStore().isBlockExist(block.getHash())) {
      * can feed timestamps manually
      */
     ImportResult tryToConnectInternal(final AionBlock block, long currTimeSeconds) {
-        return tryToConnectAndFetchSummary(block, currTimeSeconds).getLeft();
+        return tryToConnectAndFetchSummary(block, currTimeSeconds, true).getLeft();
     }
 
     /**

modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java

@@ -377,7 +377,7 @@ public synchronized ImportResult tryToConnect(final AionBlock block) {
     // TEMPORARY: here to support the ConsensusTest
     public synchronized Pair<ImportResult, AionBlockSummary> tryToConnectAndFetchSummary(
             AionBlock block) {
-        return tryToConnectAndFetchSummary(block, System.currentTimeMillis() / 1000);
+        return tryToConnectAndFetchSummary(block, System.currentTimeMillis() / 1000, true);
     }
 
     /**

modAionImpl/src/org/aion/zero/impl/cli/Arguments.java

@@ -151,6 +151,14 @@ public class Arguments {
             description = "if using leveldb, it triggers its database compaction processes")
     private boolean dbCompact;
 
+    @Option(
+            names = {"--re-import"},
+            arity = "0..1",
+            paramLabel = "<start_height>",
+            description =
+                    "drops all databases except for block and index when not given a parameter or starting from 0 and re-imports all blocks")
+    private String reImport = null;
+
     /** Compacts the account options into specific commands. */
     public static String[] preProcess(String[] arguments) {
         List<String> list = new ArrayList<>();
@@ -258,4 +266,8 @@ public String getDumpStateCount() {
     public boolean isDbCompact() {
         return dbCompact;
     }
+
+    public String isReImport() {
+        return reImport;
+    }
 }

modAionImpl/src/org/aion/zero/impl/cli/Cli.java

@@ -79,7 +79,8 @@ enum TaskPriority {
         DUMP_STATE_SIZE,
         DUMP_STATE,
         DUMP_BLOCKS,
-        DB_COMPACT
+        DB_COMPACT,
+        RE_IMPORT
     }
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
@@ -515,11 +516,33 @@ public ReturnType call(final String[] args, Cfg cfg) {
                 return EXIT;
             }
 
+            if (options.isReImport() != null) {
+                long height = 0L;
+                String parameter = options.isReImport();
+
+                if (parameter.isEmpty()) {
+                    RecoveryUtils.reimportMainChain(height);
+                    return EXIT;
+                } else {
+                    try {
+                        height = Long.parseLong(parameter);
+                    } catch (NumberFormatException e) {
+                        System.out.println(
+                                "The given argument «"
+                                        + parameter
+                                        + "» cannot be converted to a number.");
+                        return ERROR;
+                    }
+                    RecoveryUtils.reimportMainChain(height);
+                    return EXIT;
+                }
+            }
+
             // if no return happened earlier, run the kernel
             return RUN;
         } catch (Exception e) {
             // TODO: should be moved to individual procedures
-            System.out.println("");
+            System.out.println();
             e.printStackTrace();
             return ERROR;
         }
@@ -912,6 +935,9 @@ TaskPriority getBreakingTaskPriority(Arguments options) {
         if (options.isDbCompact()) {
             return TaskPriority.DB_COMPACT;
         }
+        if (options.isReImport() != null) {
+            return TaskPriority.RE_IMPORT;
+        }
         return TaskPriority.NONE;
     }
 
@@ -991,6 +1017,10 @@ Set<String> getSkippedTasks(Arguments options, TaskPriority breakingTaskPriority
         if (breakingTaskPriority.compareTo(TaskPriority.DB_COMPACT) < 0 && options.isDbCompact()) {
             skippedTasks.add("--db-compact");
         }
+        if (breakingTaskPriority.compareTo(TaskPriority.RE_IMPORT) < 0
+                && options.isReImport() != null) {
+            skippedTasks.add("--re-import");
+        }
         return skippedTasks;
     }
 

modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java

@@ -780,6 +780,21 @@ public String toString() {
                 + '}';
     }
 
+    /**
+     * Calls {@link IByteArrayKeyValueDatabase#drop()} on all the current databases except for the
+     * ones given in the list by name.
+     *
+     * @param names the names of the databases that should not be dropped
+     */
+    public void dropDatabasesExcept(List<String> names) {
+        for (IByteArrayKeyValueDatabase db : databaseGroup) {
+            if (!names.contains(db.getName().get())) {
+                LOG.warn("Dropping database " + db.toString());
+                db.drop();
+            }
+        }
+    }
+
     @Override
     public void compact() {
         rwLock.writeLock().lock();

modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java

@@ -2,18 +2,30 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import org.aion.base.type.IBlock;
 import org.aion.log.AionLoggerFactory;
 import org.aion.mcf.config.CfgDb;
+import org.aion.mcf.core.ImportResult;
 import org.aion.mcf.db.IBlockStoreBase;
 import org.aion.zero.impl.AionBlockchainImpl;
 import org.aion.zero.impl.AionGenesis;
 import org.aion.zero.impl.AionHubUtils;
 import org.aion.zero.impl.config.CfgAion;
 import org.aion.zero.impl.core.IAionBlockchain;
 import org.aion.zero.impl.types.AionBlock;
-
+import org.aion.zero.impl.types.AionBlockSummary;
+import org.apache.commons.lang3.tuple.Pair;
+
+/**
+ * Methods used by CLI calls for debugging the local blockchain data.
+ *
+ * @author Alexandra Roatis
+ * @implNote This class started off with helper functions for data recovery at runtime. It evolved
+ *     into a diverse set of CLI calls for displaying or manipulating the data. It would benefit
+ *     from refactoring to separate the different use cases.
+ */
 public class RecoveryUtils {
 
     public enum Status {
@@ -325,7 +337,7 @@ public static void pruneOrRecoverState(String pruning_type) {
         AionLoggerFactory.init(cfgLog);
 
         AionBlockchainImpl chain = AionBlockchainImpl.inst();
-        AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository();
+        AionRepositoryImpl repo = chain.getRepository();
         AionBlockStore store = repo.getBlockStore();
 
         // dropping old state database
@@ -364,4 +376,142 @@ public static void pruneOrRecoverState(String pruning_type) {
         repo.close();
         System.out.println("Reorganizing the state storage COMPLETE.");
     }
+
+    /**
+     * Alternative to performing a full sync when the database already contains the <b>blocks</b>
+     * and <b>index</b> databases. It will rebuild the entire blockchain structure other than these
+     * two databases verifying consensus properties. It only re-imports main chain blocks, i.e. does
+     * not perform the checks for side chains.
+     *
+     * <p>The minimum start height is 0, i.e. the genesis block. Specifying a height can be useful
+     * in performing the operation is sessions.
+     *
+     * @param startHeight the height from which to start importing the blocks
+     * @implNote The assumption is that the stored blocks are correct, but the code may interpret
+     *     them differently.
+     */
+    public static void reimportMainChain(long startHeight) {
+        // ensure mining is disabled
+        CfgAion cfg = CfgAion.inst();
+        cfg.dbFromXML();
+        cfg.getConsensus().setMining(false);
+        cfg.getDb().setHeapCacheEnabled(false);
+
+        System.out.println("\nRe-importing stored blocks INITIATED...\n");
+
+        Map<String, String> cfgLog = new HashMap<>();
+        cfgLog.put("GEN", "INFO");
+        cfgLog.put("DB", "WARN");
+        cfgLog.put("CONS", "ERROR");
+
+        AionLoggerFactory.init(cfgLog);
+
+        AionBlockchainImpl chain = AionBlockchainImpl.inst();
+        AionRepositoryImpl repo = chain.getRepository();
+        AionBlockStore store = repo.getBlockStore();
+
+        // determine the parameters of the rebuild
+        AionBlock block = store.getBestBlock();
+        AionBlock startBlock;
+        long currentBlock;
+        if (startHeight <= block.getNumber()) {
+            System.out.println(
+                    "\nRe-importing the main chain from block #"
+                            + startHeight
+                            + " to block #"
+                            + block.getNumber()
+                            + ". This may take a while.\n"
+                            + "The time estimates are optimistic based on current progress.\n"
+                            + "It is expected that later blocks take a longer time to import due to the increasing size of the database.\n");
+
+            if (startHeight == 0L) {
+                // dropping databases that can be inferred when starting from genesis
+                List<String> keep = List.of("block", "index");
+                repo.dropDatabasesExcept(keep);
+
+                // recover genesis
+                AionGenesis genesis = cfg.getGenesis();
+                AionHubUtils.buildGenesis(genesis, repo);
+                System.out.println("\nFinished rebuilding genesis block.");
+                startBlock = genesis;
+                currentBlock = 1L;
+            } else {
+                startBlock = store.getChainBlockByNumber(startHeight - 1);
+                currentBlock = startHeight;
+            }
+
+            chain.setBestBlock(startBlock);
+
+            long topBlockNumber = block.getNumber();
+            long stepSize = 10_000L;
+
+            Pair<ImportResult, AionBlockSummary> result;
+            final int THOUSAND_MS = 1000;
+
+            boolean fail = false;
+
+            long start = System.currentTimeMillis();
+
+            // import in increments of 10k blocks
+            while (currentBlock <= topBlockNumber) {
+                block = store.getChainBlockByNumber(currentBlock);
+                result =
+                        chain.tryToConnectAndFetchSummary(
+                                block, System.currentTimeMillis() / THOUSAND_MS, false);
+
+                if (!result.getLeft().isSuccessful()) {
+                    System.out.println("Consensus break at block:\n" + block);
+                    System.out.println(
+                            "Import attempt returned result "
+                                    + result.getLeft()
+                                    + " with summary\n"
+                                    + result.getRight());
+                    fail = true;
+                    break;
+                }
+
+                if (currentBlock % stepSize == 0) {
+                    long time = System.currentTimeMillis() - start;
+
+                    double timePerBlock = time / currentBlock;
+                    long remainingBlocks = topBlockNumber - currentBlock;
+                    double estimate = (timePerBlock * remainingBlocks) / 60_000 + 1; // in minutes
+                    System.out.println(
+                            "Finished with blocks up to "
+                                    + currentBlock
+                                    + " in "
+                                    + time
+                                    + " ms (under "
+                                    + (time / 60_000 + 1)
+                                    + " min). The average time per block is < "
+                                    + String.format("%.0f", timePerBlock + 1)
+                                    + " ms. Completion for remaining "
+                                    + remainingBlocks
+                                    + " blocks estimated to take "
+                                    + String.format("%.0f", estimate)
+                                    + " min.");
+                }
+
+                currentBlock++;
+            }
+
+            if (fail) {
+                System.out.println("Re-importing stored blocks FAILED due to consensus issues.");
+            } else {
+                System.out.println("Re-importing stored blocks SUCCESSFUL.");
+            }
+        } else {
+            System.out.println(
+                    "The given height "
+                            + startHeight
+                            + " is above the best known block "
+                            + block.getNumber()
+                            + ". Nothing to do.");
+        }
+
+        System.out.println("Closing databases...");
+        repo.close();
+
+        System.out.println("Re-import COMPLETE.");
+    }
 }