[PAN-2818] Database versioning and enable multi-column database (#1830)

* Database Versioning: The behavior is to load the database at the existing version if it
already exists or create the newest version if it doesn't

* multi-column by default: This makes the separated world state storage column required by mark sweep on by default

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/DatabaseMetadata.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.ethereum.storage.keyvalue;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Path;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class DatabaseMetadata {
+  static final String METADATA_FILENAME = "DATABASE_METADATA.json";
+  private static ObjectMapper MAPPER = new ObjectMapper();
+  private final int version;
+
+  @JsonCreator
+  DatabaseMetadata(@JsonProperty("version") final int version) {
+    this.version = version;
+  }
+
+  public int getVersion() {
+    return version;
+  }
+
+  static DatabaseMetadata fromDirectory(final Path databaseDir) throws IOException {
+    final File metadataFile = getDefaultMetadataFile(databaseDir);
+    try {
+      return MAPPER.readValue(metadataFile, DatabaseMetadata.class);
+    } catch (FileNotFoundException fnfe) {
+      return new DatabaseMetadata(0);
+    } catch (JsonProcessingException jpe) {
+      throw new IllegalStateException(
+          String.format("Invalid metadata file %s", metadataFile.getAbsolutePath()), jpe);
+    }
+  }
+
+  void writeToDirectory(final Path databaseDir) throws IOException {
+    MAPPER.writeValue(getDefaultMetadataFile(databaseDir), this);
+  }
+
+  private static File getDefaultMetadataFile(final Path databaseDir) {
+    return databaseDir.resolve(METADATA_FILENAME).toFile();
+  }
+}

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/RocksDbStorageProvider.java

@@ -12,6 +12,7 @@
  */
 package tech.pegasys.pantheon.ethereum.storage.keyvalue;
 
+import static java.util.AbstractMap.SimpleEntry;
 import static java.util.Arrays.asList;
 
 import tech.pegasys.pantheon.ethereum.storage.StorageProvider;
@@ -27,13 +28,24 @@
 
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 public class RocksDbStorageProvider {
   public static long DEFAULT_WORLD_STATE_PREIMAGE_CACHE_SIZE = 5_000L;
   private static final Logger LOG = LogManager.getLogger();
+  public static final int DEFAULT_VERSION = 1;
+  /** This key is the version and the value is the function used to create or load the database. */
+  private static final TreeMap<Integer, StorageProviderFunction> PROVIDERS_BY_VERSION =
+      new TreeMap<>(
+          Map.ofEntries(
+              new SimpleEntry<>(0, RocksDbStorageProvider::ofUnsegmented),
+              new SimpleEntry<>(1, RocksDbStorageProvider::ofSegmented)));
 
   public static StorageProvider create(
       final RocksDbConfiguration rocksDbConfiguration, final MetricsSystem metricsSystem)
@@ -46,34 +58,50 @@ public static StorageProvider create(
       final MetricsSystem metricsSystem,
       final long worldStatePreimageCacheSize)
       throws IOException {
-    if (rocksDbConfiguration.useColumns()) {
-      return createSegmentedProvider(
-          rocksDbConfiguration, metricsSystem, worldStatePreimageCacheSize);
+
+    final Path databaseDir = rocksDbConfiguration.getDatabaseDir();
+    final boolean databaseExists = databaseDir.resolve("IDENTITY").toFile().exists();
+    final int databaseVersion;
+    if (databaseExists) {
+      databaseVersion = DatabaseMetadata.fromDirectory(databaseDir).getVersion();
+      LOG.info("Existing database detected at {}. Version {}", databaseDir, databaseVersion);
     } else {
-      return createUnsegmentedProvider(
-          rocksDbConfiguration, metricsSystem, worldStatePreimageCacheSize);
+      databaseVersion = DEFAULT_VERSION;
+      LOG.info(
+          "No existing database detected at {}. Using version {}", databaseDir, databaseVersion);
+      Files.createDirectories(databaseDir);
+      new DatabaseMetadata(databaseVersion).writeToDirectory(databaseDir);
     }
+
+    final StorageProviderFunction providerFunction =
+        Optional.ofNullable(PROVIDERS_BY_VERSION.get(databaseVersion))
+            .orElseThrow(
+                () ->
+                    new IllegalStateException(
+                        String.format(
+                            "Invalid database version %d. Valid versions are: %s. Default version is %d",
+                            databaseVersion,
+                            PROVIDERS_BY_VERSION.navigableKeySet().toString(),
+                            DEFAULT_VERSION)));
+
+    return providerFunction.apply(rocksDbConfiguration, metricsSystem, worldStatePreimageCacheSize);
   }
 
-  private static StorageProvider createUnsegmentedProvider(
+  private static StorageProvider ofUnsegmented(
       final RocksDbConfiguration rocksDbConfiguration,
       final MetricsSystem metricsSystem,
-      final long worldStatePreimageCacheSize)
-      throws IOException {
-    Files.createDirectories(rocksDbConfiguration.getDatabaseDir());
+      final long worldStatePreimageCacheSize) {
     final KeyValueStorage kv = RocksDbKeyValueStorage.create(rocksDbConfiguration, metricsSystem);
     final KeyValueStorage preimageKv =
         new LimitedInMemoryKeyValueStorage(worldStatePreimageCacheSize);
     return new KeyValueStorageProvider(kv, kv, preimageKv, kv, kv, kv);
   }
 
-  private static StorageProvider createSegmentedProvider(
+  private static StorageProvider ofSegmented(
       final RocksDbConfiguration rocksDbConfiguration,
       final MetricsSystem metricsSystem,
-      final long worldStatePreimageCacheSize)
-      throws IOException {
-    LOG.info("Using RocksDB columns");
-    Files.createDirectories(rocksDbConfiguration.getDatabaseDir());
+      final long worldStatePreimageCacheSize) {
+
     final SegmentedKeyValueStorage<?> columnarStorage =
         ColumnarRocksDbKeyValueStorage.create(
             rocksDbConfiguration, asList(RocksDbSegment.values()), metricsSystem);
@@ -112,4 +140,12 @@ public byte[] getId() {
       return id;
     }
   }
+
+  private interface StorageProviderFunction {
+    StorageProvider apply(
+        final RocksDbConfiguration rocksDbConfiguration,
+        final MetricsSystem metricsSystem,
+        final long worldStatePreimageCacheSize)
+        throws IOException;
+  }
 }

ethereum/core/src/test/java/tech/pegasys/pantheon/ethereum/storage/keyvalue/RocksDbStorageProviderTest.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.ethereum.storage.keyvalue;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+
+import tech.pegasys.pantheon.metrics.MetricsSystem;
+import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem;
+import tech.pegasys.pantheon.services.kvstore.RocksDbConfiguration;
+
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RocksDbStorageProviderTest {
+
+  @Mock private RocksDbConfiguration rocksDbConfiguration;
+  @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private final MetricsSystem metricsSystem = new NoOpMetricsSystem();
+
+  @Test
+  public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception {
+    final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db");
+    when(rocksDbConfiguration.getDatabaseDir()).thenReturn(tempDatabaseDir);
+    RocksDbStorageProvider.create(rocksDbConfiguration, metricsSystem);
+    assertEquals(
+        RocksDbStorageProvider.DEFAULT_VERSION,
+        DatabaseMetadata.fromDirectory(rocksDbConfiguration.getDatabaseDir()).getVersion());
+  }
+
+  @Test
+  public void shouldDetectVersion0DatabaseIfNoMetadataFileFound() throws Exception {
+    final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db");
+    Files.createDirectories(tempDatabaseDir);
+    tempDatabaseDir.resolve("IDENTITY").toFile().createNewFile();
+    when(rocksDbConfiguration.getDatabaseDir()).thenReturn(tempDatabaseDir);
+    RocksDbStorageProvider.create(rocksDbConfiguration, metricsSystem);
+    assertEquals(0, DatabaseMetadata.fromDirectory(tempDatabaseDir).getVersion());
+  }
+
+  @Test
+  public void shouldDetectCorrectVersionIfMetadataFileExists() throws Exception {
+    final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db");
+    Files.createDirectories(tempDatabaseDir);
+    tempDatabaseDir.resolve("IDENTITY").toFile().createNewFile();
+    new DatabaseMetadata(1).writeToDirectory(tempDatabaseDir);
+    when(rocksDbConfiguration.getDatabaseDir()).thenReturn(tempDatabaseDir);
+    RocksDbStorageProvider.create(rocksDbConfiguration, metricsSystem);
+    assertEquals(1, DatabaseMetadata.fromDirectory(tempDatabaseDir).getVersion());
+  }
+
+  @Test
+  public void shouldThrowExceptionWhenVersionNumberIsInvalid() throws Exception {
+    final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db");
+    Files.createDirectories(tempDatabaseDir);
+    tempDatabaseDir.resolve("IDENTITY").toFile().createNewFile();
+    new DatabaseMetadata(-1).writeToDirectory(tempDatabaseDir);
+    when(rocksDbConfiguration.getDatabaseDir()).thenReturn(tempDatabaseDir);
+    assertThatThrownBy(() -> RocksDbStorageProvider.create(rocksDbConfiguration, metricsSystem))
+        .isInstanceOf(IllegalStateException.class);
+  }
+
+  @Test
+  public void shouldThrowExceptionWhenMetaDataFileIsCorrupted() throws Exception {
+    final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db");
+    Files.createDirectories(tempDatabaseDir);
+    when(rocksDbConfiguration.getDatabaseDir()).thenReturn(tempDatabaseDir);
+    tempDatabaseDir.resolve("IDENTITY").toFile().createNewFile();
+
+    final String badVersion = "{\"🦄\":1}";
+    Files.write(
+        tempDatabaseDir.resolve(DatabaseMetadata.METADATA_FILENAME),
+        badVersion.getBytes(Charset.defaultCharset()));
+    assertThatThrownBy(() -> RocksDbStorageProvider.create(rocksDbConfiguration, metricsSystem))
+        .isInstanceOf(IllegalStateException.class);
+
+    final String badValue = "{\"version\":\"iomedae\"}";
+    Files.write(
+        tempDatabaseDir.resolve(DatabaseMetadata.METADATA_FILENAME),
+        badValue.getBytes(Charset.defaultCharset()));
+    assertThatThrownBy(() -> RocksDbStorageProvider.create(rocksDbConfiguration, metricsSystem))
+        .isInstanceOf(IllegalStateException.class);
+  }
+}

services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/ColumnarRocksDbKeyValueStorage.java

@@ -12,6 +12,8 @@
  */
 package tech.pegasys.pantheon.services.kvstore;
 
+import static java.util.Objects.requireNonNullElse;
+
 import tech.pegasys.pantheon.metrics.MetricsSystem;
 import tech.pegasys.pantheon.metrics.OperationTimer;
 import tech.pegasys.pantheon.services.util.RocksDbUtil;
@@ -110,13 +112,12 @@ private ColumnarRocksDbKeyValueStorage(
                   Collectors.toMap(segment -> BytesValue.wrap(segment.getId()), Segment::getName));
 
       final ImmutableMap.Builder<String, ColumnFamilyHandle> builder = ImmutableMap.builder();
-      for (final ColumnFamilyHandle columnHandle : columnHandles) {
-        final String segmentName = segmentsById.get(BytesValue.wrap(columnHandle.getName()));
-        if (segmentName != null) {
-          builder.put(segmentName, columnHandle);
-        } else {
-          builder.put(DEFAULT_COLUMN, columnHandle);
-        }
+
+      for (ColumnFamilyHandle columnHandle : columnHandles) {
+        final String segmentName =
+            requireNonNullElse(
+                segmentsById.get(BytesValue.wrap(columnHandle.getName())), DEFAULT_COLUMN);
+        builder.put(segmentName, columnHandle);
       }
       columnHandlesByName = builder.build();
 

services/kvstore/src/main/java/tech/pegasys/pantheon/services/kvstore/RocksDbConfiguration.java

@@ -27,20 +27,17 @@ public class RocksDbConfiguration {
   private final String label;
   private final int maxBackgroundCompactions;
   private final int backgroundThreadCount;
-  private final boolean useColumns;
   private final long cacheCapacity;
 
   private RocksDbConfiguration(
       final Path databaseDir,
       final int maxOpenFiles,
       final int maxBackgroundCompactions,
       final int backgroundThreadCount,
-      final boolean useColumns,
       final long cacheCapacity,
       final String label) {
     this.maxBackgroundCompactions = maxBackgroundCompactions;
     this.backgroundThreadCount = backgroundThreadCount;
-    this.useColumns = useColumns;
     RocksDbUtil.loadNativeLibrary();
     this.databaseDir = databaseDir;
     this.maxOpenFiles = maxOpenFiles;
@@ -76,10 +73,6 @@ public String getLabel() {
     return label;
   }
 
-  public boolean useColumns() {
-    return useColumns;
-  }
-
   public static class Builder {
 
     Path databaseDir;
@@ -89,7 +82,6 @@ public static class Builder {
     long cacheCapacity = DEFAULT_CACHE_CAPACITY;
     int maxBackgroundCompactions = DEFAULT_MAX_BACKGROUND_COMPACTIONS;
     int backgroundThreadCount = DEFAULT_BACKGROUND_THREAD_COUNT;
-    boolean useColumns = false;
 
     private Builder() {}
 
@@ -123,18 +115,12 @@ public Builder backgroundThreadCount(final int backgroundThreadCount) {
       return this;
     }
 
-    public Builder useColumns(final boolean useColumns) {
-      this.useColumns = useColumns;
-      return this;
-    }
-
     public RocksDbConfiguration build() {
       return new RocksDbConfiguration(
           databaseDir,
           maxOpenFiles,
           maxBackgroundCompactions,
           backgroundThreadCount,
-          useColumns,
           cacheCapacity,
           label);
     }

services/kvstore/src/test/java/tech/pegasys/pantheon/services/kvstore/RocksDbKeyValueStorageTest.java

@@ -121,9 +121,6 @@ public void createStoreMustCreateMetrics() throws Exception {
   }
 
   private RocksDbConfiguration config() throws Exception {
-    return RocksDbConfiguration.builder()
-        .databaseDir(folder.newFolder().toPath())
-        .useColumns(false)
-        .build();
+    return RocksDbConfiguration.builder().databaseDir(folder.newFolder().toPath()).build();
   }
 }