From 594ec1db6107c943e713d18c6bd8c31dd5bbc3e0 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 4 Feb 2025 09:06:09 +1000 Subject: [PATCH 1/4] increase cache to a more realistic number Signed-off-by: Gabriel Fukushima --- .../teku/storage/server/kvstore/KvStoreConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java index f921f8dbcfa..937c852d1ee 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java @@ -51,7 +51,7 @@ public class KvStoreConfiguration { public static final int DEFAULT_MAX_BACKGROUND_JOBS = 6; public static final int DEFAULT_BACKGROUND_THREAD_COUNT = 6; - public static final long DEFAULT_CACHE_CAPACITY = 8 << 20; // 8MB + public static final long DEFAULT_CACHE_CAPACITY = 134217728; // 128MB public static final long DEFAULT_WRITE_BUFFER_CAPACITY = 128 << 20; private static final boolean DEFAULT_OPTIMISE_FOR_SMALL_DB = false; From d9599208ba2965264ff6f4b74c91899d5fb5c724 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 4 Feb 2025 09:08:14 +1000 Subject: [PATCH 2/4] add extra config to recicle logfile and set a max total walsize Signed-off-by: Gabriel Fukushima --- .../storage/server/kvstore/KvStoreConfiguration.java | 6 ++++++ .../server/rocksdb/RocksDbInstanceFactory.java | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java index 937c852d1ee..e10704b0801 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java @@ -61,6 +61,12 @@ public class KvStoreConfiguration { /** RocksDb Time to roll a log file (1 day = 3600 * 24 seconds) */ public static final long TIME_TO_ROLL_LOG_FILE = 86_400L; + /** Max total size of all WAL file, after which a flush is triggered */ + public static final long WAL_MAX_TOTAL_SIZE = 1_073_741_824L; + + /** Expected size of a single WAL file, to determine how many WAL files to keep around */ + public static final long EXPECTED_WAL_FILE_SIZE = 67_108_864L; + public static final long ROCKSDB_BLOCK_SIZE = 32_768; /* --------------- Safe to Change Properties ------------ */ diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java index fc352ac1b69..29368d56a7a 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java @@ -14,9 +14,11 @@ package tech.pegasys.teku.storage.server.rocksdb; import static com.google.common.base.Preconditions.checkArgument; +import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.EXPECTED_WAL_FILE_SIZE; import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.NUMBER_OF_LOG_FILES_TO_KEEP; import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.ROCKSDB_BLOCK_SIZE; import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.TIME_TO_ROLL_LOG_FILE; +import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.WAL_MAX_TOTAL_SIZE; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; @@ -151,7 +153,10 @@ private static DBOptions createDBOptions( .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE) .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP) .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount())) - .setStatistics(stats); + .setStatistics(stats) + .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE) + .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE); + ; // Java docs suggests this if db is under 1GB, nearly impossible atm if (configuration.optimizeForSmallDb()) { @@ -165,7 +170,7 @@ private static ColumnFamilyOptions createColumnFamilyOptions( final KvStoreConfiguration configuration, final Cache cache) { return new ColumnFamilyOptions() .setCompressionType(configuration.getCompressionType()) - .setBottommostCompressionType(configuration.getBottomMostCompressionType()) + .setLevelCompactionDynamicLevelBytes(true) .setTtl(0) .setTableFormatConfig(createBlockBasedTableConfig(cache)); } @@ -189,7 +194,7 @@ private static BlockBasedTableConfig createBlockBasedTableConfig(final Cache cac .setBlockCache(cache) .setFilterPolicy(new BloomFilter(10, false)) .setPartitionFilters(true) - .setCacheIndexAndFilterBlocks(true) + .setCacheIndexAndFilterBlocks(false) .setBlockSize(ROCKSDB_BLOCK_SIZE); } } From fa717b71535701cfbe47b43d09759ce03267e6b7 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 6 Feb 2025 10:21:33 +1000 Subject: [PATCH 3/4] add back setBottommostCompressionType config there Signed-off-by: Gabriel Fukushima --- .../teku/storage/server/rocksdb/RocksDbInstanceFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java index 29368d56a7a..d1b9f4d10f3 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java @@ -156,7 +156,6 @@ private static DBOptions createDBOptions( .setStatistics(stats) .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE) .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE); - ; // Java docs suggests this if db is under 1GB, nearly impossible atm if (configuration.optimizeForSmallDb()) { @@ -170,6 +169,7 @@ private static ColumnFamilyOptions createColumnFamilyOptions( final KvStoreConfiguration configuration, final Cache cache) { return new ColumnFamilyOptions() .setCompressionType(configuration.getCompressionType()) + .setBottommostCompressionType(configuration.getBottomMostCompressionType()) .setLevelCompactionDynamicLevelBytes(true) .setTtl(0) .setTableFormatConfig(createBlockBasedTableConfig(cache)); From 2739b5b2d47ebbdc09333d69caa62c3652cbaae6 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 6 Feb 2025 10:30:07 +1000 Subject: [PATCH 4/4] use bit shift notation for cache capacity Signed-off-by: Gabriel Fukushima --- .../teku/storage/server/kvstore/KvStoreConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java index e10704b0801..70b0cd0acbc 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java @@ -51,7 +51,7 @@ public class KvStoreConfiguration { public static final int DEFAULT_MAX_BACKGROUND_JOBS = 6; public static final int DEFAULT_BACKGROUND_THREAD_COUNT = 6; - public static final long DEFAULT_CACHE_CAPACITY = 134217728; // 128MB + public static final long DEFAULT_CACHE_CAPACITY = 128 << 20; public static final long DEFAULT_WRITE_BUFFER_CAPACITY = 128 << 20; private static final boolean DEFAULT_OPTIMISE_FOR_SMALL_DB = false;