Rocksdb performance improvements (Consensys#9080)

* increase cache to a more realistic number Signed-off-by: Gabriel Fukushima <[email protected]> * add extra config to recicle logfile and set a max total walsize Signed-off-by: Gabriel Fukushima <[email protected]> * add back setBottommostCompressionType config there Signed-off-by: Gabriel Fukushima <[email protected]> * use bit shift notation for cache capacity Signed-off-by: Gabriel Fukushima <[email protected]> --------- Signed-off-by: Gabriel Fukushima <[email protected]>
mehdi-aouadi · Feb 6, 2025 · 0fdf7be · 0fdf7be
1 parent fe1947a
commit 0fdf7be
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 3 deletions.
diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java
@@ -51,7 +51,7 @@ public class KvStoreConfiguration {
 
   public static final int DEFAULT_MAX_BACKGROUND_JOBS = 6;
   public static final int DEFAULT_BACKGROUND_THREAD_COUNT = 6;
-  public static final long DEFAULT_CACHE_CAPACITY = 8 << 20; // 8MB
+  public static final long DEFAULT_CACHE_CAPACITY = 128 << 20;
   public static final long DEFAULT_WRITE_BUFFER_CAPACITY = 128 << 20;
   private static final boolean DEFAULT_OPTIMISE_FOR_SMALL_DB = false;
 
@@ -61,6 +61,12 @@ public class KvStoreConfiguration {
   /** RocksDb Time to roll a log file (1 day = 3600 * 24 seconds) */
   public static final long TIME_TO_ROLL_LOG_FILE = 86_400L;
 
+  /** Max total size of all WAL file, after which a flush is triggered */
+  public static final long WAL_MAX_TOTAL_SIZE = 1_073_741_824L;
+
+  /** Expected size of a single WAL file, to determine how many WAL files to keep around */
+  public static final long EXPECTED_WAL_FILE_SIZE = 67_108_864L;
+
   public static final long ROCKSDB_BLOCK_SIZE = 32_768;
 
   /* --------------- Safe to Change Properties ------------ */

diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java
@@ -14,9 +14,11 @@
 package tech.pegasys.teku.storage.server.rocksdb;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.EXPECTED_WAL_FILE_SIZE;
 import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.NUMBER_OF_LOG_FILES_TO_KEEP;
 import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.ROCKSDB_BLOCK_SIZE;
 import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.TIME_TO_ROLL_LOG_FILE;
+import static tech.pegasys.teku.storage.server.kvstore.KvStoreConfiguration.WAL_MAX_TOTAL_SIZE;
 
 import com.google.common.collect.ImmutableMap;
 import java.util.ArrayList;
@@ -151,7 +153,9 @@ private static DBOptions createDBOptions(
             .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE)
             .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP)
             .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount()))
-            .setStatistics(stats);
+            .setStatistics(stats)
+            .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE)
+            .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE);
 
     // Java docs suggests this if db is under 1GB, nearly impossible atm
     if (configuration.optimizeForSmallDb()) {
@@ -166,6 +170,7 @@ private static ColumnFamilyOptions createColumnFamilyOptions(
     return new ColumnFamilyOptions()
         .setCompressionType(configuration.getCompressionType())
         .setBottommostCompressionType(configuration.getBottomMostCompressionType())
+        .setLevelCompactionDynamicLevelBytes(true)
         .setTtl(0)
         .setTableFormatConfig(createBlockBasedTableConfig(cache));
   }
@@ -189,7 +194,7 @@ private static BlockBasedTableConfig createBlockBasedTableConfig(final Cache cac
         .setBlockCache(cache)
         .setFilterPolicy(new BloomFilter(10, false))
         .setPartitionFilters(true)
-        .setCacheIndexAndFilterBlocks(true)
+        .setCacheIndexAndFilterBlocks(false)
         .setBlockSize(ROCKSDB_BLOCK_SIZE);
   }
 }