Add rocksdb_block_cache_numshardbits for issue 1336

This fixes facebook#1336 The default value is -1 to match existing behavior. When -1 RocksDB code will determine the number of block cache shards as min(6, rocksdb_block_cache_size / min_shard_size) and today min_shard_size is 512K for LRU and 32M for Hyper. The math above frequently results in a block cache with too many small shards when rocksdb_block_cache_size is not too big (a few GB is not too big) and there will be perf problems that are hard to debug in such a case.
mdcallag · Jul 19, 2023 · 8d6bd1e · 8d6bd1e
1 parent 6893ef0
commit 8d6bd1e
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 3 deletions.
diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result
@@ -907,6 +907,7 @@ rocksdb_binlog_ttl	OFF
 rocksdb_binlog_ttl_compaction_ts_interval_secs	3600
 rocksdb_binlog_ttl_compaction_ts_offset_secs	60
 rocksdb_blind_delete_primary_key	OFF
+rocksdb_block_cache_numshardbits	-1
 rocksdb_block_cache_size	536870912
 rocksdb_block_restart_interval	16
 rocksdb_block_size	4096

diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_block_cache_numshardbits_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_block_cache_numshardbits_basic.result
@@ -0,0 +1,7 @@
+SET @start_global_value = @@global.ROCKSDB_BLOCK_CACHE_NUMSHARDBITS;
+SELECT @start_global_value;
+@start_global_value
+-1
+"Trying to set variable @@global.ROCKSDB_BLOCK_CACHE_NUMSHARDBITS to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_BLOCK_CACHE_NUMSHARDBITS   = 444;
+ERROR HY000: Variable 'rocksdb_block_cache_numshardbits' is a read only variable
diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_block_cache_numshardbits_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_block_cache_numshardbits_basic.test
@@ -0,0 +1,6 @@
+--source include/have_rocksdb.inc
+
+--let $sys_var=ROCKSDB_BLOCK_CACHE_NUMSHARDBITS
+--let $read_only=1
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc
@@ -920,6 +920,7 @@ static long long rocksdb_compaction_sequential_deletes_file_size = 0l;
 static uint32_t rocksdb_validate_tables = 1;
 char *rocksdb_datadir;
 static uint32_t rocksdb_max_bottom_pri_background_compactions = 0;
+static int32_t rocksdb_block_cache_numshardbits = -1;
 static uint32_t rocksdb_table_stats_sampling_pct;
 static uint32_t rocksdb_table_stats_recalc_threshold_pct = 10;
 static unsigned long long rocksdb_table_stats_recalc_threshold_count = 100ul;
@@ -2052,6 +2053,13 @@ static MYSQL_SYSVAR_INT(table_cache_numshardbits,
                         // fails to create a cache and returns a nullptr
                         /* min */ 0, /* max */ 19, 0);
 
+static MYSQL_SYSVAR_INT(block_cache_numshardbits,
+                        rocksdb_block_cache_numshardbits,
+                        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+                        "Block cache numshardbits for RocksDB",
+                        nullptr, nullptr,
+                       /* default */ -1, /* min */ -1, /* max */ 8, 0);
+
 static MYSQL_SYSVAR_UINT64_T(wal_ttl_seconds,
                              rocksdb_db_options->WAL_ttl_seconds,
                              PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
@@ -3039,6 +3047,7 @@ static struct SYS_VAR *rocksdb_system_variables[] = {
     MYSQL_SYSVAR(keep_log_file_num),
     MYSQL_SYSVAR(max_manifest_file_size),
     MYSQL_SYSVAR(table_cache_numshardbits),
+    MYSQL_SYSVAR(block_cache_numshardbits),
     MYSQL_SYSVAR(wal_ttl_seconds),
     MYSQL_SYSVAR(wal_size_limit_mb),
     MYSQL_SYSVAR(manifest_preallocation_size),
@@ -7952,13 +7961,13 @@ static int rocksdb_init_internal(void *const p) {
         rocksdb_use_hyper_clock_cache
             ? rocksdb::HyperClockCacheOptions(
                   rocksdb_block_cache_size, rocksdb_tbl_options->block_size,
-                  -1
-                  /* num_shard_bits */,
+                  rocksdb_block_cache_numshardbits /* num_shard_bits */,
                   false /* strict_capacity_limit */, memory_allocator)
                   .MakeSharedCache()
 
             : rocksdb::NewLRUCache(
-                  rocksdb_block_cache_size, -1 /*num_shard_bits*/,
+                  rocksdb_block_cache_size,
+                  rocksdb_block_cache_numshardbits /*num_shard_bits*/,
                   false /*strict_capcity_limit*/,
                   rocksdb_cache_high_pri_pool_ratio, memory_allocator);
     if (rocksdb_sim_cache_size > 0) {