cdc: adjust sorter options to avoid Seek CPU usage exploding (#11099) (…

…#11133) close #11078
pingcap · Nov 4, 2024 · 7c67a5d · 7c67a5d
1 parent 1bbdac3
commit 7c67a5d
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 7 deletions.
diff --git a/cdc/processor/sourcemanager/engine/pebble/db.go b/cdc/processor/sourcemanager/engine/pebble/db.go
@@ -114,7 +114,8 @@ func buildPebbleOption(cfg *config.DBConfig) (opts *pebble.Options) {
 	opts.DisableWAL = false // Delete range requires WAL.
 	opts.MaxOpenFiles = cfg.MaxOpenFiles / cfg.Count
 	opts.MaxConcurrentCompactions = 6
-	opts.L0CompactionThreshold = cfg.CompactionL0Trigger
+	opts.L0CompactionThreshold = 4 // Default for PebbleDB.
+	opts.L0CompactionFileThreshold = cfg.CompactionL0Trigger
 	opts.L0StopWritesThreshold = cfg.WriteL0PauseTrigger
 	opts.LBaseMaxBytes = 64 << 20 // 64 MB
 	opts.MemTableSize = cfg.WriterBufferSize

diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go
@@ -175,7 +175,7 @@ func TestParseCfg(t *testing.T) {
 				WriterBufferSize:            8388608,
 				Compression:                 "snappy",
 				WriteL0PauseTrigger:         math.MaxInt32,
-				CompactionL0Trigger:         160,
+				CompactionL0Trigger:         16,
 				CompactionDeletionThreshold: 10485760,
 				CompactionPeriod:            1800,
 				IteratorMaxAliveDuration:    10000,
@@ -465,7 +465,7 @@ cert-allowed-cn = ["dd","ee"]
 				WriterBufferSize:            8388608,
 				Compression:                 "snappy",
 				WriteL0PauseTrigger:         math.MaxInt32,
-				CompactionL0Trigger:         160,
+				CompactionL0Trigger:         16,
 				CompactionDeletionThreshold: 10485760,
 				CompactionPeriod:            1800,
 				IteratorMaxAliveDuration:    10000,
@@ -532,7 +532,7 @@ unknown3 = 3
 			WriterBufferSize:            8388608,
 			Compression:                 "snappy",
 			WriteL0PauseTrigger:         math.MaxInt32,
-			CompactionL0Trigger:         160,
+			CompactionL0Trigger:         16,
 			CompactionDeletionThreshold: 10485760,
 			CompactionPeriod:            1800,
 			IteratorMaxAliveDuration:    10000,

diff --git a/pkg/config/config_test_data.go b/pkg/config/config_test_data.go
@@ -137,7 +137,7 @@ const (
       "writer-buffer-size": 8388608,
       "compression": "snappy",
       "write-l0-pause-trigger": 2147483647,
-      "compaction-l0-trigger": 160,
+      "compaction-l0-trigger": 16,
       "compaction-deletion-threshold": 10485760,
       "compaction-period": 1800,
       "iterator-max-alive-duration": 10000,

diff --git a/pkg/config/db.go b/pkg/config/db.go
@@ -51,7 +51,7 @@ type DBConfig struct {
 	// CompactionL0Trigger defines number of db sst file at level-0 that will
 	// trigger compaction.
 	//
-	// The default value is 160.
+	// The default value is 16, which is based on a performance test on 4K tables.
 	CompactionL0Trigger int `toml:"compaction-l0-trigger" json:"compaction-l0-trigger"`
 	// CompactionDeletionThreshold defines the threshold of the number of deletion that
 	// trigger compaction.

diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go
@@ -130,7 +130,7 @@ var defaultServerConfig = &ServerConfig{
 			WriterBufferSize:            8388608,
 			Compression:                 "snappy",
 			WriteL0PauseTrigger:         math.MaxInt32,
-			CompactionL0Trigger:         160,
+			CompactionL0Trigger:         16,
 			CompactionDeletionThreshold: 10485760,
 			CompactionPeriod:            1800,
 			IteratorMaxAliveDuration:    10000,