Skip to content

Commit 4f3ed77

Browse files
xperimentalxperimental
authored
fix(operator): Use a minimum value for replay memory ceiling (#13066)
1 parent 00d3c7a commit 4f3ed77

File tree

2 files changed

+44
-34
lines changed

2 files changed

+44
-34
lines changed

‎operator/internal/manifests/internal/config/build_test.go

+34-34
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ ingester:
6969
wal:
7070
enabled: true
7171
dir: /tmp/wal
72-
replay_memory_ceiling: 2500
72+
replay_memory_ceiling: 536870912
7373
ingester_client:
7474
grpc_client_config:
7575
max_recv_msg_size: 67108864
@@ -240,7 +240,7 @@ overrides:
240240
},
241241
WriteAheadLog: WriteAheadLog{
242242
Directory: "/tmp/wal",
243-
IngesterMemoryRequest: 5000,
243+
IngesterMemoryRequest: 0,
244244
},
245245
ObjectStorage: storage.Options{
246246
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -327,7 +327,7 @@ ingester:
327327
wal:
328328
enabled: true
329329
dir: /tmp/wal
330-
replay_memory_ceiling: 2500
330+
replay_memory_ceiling: 2147483648
331331
ingester_client:
332332
grpc_client_config:
333333
max_recv_msg_size: 67108864
@@ -588,7 +588,7 @@ overrides:
588588
},
589589
WriteAheadLog: WriteAheadLog{
590590
Directory: "/tmp/wal",
591-
IngesterMemoryRequest: 5000,
591+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
592592
},
593593
ObjectStorage: storage.Options{
594594
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -673,7 +673,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
673673
},
674674
WriteAheadLog: WriteAheadLog{
675675
Directory: "/tmp/wal",
676-
IngesterMemoryRequest: 5000,
676+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
677677
},
678678
ObjectStorage: storage.Options{
679679
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -754,7 +754,7 @@ ingester:
754754
wal:
755755
enabled: true
756756
dir: /tmp/wal
757-
replay_memory_ceiling: 2500
757+
replay_memory_ceiling: 2147483648
758758
ingester_client:
759759
grpc_client_config:
760760
max_recv_msg_size: 67108864
@@ -1026,7 +1026,7 @@ overrides:
10261026
},
10271027
WriteAheadLog: WriteAheadLog{
10281028
Directory: "/tmp/wal",
1029-
IngesterMemoryRequest: 5000,
1029+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
10301030
},
10311031
ObjectStorage: storage.Options{
10321032
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -1113,7 +1113,7 @@ ingester:
11131113
wal:
11141114
enabled: true
11151115
dir: /tmp/wal
1116-
replay_memory_ceiling: 2500
1116+
replay_memory_ceiling: 2147483648
11171117
ingester_client:
11181118
grpc_client_config:
11191119
max_recv_msg_size: 67108864
@@ -1386,7 +1386,7 @@ overrides:
13861386
},
13871387
WriteAheadLog: WriteAheadLog{
13881388
Directory: "/tmp/wal",
1389-
IngesterMemoryRequest: 5000,
1389+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
13901390
},
13911391
ObjectStorage: storage.Options{
13921392
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -1473,7 +1473,7 @@ ingester:
14731473
wal:
14741474
enabled: true
14751475
dir: /tmp/wal
1476-
replay_memory_ceiling: 2500
1476+
replay_memory_ceiling: 2147483648
14771477
ingester_client:
14781478
grpc_client_config:
14791479
max_recv_msg_size: 67108864
@@ -1776,7 +1776,7 @@ overrides:
17761776
},
17771777
WriteAheadLog: WriteAheadLog{
17781778
Directory: "/tmp/wal",
1779-
IngesterMemoryRequest: 5000,
1779+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
17801780
},
17811781
ObjectStorage: storage.Options{
17821782
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -1866,7 +1866,7 @@ ingester:
18661866
wal:
18671867
enabled: true
18681868
dir: /tmp/wal
1869-
replay_memory_ceiling: 2500
1869+
replay_memory_ceiling: 2147483648
18701870
ingester_client:
18711871
grpc_client_config:
18721872
max_recv_msg_size: 67108864
@@ -2112,7 +2112,7 @@ overrides:
21122112
},
21132113
WriteAheadLog: WriteAheadLog{
21142114
Directory: "/tmp/wal",
2115-
IngesterMemoryRequest: 5000,
2115+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
21162116
},
21172117
ObjectStorage: storage.Options{
21182118
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -2202,7 +2202,7 @@ ingester:
22022202
wal:
22032203
enabled: true
22042204
dir: /tmp/wal
2205-
replay_memory_ceiling: 2500
2205+
replay_memory_ceiling: 2147483648
22062206
ingester_client:
22072207
grpc_client_config:
22082208
max_recv_msg_size: 67108864
@@ -2535,7 +2535,7 @@ overrides:
25352535
},
25362536
WriteAheadLog: WriteAheadLog{
25372537
Directory: "/tmp/wal",
2538-
IngesterMemoryRequest: 5000,
2538+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
25392539
},
25402540
ObjectStorage: storage.Options{
25412541
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -2635,7 +2635,7 @@ ingester:
26352635
wal:
26362636
enabled: true
26372637
dir: /tmp/wal
2638-
replay_memory_ceiling: 2500
2638+
replay_memory_ceiling: 2147483648
26392639
ingester_client:
26402640
grpc_client_config:
26412641
max_recv_msg_size: 67108864
@@ -2881,7 +2881,7 @@ overrides:
28812881
},
28822882
WriteAheadLog: WriteAheadLog{
28832883
Directory: "/tmp/wal",
2884-
IngesterMemoryRequest: 5000,
2884+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
28852885
},
28862886
ObjectStorage: storage.Options{
28872887
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -2967,7 +2967,7 @@ ingester:
29672967
wal:
29682968
enabled: true
29692969
dir: /tmp/wal
2970-
replay_memory_ceiling: 2500
2970+
replay_memory_ceiling: 2147483648
29712971
ingester_client:
29722972
grpc_client_config:
29732973
max_recv_msg_size: 67108864
@@ -3377,7 +3377,7 @@ overrides:
33773377
},
33783378
WriteAheadLog: WriteAheadLog{
33793379
Directory: "/tmp/wal",
3380-
IngesterMemoryRequest: 5000,
3380+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
33813381
},
33823382
ObjectStorage: storage.Options{
33833383
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -3465,7 +3465,7 @@ ingester:
34653465
wal:
34663466
enabled: true
34673467
dir: /tmp/wal
3468-
replay_memory_ceiling: 2500
3468+
replay_memory_ceiling: 2147483648
34693469
ingester_client:
34703470
grpc_client_config:
34713471
max_recv_msg_size: 67108864
@@ -3638,7 +3638,7 @@ overrides:
36383638
},
36393639
WriteAheadLog: WriteAheadLog{
36403640
Directory: "/tmp/wal",
3641-
IngesterMemoryRequest: 5000,
3641+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
36423642
},
36433643
ObjectStorage: storage.Options{
36443644
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -3727,7 +3727,7 @@ ingester:
37273727
wal:
37283728
enabled: true
37293729
dir: /tmp/wal
3730-
replay_memory_ceiling: 2500
3730+
replay_memory_ceiling: 2147483648
37313731
ingester_client:
37323732
grpc_client_config:
37333733
max_recv_msg_size: 67108864
@@ -3901,7 +3901,7 @@ overrides:
39013901
},
39023902
WriteAheadLog: WriteAheadLog{
39033903
Directory: "/tmp/wal",
3904-
IngesterMemoryRequest: 5000,
3904+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
39053905
},
39063906
ObjectStorage: storage.Options{
39073907
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -3990,7 +3990,7 @@ ingester:
39903990
wal:
39913991
enabled: true
39923992
dir: /tmp/wal
3993-
replay_memory_ceiling: 2500
3993+
replay_memory_ceiling: 2147483648
39943994
ingester_client:
39953995
grpc_client_config:
39963996
max_recv_msg_size: 67108864
@@ -4162,7 +4162,7 @@ overrides:
41624162
},
41634163
WriteAheadLog: WriteAheadLog{
41644164
Directory: "/tmp/wal",
4165-
IngesterMemoryRequest: 5000,
4165+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
41664166
},
41674167
ObjectStorage: storage.Options{
41684168
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -4254,7 +4254,7 @@ ingester:
42544254
wal:
42554255
enabled: true
42564256
dir: /tmp/wal
4257-
replay_memory_ceiling: 2500
4257+
replay_memory_ceiling: 2147483648
42584258
ingester_client:
42594259
grpc_client_config:
42604260
max_recv_msg_size: 67108864
@@ -4460,7 +4460,7 @@ overrides:
44604460
},
44614461
WriteAheadLog: WriteAheadLog{
44624462
Directory: "/tmp/wal",
4463-
IngesterMemoryRequest: 5000,
4463+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
44644464
},
44654465
ObjectStorage: storage.Options{
44664466
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -4554,7 +4554,7 @@ ingester:
45544554
wal:
45554555
enabled: true
45564556
dir: /tmp/wal
4557-
replay_memory_ceiling: 2500
4557+
replay_memory_ceiling: 2147483648
45584558
ingester_client:
45594559
grpc_client_config:
45604560
max_recv_msg_size: 67108864
@@ -4760,7 +4760,7 @@ overrides:
47604760
},
47614761
WriteAheadLog: WriteAheadLog{
47624762
Directory: "/tmp/wal",
4763-
IngesterMemoryRequest: 5000,
4763+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
47644764
},
47654765
ObjectStorage: storage.Options{
47664766
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -4852,7 +4852,7 @@ ingester:
48524852
wal:
48534853
enabled: true
48544854
dir: /tmp/wal
4855-
replay_memory_ceiling: 2500
4855+
replay_memory_ceiling: 2147483648
48564856
ingester_client:
48574857
grpc_client_config:
48584858
max_recv_msg_size: 67108864
@@ -5019,7 +5019,7 @@ overrides:
50195019
},
50205020
WriteAheadLog: WriteAheadLog{
50215021
Directory: "/tmp/wal",
5022-
IngesterMemoryRequest: 5000,
5022+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
50235023
},
50245024
ObjectStorage: storage.Options{
50255025
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -5109,7 +5109,7 @@ func defaultOptions() Options {
51095109
},
51105110
WriteAheadLog: WriteAheadLog{
51115111
Directory: "/tmp/wal",
5112-
IngesterMemoryRequest: 5000,
5112+
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
51135113
},
51145114
ObjectStorage: storage.Options{
51155115
SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -5343,7 +5343,7 @@ ingester:
53435343
wal:
53445344
enabled: true
53455345
dir: /tmp/wal
5346-
replay_memory_ceiling: 2500
5346+
replay_memory_ceiling: 2147483648
53475347
ingester_client:
53485348
grpc_client_config:
53495349
max_recv_msg_size: 67108864
@@ -5518,7 +5518,7 @@ ingester:
55185518
wal:
55195519
enabled: true
55205520
dir: /tmp/wal
5521-
replay_memory_ceiling: 2500
5521+
replay_memory_ceiling: 2147483648
55225522
ingester_client:
55235523
grpc_client_config:
55245524
max_recv_msg_size: 67108864

‎operator/internal/manifests/internal/config/options.go

+10
Original file line numberDiff line numberDiff line change
@@ -223,10 +223,20 @@ type WriteAheadLog struct {
223223
IngesterMemoryRequest int64
224224
}
225225

226+
const (
227+
// minimumReplayCeiling contains the minimum value that will be used for the replay_memory_ceiling.
228+
// It is set, so that even when the ingester has a low memory request, the replay will not flush each block
229+
// on its own.
230+
minimumReplayCeiling = 512 * 1024 * 1024
231+
)
232+
226233
// ReplayMemoryCeiling calculates 50% of the ingester memory
227234
// for the ingester to use for the write-ahead-log capbability.
228235
func (w WriteAheadLog) ReplayMemoryCeiling() string {
229236
value := int64(math.Ceil(float64(w.IngesterMemoryRequest) * float64(0.5)))
237+
if value < minimumReplayCeiling {
238+
value = minimumReplayCeiling
239+
}
230240
return fmt.Sprintf("%d", value)
231241
}
232242

0 commit comments

Comments
 (0)