Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

store: Add Memcached Index Cache mixin #106

Merged
merged 7 commits into from
Apr 8, 2020
Merged

store: Add Memcached Index Cache mixin #106

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ce4e158
Add thanos store memcached mixin
kakkoyun Apr 6, 2020
3251d5c
Upgrade Thanos, update changed flags
kakkoyun Apr 6, 2020
a9407c7
Increase dns update interval
kakkoyun Apr 7, 2020
83cb575
Fix styling
kakkoyun Apr 8, 2020
7c20972
Use default values in Thanos
kakkoyun Apr 8, 2020
266e344
Address review issues
kakkoyun Apr 8, 2020
479b1a8
Fix indentation
kakkoyun Apr 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 31 additions & 6 deletions all.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,36 +33,60 @@ local b = t.bucket + commonConfig + {
},
};

local c = t.compact + t.compact.withVolumeClaimTemplate + t.compact.withServiceMonitor + commonConfig + {
local c = t.compact +
t.compact.withVolumeClaimTemplate +
t.compact.withServiceMonitor +
commonConfig + {
config+:: {
name: 'thanos-compact',
replicas: 1,
},
};

local re = t.receive + t.receive.withVolumeClaimTemplate + t.receive.withServiceMonitor + commonConfig + {
local re = t.receive + t.receive.withVolumeClaimTemplate +
t.receive.withServiceMonitor +
commonConfig + {
config+:: {
name: 'thanos-receive',
replicas: 1,
replicationFactor: 1,
},
};

local ru = t.rule + t.rule.withVolumeClaimTemplate + t.rule.withServiceMonitor + commonConfig + {
local ru = t.rule + t.rule.withVolumeClaimTemplate +
t.rule.withServiceMonitor +
commonConfig + {
config+:: {
name: 'thanos-rule',
replicas: 1,
},
};

local s = t.store + t.store.withVolumeClaimTemplate + t.store.withServiceMonitor + commonConfig + {
local s = t.store + t.store.withVolumeClaimTemplate +
t.store.withServiceMonitor +
commonConfig + {
config+:: {
name: 'thanos-store',
replicas: 1,
},
};

local q = t.query + t.query.withServiceMonitor + commonConfig + {
local swm = t.store +
t.store.withVolumeClaimTemplate +
t.store.withServiceMonitor + t.store.withMemcachedIndexCache +
commonConfig + {
config+:: {
name: 'thanos-store',
replicas: 1,
memcached+: {
addresses: [],
},
},
};

local q = t.query +
t.query.withServiceMonitor +
commonConfig + {
config+:: {
name: 'thanos-query',
replicas: 1,
Expand All @@ -85,4 +109,5 @@ local finalRu = ru {
{ ['thanos-receive-' + name]: re[name] for name in std.objectFields(re) } +
{ ['thanos-rule-' + name]: finalRu[name] for name in std.objectFields(finalRu) } +
{ ['thanos-store-' + name]: s[name] for name in std.objectFields(s) } +
{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) }
{ ['thanos-query-' + name]: q[name] for name in std.objectFields(q) } +
{ 'thanos-store-statefulSet-with-memcached': swm.statefulSet }
93 changes: 93 additions & 0 deletions examples/all/manifests/thanos-store-statefulSet-with-memcached.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app.kubernetes.io/component: object-store-gateway
app.kubernetes.io/instance: thanos-store
app.kubernetes.io/name: thanos-store
app.kubernetes.io/version: v0.12.0-rc.0
name: thanos-store
namespace: thanos
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/component: object-store-gateway
app.kubernetes.io/instance: thanos-store
app.kubernetes.io/name: thanos-store
serviceName: thanos-store
template:
metadata:
labels:
app.kubernetes.io/component: object-store-gateway
app.kubernetes.io/instance: thanos-store
app.kubernetes.io/name: thanos-store
app.kubernetes.io/version: v0.12.0-rc.0
spec:
containers:
- args:
- store
- --data-dir=/var/thanos/store
- --grpc-address=0.0.0.0:10901
- --http-address=0.0.0.0:10902
- --objstore.config=$(OBJSTORE_CONFIG)
- --experimental.enable-index-cache-postings-compression
- |-
--index-cache.config="config":
"addresses": []
"dns_provider_update_interval": "3s"
"max_async_buffer_size": 0
"max_async_concurrency": 0
"max_get_multi_batch_size": 0
"max_get_multi_concurrency": 0
"max_idle_connections": 0
"max_item_size": "1MiB"
"timeout": "0s"
"type": "MEMCACHED"
env:
- name: OBJSTORE_CONFIG
valueFrom:
secretKeyRef:
key: thanos.yaml
name: thanos-objectstorage
image: quay.io/thanos/thanos:v0.12.0-rc.0
livenessProbe:
failureThreshold: 8
httpGet:
path: /-/healthy
port: 10902
scheme: HTTP
periodSeconds: 30
name: thanos-store
ports:
- containerPort: 10901
name: grpc
- containerPort: 10902
name: http
readinessProbe:
failureThreshold: 20
httpGet:
path: /-/ready
port: 10902
scheme: HTTP
periodSeconds: 5
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/thanos/store
name: data
readOnly: false
terminationGracePeriodSeconds: 120
volumes: []
volumeClaimTemplates:
- metadata:
labels:
app.kubernetes.io/component: object-store-gateway
app.kubernetes.io/instance: thanos-store
app.kubernetes.io/name: thanos-store
name: data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
50 changes: 50 additions & 0 deletions jsonnet/kube-thanos/kube-thanos-store.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,56 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
},
},

withMemcachedIndexCache:: {
local ts = self,
config+:: {
memcached+: {
addresses: error 'must provide memcached addresses',
timeout: '0s',
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pracucci @pstibrany I'm trying to come up with sane default for memcached, could point me to the right direction? What works for you best?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It really depends on the scale you wanna address. I still think the defaults I've set into Thanos are OK for a small/medium scale:
https://github.com/thanos-io/thanos/blob/1a2edf19475b854181e5abaa4b6f1c9caf2a1547/pkg/cacheutil/memcached_client.go#L35-L44

If you wanna be a bit more aggressive you may consider:

timeout: '2s',
maxIdleConnections: 1000,
maxAsyncConcurrency: 100,
maxAsyncBufferSize: 100000,
maxGetMultiConcurrency: 900,
maxGetMultiBatchSize: 1000,

I've also seen you set dnsProviderUpdateInterval: '3s'. I'm not sure there's a real value of being that aggressive here. The default 10s could be enough (but maybe you find a good reason to lower it).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks a lot for the response, this was really helpful.

maxIdleConnections: 0,
maxAsyncConcurrency: 0,
maxAsyncBufferSize: 0,
maxItemSize: '1MiB',
maxGetMultiConcurrency: 0,
maxGetMultiBatchSize: 0,
dnsProviderUpdateInterval: '3s',
},
},
local m = ts.config.memcached,
local cfg =
{
type: 'MEMCACHED',
config: {
addresses: m.addresses,
timeout: m.timeout,
max_idle_connections: m.maxIdleConnections,
max_async_concurrency: m.maxAsyncConcurrency,
max_async_buffer_size: m.maxGetMultiBatchSize,
max_item_size: m.maxItemSize,
max_get_multi_concurrency: m.maxGetMultiConcurrency,
max_get_multi_batch_size: m.maxGetMultiBatchSize,
dns_provider_update_interval: m.dnsProviderUpdateInterval,
},
},
statefulSet+: {
spec+: {
template+: {
spec+: {
containers: [
if c.name == 'thanos-store' then c {
args+: [
'--experimental.enable-index-cache-postings-compression',
'--index-cache.config=' + std.manifestYamlDoc(cfg),
],
} else c
for c in super.containers
],
},
},
},
},
},

withServiceMonitor:: {
local ts = self,
serviceMonitor: {
Expand Down