Skip to content

Commit

Permalink
reduce data size; add grafana panel
Browse files Browse the repository at this point in the history 
Signed-off-by: Ping Yu <[email protected]>
  • Loading branch information
@pingyu
pingyu committed Feb 13, 2024
1 parent 9430c95 commit 25de855
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 18 deletions.
64 changes: 48 additions & 16 deletions cdc/metrics/grafana/tikv-cdc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10371,7 +10371,7 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "Percentiles of TiKV sink write duration by each server.\n",
"description": "Percentiles of $sink_type sink write duration by each server.\n",
"fieldConfig": {
"defaults": {
"links": []
Expand Down Expand Up @@ -10419,7 +10419,7 @@
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (le,instance, changefeed))",
"expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (le,instance, changefeed))",
"format": "time_series",
"hide": false,
"interval": "",
Expand All @@ -10429,7 +10429,7 @@
},
{
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (le,instance, changefeed))",
"expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (le,instance, changefeed))",
"format": "time_series",
"hide": false,
"interval": "",
Expand All @@ -10439,7 +10439,7 @@
},
{
"exemplar": true,
"expr": "histogram_quantile(0.999, sum(rate(tikv_cdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (le,instance, changefeed))",
"expr": "histogram_quantile(0.999, sum(rate(tikv_cdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (le,instance, changefeed))",
"format": "time_series",
"hide": false,
"interval": "",
Expand All @@ -10452,7 +10452,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "TiKV sink write duration percentile",
"title": "$sink_type sink write duration percentile",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -10543,7 +10543,7 @@
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(0.90, sum(rate(tikv_cdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (le, instance, changefeed))",
"expr": "histogram_quantile(0.90, sum(rate(tikv_cdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (le, instance, changefeed))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
Expand All @@ -10552,7 +10552,7 @@
},
{
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (le, instance, changefeed))",
"expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (le, instance, changefeed))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
Expand All @@ -10561,7 +10561,7 @@
},
{
"exemplar": true,
"expr": "histogram_quantile(0.999, sum(rate(tikv_cdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (le, instance, changefeed))",
"expr": "histogram_quantile(0.999, sum(rate(tikv_cdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (le, instance, changefeed))",
"format": "time_series",
"hide": true,
"interval": "",
Expand All @@ -10574,7 +10574,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "TiKV sink write batch size percentile",
"title": "$sink_type sink write batch size percentile",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -10617,7 +10617,7 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "The number of changed entires that are written to downstream TiKV",
"description": "The number of changed entires that are written to downstream sink",
"fieldConfig": {
"defaults": {
"links": []
Expand Down Expand Up @@ -10664,7 +10664,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum (rate(tikv_cdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (instance)",
"expr": "sum (rate(tikv_cdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (instance)",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
Expand All @@ -10673,7 +10673,7 @@
},
{
"exemplar": true,
"expr": "sum (rate(tikv_cdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (changefeed)",
"expr": "sum (rate(tikv_cdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (changefeed)",
"format": "time_series",
"hide": false,
"interval": "",
Expand All @@ -10686,7 +10686,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "TiKV sink write rows count/s",
"title": "$sink_type sink write rows count/s",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -10773,7 +10773,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(delta(tikv_cdc_sink_execution_error{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"TiKV\"}[1m])) by (capture)",
"expr": "sum(delta(tikv_cdc_sink_execution_error{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\", type=\"$sink_type\"}[1m])) by (capture)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{capture}}",
Expand All @@ -10784,7 +10784,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "TiKV sink error count/m",
"title": "$sink_type sink error count/m",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -10822,7 +10822,8 @@
}
}
],
"title": "TiKV Sink",
"repeat": "sink_type",
"title": "$sink_type Sink",
"type": "row"
}
],
Expand Down Expand Up @@ -10957,6 +10958,37 @@
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": "${DS_TEST-CLUSTER}",
"definition": "label_values(tikv_cdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\"}, type)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "Sink",
"multi": false,
"name": "sink_type",
"options": [],
"query": {
"query": "label_values(tikv_cdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\"}, type)",
"refId": "${DS_TEST-CLUSTER}-sink_type-Variable-Query"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": "9999999999",
"current": {
Expand Down
2 changes: 1 addition & 1 deletion cdc/tests/integration_tests/flow_control/config/workload
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
# Default data size: 1 KB records (10 fields, 100 bytes each, plus key)
# Request distribution: zipfian

recordcount=10000000
recordcount=5000000
workload=core

readallfields=true
Expand Down
2 changes: 1 addition & 1 deletion cdc/tests/integration_tests/flow_control/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ function run() {
cd $WORK_DIR

start_ts=$(get_start_ts $UP_PD)
go-ycsb load tikv -P $CUR/config/workload -p tikv.pd="$UP_PD" -p tikv.type="raw" -p tikv.apiversion=V2 --threads 400 # About 1G
go-ycsb load tikv -P $CUR/config/workload -p tikv.pd="$UP_PD" -p tikv.type="raw" -p tikv.apiversion=V2 --threads 400 # About 500MiB

cat - >"$WORK_DIR/tikv-cdc-config.toml" <<EOF
per-changefeed-memory-quota=10485760 #10M
Expand Down

0 comments on commit 25de855

Please sign in to comment.