From 7488e6c827347ed0ca616ddbe0ec1c54c4d00eb6 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Fri, 19 Jul 2019 14:42:04 +0800 Subject: [PATCH] scripts: add some metrics for tidb (#830) --- scripts/tidb.json | 5603 +++++++++++++++++++++++++++++++++------------ 1 file changed, 4082 insertions(+), 1521 deletions(-) diff --git a/scripts/tidb.json b/scripts/tidb.json index 5bf455980..f89674e6c 100644 --- a/scripts/tidb.json +++ b/scripts/tidb.json @@ -44,7 +44,7 @@ }, "editable": true, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [ { @@ -85,10 +85,14 @@ }, "id": 80, "legend": { + "alignAsTable": false, "avg": false, "current": false, + "hideEmpty": false, + "hideZero": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -1614,6 +1618,7 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", @@ -1627,7 +1632,7 @@ "x": 0, "y": 3 }, - "id": 8, + "id": 184, "legend": { "alignAsTable": true, "avg": false, @@ -1644,6 +1649,7 @@ "links": [], "nullPointMode": "null as zero", "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 5, "points": false, "renderer": "flot", @@ -1655,31 +1661,24 @@ } ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": true, "targets": [ { - "expr": "tidb_server_connections", + "expr": "(time() - process_start_time_seconds{job=\"tidb\"})", "format": "time_series", + "hide": false, "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 40 - }, - { - "expr": "sum(tidb_server_connections)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "total", - "refId": "B", - "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Connection Count", + "title": "Uptime", "tooltip": { "msResolution": false, "shared": true, @@ -1696,7 +1695,7 @@ }, "yaxes": [ { - "format": "short", + "format": "dtdurations", "label": null, "logBase": 1, "max": null, @@ -1762,21 +1761,26 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{job=~\"tidb.*\"}", + "expr": "process_resident_memory_bytes{job=\"tidb\"}", "format": "time_series", "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "go_memstats_heap_inuse_bytes", - "refId": "B", - "step": 40 + "intervalFactor": 1, + "legendFormat": "process-{{instance}}", + "refId": "A" + }, + { + "expr": "go_memstats_heap_alloc_bytes{job=\"tidb\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "heap-{{instance}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Heap Memory Usage", + "title": "Memory Usage", "tooltip": { "msResolution": true, "shared": true, @@ -1820,19 +1824,22 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 }, - "id": 61, + "id": 168, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, @@ -1842,30 +1849,39 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": " go_goroutines{job=~\"tidb.*\"}", + "expr": "rate(process_cpu_seconds_total{job=\"tidb\"}[1m])", "format": "time_series", + "hide": false, "intervalFactor": 2, "legendFormat": "{{instance}}", - "refId": "A" + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Goroutine Count", + "title": "CPU Usage", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1880,11 +1896,11 @@ }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -1907,53 +1923,71 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB Server events.", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 10 }, - "id": 49, + "id": 8, "legend": { "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, - "stack": false, - "steppedLine": false, + "stack": true, + "steppedLine": true, "targets": [ { - "expr": "increase(tidb_server_event_total[10m])", + "expr": "tidb_server_connections", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "refId": "A" + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(tidb_server_connections)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Events OPM", + "title": "Connection Count", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1972,7 +2006,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -2002,7 +2036,7 @@ "x": 0, "y": 17 }, - "id": 54, + "id": 188, "legend": { "alignAsTable": true, "avg": false, @@ -2028,27 +2062,18 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tidb_server_panic_total[1m])", + "expr": "process_open_fds{job=\"tidb\"}", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "panic-{{instance}}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "expr": "increase(tidb_server_critical_error_total[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "critical-{{instance}}", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Uncommon Error OPM", + "title": "Open FD Count", "tooltip": { "shared": true, "sort": 0, @@ -2098,17 +2123,17 @@ "x": 12, "y": 17 }, - "id": 82, + "id": 61, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -2124,7 +2149,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_monitor_keep_alive_total[1m])) by (instance)", + "expr": " go_goroutines{job=~\"tidb.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2135,7 +2160,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Keep Alive OPM", + "title": "Goroutine Count", "tooltip": { "shared": true, "sort": 0, @@ -2178,51 +2203,64 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration (us) for getting token, it should be small until concurrency limit is reached.", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 24 }, - "id": 111, + "id": 183, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket[1m])) by (le))", + "expr": "rate(go_gc_duration_seconds_sum{job=\"tidb\"}[1m])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99", - "refId": "A" + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Get Token Duration", + "title": "Go GC Duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2237,11 +2275,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "dtdurations", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -2257,39 +2295,21 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Server", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 141, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 4 + "x": 12, + "y": 24 }, - "id": 69, + "id": 186, "legend": { "alignAsTable": true, "avg": false, @@ -2304,7 +2324,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2315,21 +2335,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_transaction_total[1m])) by (type, sql_type)", + "expr": " go_threads{job=\"tidb\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}-{{sql_type}}", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Transaction OPS", + "title": "Go Threads", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2348,7 +2366,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -2375,20 +2393,20 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 4 + "x": 0, + "y": 31 }, - "id": 72, + "id": 185, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -2404,32 +2422,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))", + "expr": " rate(go_gc_duration_seconds_count{job=\"tidb\"}[1m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "99-{{sql_type}}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95-{{sql_type}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "80-{{sql_type}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Duration", + "title": "Go GC Count", "tooltip": { "shared": true, "sort": 0, @@ -2445,9 +2449,9 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -2472,59 +2476,65 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The fraction of this program's available CPU time used by the GC since the program started.", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 0, - "y": 11 + "w": 12, + "x": 12, + "y": 31 }, - "id": 74, + "id": 187, "legend": { "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99-{{sql_type}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))", + "expr": "go_memstats_gc_cpu_fraction{job=\"tidb\"}", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "80-{{sql_type}}", - "refId": "B" + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Transaction Statement Num", + "title": "Go GC CPU Usage", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2539,11 +2549,11 @@ }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -2566,22 +2576,22 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, + "description": "TiDB Server events.", "fill": 1, - "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 11 + "w": 12, + "x": 0, + "y": 38 }, - "id": 36, + "id": 49, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2589,7 +2599,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2600,21 +2610,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_retry_error_total[30s])) by (type, sql_type)", + "expr": "increase(tidb_server_event_total[10m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}-{{sql_type}}", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}}-server {{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Session Retry Error OPS", + "title": "Events OPM", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2633,7 +2641,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -2656,17 +2664,14 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 11 + "w": 12, + "x": 12, + "y": 38 }, - "id": 67, + "id": 82, "legend": { "alignAsTable": true, "avg": false, @@ -2681,7 +2686,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2692,35 +2697,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "100", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))", + "expr": "sum(increase(tidb_monitor_keep_alive_total[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "90", - "refId": "C" + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Transaction Retry Num", + "title": "Keep Alive OPM", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -2739,7 +2728,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -2755,52 +2744,32 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Transaction", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 142, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 5 + "y": 45 }, - "id": 76, + "id": 165, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": null, - "sortDesc": null, "total": false, "values": true }, @@ -2812,30 +2781,44 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "sum(rate(tidb_executor_expensive_total[1m])) by (type)", + "expr": "tidb_server_prepared_stmts", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{instance}}", "refId": "A", - "step": 30 + "step": 40 + }, + { + "expr": "sum(tidb_server_prepared_stmts)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Expensive Executors OPS", + "title": "Prepare Statement Count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -2849,7 +2832,7 @@ { "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": "0", "show": true @@ -2874,37 +2857,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 5 + "y": 45 }, - "id": 91, + "id": 166, "legend": { "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "avg": false, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": null, - "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2915,24 +2890,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_plan_cache_total[1m])) by (type)", + "expr": "sum(increase(tidb_monitor_time_jump_back_total[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 30 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Queries Using Plan Cache OPS", + "title": "Time Jump Back OPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -2946,9 +2919,9 @@ { "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -2965,240 +2938,20 @@ "alignLevel": null } }, - { - "type": "graph", - "title": "Compile Duration", - "gridPos": { - "x": 0, - "y": 13, - "w": 12, - "h": 8 - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 154, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket[1m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 30, - "instant": false - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "decimals": null - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - }, - "description": "The time cost of building the query plan", - "interval": "" - }, - { - "type": "graph", - "title": "Parse Duration", - "gridPos": { - "x": 12, - "y": 13, - "w": 12, - "h": 8 - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 156, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket[1m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 30, - "instant": false - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "decimals": null - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - }, - "description": "The time cost of parsing SQL to AST", - "interval": "" - } - ], - "repeat": null, - "title": "Executor", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 143, - "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 6 + "y": 52 }, - "id": 12, + "id": 54, "legend": { "alignAsTable": true, "avg": false, @@ -3211,13 +2964,9 @@ "values": false }, "lines": true, - "linewidth": 2, - "links": [ - { - "type": "dashboard" - } - ], - "nullPointMode": "null as zero", + "linewidth": 1, + "links": [], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -3228,48 +2977,31 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "999-{{type}}", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", + "expr": "increase(tidb_server_panic_total[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "99-{{type}}", - "metric": "tidb_distsql_handle_query_duration_seconds_bucket", - "refId": "A", - "step": 4 + "legendFormat": "panic-{{instance}}", + "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", + "expr": "increase(tidb_server_critical_error_total[1m])", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "90-{{type}}", + "legendFormat": "critical-{{instance}}", "refId": "B" - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "50-{{type}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Distsql Duration", + "title": "Uncommon Error OPM", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -3281,11 +3013,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": "0.0005", + "min": null, "show": true }, { @@ -3308,31 +3040,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 6 + "y": 52 }, - "id": 14, + "id": 167, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, - "show": false, + "rightSide": true, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -3343,25 +3073,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_distsql_handle_query_duration_seconds_count[1m]))", + "expr": "sum(increase(tidb_server_handshake_error_total[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "metric": "tidb_distsql_query_total", - "refId": "A", - "step": 4 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Distsql QPS", + "title": "Handshake Error OPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -3377,7 +3104,7 @@ "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -3400,31 +3127,28 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, + "description": "Duration (us) for getting token, it should be small until concurrency limit is reached.", "fill": 1, - "grid": {}, "gridPos": { "h": 7, - "w": 8, + "w": 12, "x": 0, - "y": 13 + "y": 59 }, - "id": 60, + "id": 111, "legend": { "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -3435,25 +3159,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_distsql_scan_keys_partial_num_count[1m]))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket[1m])) by (le))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "tidb_distsql_query_total", - "refId": "A", - "step": 4 + "intervalFactor": 1, + "legendFormat": "99", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Distsql Partial QPS", + "title": "Get Token Duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -3465,11 +3186,11 @@ }, "yaxes": [ { - "format": "short", + "format": "µs", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -3485,34 +3206,54 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 141, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 13 + "x": 0, + "y": 82 }, - "id": 57, + "id": 69, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -3523,33 +3264,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "100", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))", + "expr": "sum(rate(tidb_session_transaction_total[1m])) by (type, sql_type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "50", - "refId": "C" + "legendFormat": "{{type}}-{{sql_type}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Scan Keys Num", + "title": "Transaction OPS", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -3566,9 +3295,9 @@ { "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -3595,15 +3324,17 @@ "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 13 + "x": 8, + "y": 82 }, - "id": 58, + "id": 72, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -3622,24 +3353,24 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "100", + "legendFormat": "99-{{sql_type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "90", + "legendFormat": "95-{{sql_type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "50", + "legendFormat": "80-{{sql_type}}", "refId": "C" } ], @@ -3647,7 +3378,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Scan Keys Partial Num", + "title": "Duration", "tooltip": { "shared": true, "sort": 0, @@ -3663,7 +3394,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 2, "max": null, @@ -3690,19 +3421,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 20 + "x": 16, + "y": 82 }, - "id": 59, + "id": 67, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -3710,7 +3446,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -3721,24 +3457,25 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))", + "expr": "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "100", - "refId": "A" + "refId": "A", + "step": 10 }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "90", + "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "50", + "legendFormat": "90", "refId": "C" } ], @@ -3746,8 +3483,9 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Partial Num", + "title": "Transaction Retry Num", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -3764,9 +3502,9 @@ { "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -3789,34 +3527,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 20 + "x": 0, + "y": 89 }, - "id": 41, + "id": 74, "legend": { "alignAsTable": true, "avg": false, "current": false, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "max", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -3827,24 +3560,29 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_duration_seconds_bucket[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 + "legendFormat": "99-{{sql_type}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "80-{{sql_type}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Coprocessor Seconds 999", + "title": "Transaction Statement Num", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -3856,11 +3594,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -3876,22 +3614,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Distsql", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 144, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -3904,26 +3627,22 @@ "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 7 + "w": 8, + "x": 8, + "y": 89 }, - "id": 6, + "id": 36, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "max": true, + "max": false, "min": false, - "rightSide": false, "show": true, - "sort": null, - "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -3936,25 +3655,113 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))", + "expr": "sum(rate(tidb_session_retry_error_total[30s])) by (type, sql_type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "9999", + "legendFormat": "{{type}}-{{sql_type}}", "refId": "A", - "step": 40 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "80", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Session Retry Error OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 89 + }, + "id": 175, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_local_latch_wait_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_local_latch_wait_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_local_latch_wait_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "80", "refId": "C" } ], @@ -3962,7 +3769,121 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Retry Duration", + "title": "Local Latch Wait Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "title": "Transaction", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 142, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "description": "The time cost of parsing SQL to AST", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 156, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket[1m])) by (le, sql_type))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{sql_type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Parse Duration", "tooltip": { "msResolution": false, "shared": true, @@ -3979,11 +3900,2685 @@ }, "yaxes": [ { - "format": "s", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "description": "The time cost of building the query plan", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 97 + }, + "id": 154, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket[1m])) by (le, sql_type))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{sql_type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compile Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "description": "The time cost of executing the SQL which does not include the time to get the results of the query .", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 169, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket[1m])) by (le, sql_type))", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{sql_type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Execution Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 105 + }, + "id": 76, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_executor_expensive_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expensive Executors OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 91, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_server_plan_cache_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queries Using Plan Cache OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 2, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "title": "Executor", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 143, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "type": "dashboard" + } + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "999-{{type}}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "99-{{type}}", + "metric": "tidb_distsql_handle_query_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90-{{type}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "50-{{type}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Distsql Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": "0.0005", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 122 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_distsql_handle_query_duration_seconds_count[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "tidb_distsql_query_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Distsql QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 129 + }, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_distsql_scan_keys_partial_num_count[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "tidb_distsql_query_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Distsql Partial QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 129 + }, + "id": 57, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan Keys Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 129 + }, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan Keys Partial Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 136 + }, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Partial Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 136 + }, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_duration_seconds_bucket[1m])) by (le, instance))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Coprocessor Seconds 999", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "title": "Distsql", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 144, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "9999", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "80", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Retry Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tidb_server_session_execute_parse_duration_count", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(tidb_tikvclient_region_err_total{type=\"server_is_busy\"}[1m]))", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "sum", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TiClient Region Error OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_backoff_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Backoff OPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tidb_tikvclient_lock_resolver_actions_total", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lock Resolve OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_lock_cleanup_task_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "cleanup_secondary_failure_{{type}}", + "metric": "tidb_tikvclient_lock_resolver_actions_total", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{type=\"fail\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load_safepoint_failure", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Other Errors OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "title": "KV Errors", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 145, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 166 + }, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\"GC\"}[1m])) by (le, store))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "store-{{store}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Request Duration 999 by store", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 166 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\"GC\"}[1m])) by (le,type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Request Duration 999 by type", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 173 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Cmd Duration 9999", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 173 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Cmd Duration 99", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "repeat": null, + "title": "KV Duration", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 146, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 181 + }, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bytes_bucket[1m])) by (le, instance))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Txn Write Size Bytes 100", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 181 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_txn_total[1m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Txn OPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 181 + }, + "id": 171, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_snapshot_total[1m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Snapshot OPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 188 + }, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{type=\"ok\"}[1m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Safepoint OPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 188 + }, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_num_bucket[1m])) by (le, instance))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100 {{instance}}", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Txn Write KV Num 100", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 188 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_tikvclient_txn_regions_num_bucket[1m])) by (le, instance))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Txn Regions Num 90", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 195 + }, + "id": 172, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_txn_cmd_total[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{type}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "KV Cmd OPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -3999,40 +6594,52 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "KV Count", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 147, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, "editable": true, "error": false, - "fill": 0, + "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 7 + "w": 8, + "x": 0, + "y": 10 }, - "id": 11, + "id": 20, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -4045,28 +6652,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type)", + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type!=\"tso\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", - "metric": "tidb_server_session_execute_parse_duration_count", "refId": "A", - "step": 40 - }, - { - "expr": "sum(rate(tidb_tikvclient_region_err_total{type=\"server_is_busy\"}[1m]))", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "sum", - "refId": "B" + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiClient Region Error OPS", + "title": "PD Client CMD OPS", "tooltip": { "msResolution": false, "shared": true, @@ -4087,7 +6685,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -4110,29 +6708,31 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 14 + "w": 8, + "x": 8, + "y": 10 }, - "id": 53, + "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "total": true, - "values": true + "total": false, + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -4145,22 +6745,38 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_backoff_total[1m])) by (type)", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!=\"tso\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" + "legendFormat": "999-{{type}}", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!=\"tso\"}[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99-{{type}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!=\"tso\"}[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90-{{type}}", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Backoff OPS", + "title": "PD Client CMD Duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4172,7 +6788,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -4205,23 +6821,21 @@ "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 14 + "w": 8, + "x": 16, + "y": 10 }, - "id": 32, + "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 2, @@ -4237,20 +6851,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)", + "expr": "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", - "metric": "tidb_tikvclient_lock_resolver_actions_total", "refId": "A", - "step": 40 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Lock Resolve OPS", + "title": "PD Client CMD Fail OPS", "tooltip": { "msResolution": false, "shared": true, @@ -4294,32 +6907,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of a client calling GetTSAsync until received the TS result.", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, + "w": 8, "x": 0, - "y": 21 + "y": 17 }, - "id": 84, + "id": 79, "legend": { - "alignAsTable": true, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, - "max": true, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -4332,27 +6942,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_lock_cleanup_task_total[1m])) by (type)", + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type=\"tso\"}[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "cleanup_secondary_failure_{{type}}", - "metric": "tidb_tikvclient_lock_resolver_actions_total", - "refId": "A", - "step": 40 + "legendFormat": "cmd", + "refId": "C" }, { - "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{type=\"fail\"}[1m]))", + "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{type=\"tso\"}[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "load_safepoint_failure", - "refId": "B" + "legendFormat": "request", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Other Errors OPS", + "title": "PD TSO OPS", "tooltip": { "msResolution": false, "shared": true, @@ -4369,11 +6977,11 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -4389,51 +6997,33 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "KV Errors", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 145, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of a client calling GetTSAsync until received the TS result.", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 8 + "w": 8, + "x": 8, + "y": 17 }, - "id": 48, + "id": 77, "legend": { - "alignAsTable": true, "avg": false, "current": false, - "max": true, + "max": false, "min": false, - "rightSide": true, "show": true, - "sort": "max", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, @@ -4449,24 +7039,38 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\"GC\"}[1m])) by (le, store))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "store-{{store}}", + "legendFormat": "999", "refId": "A", - "step": 40 + "step": 10 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Request Duration 999 by store", + "title": "PD TSO Wait Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4505,29 +7109,26 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of a client sending TSO request until received the response.", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 8 + "w": 8, + "x": 16, + "y": 17 }, - "id": 30, + "id": 78, "legend": { - "alignAsTable": true, "avg": false, "current": false, - "max": true, + "max": false, "min": false, - "rightSide": true, "show": true, - "sort": "max", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, @@ -4543,24 +7144,38 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\"GC\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "999", "refId": "A", - "step": 40 + "step": 10 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Request Duration 999 by type", + "title": "PD TSO RPC Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4599,30 +7214,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of the waiting time for getting the start timestamp oracle", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, - "w": 12, + "w": 8, "x": 0, - "y": 15 + "y": 24 }, - "id": 22, + "id": 159, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -4635,19 +7249,33 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "999", "refId": "A", - "step": 40 + "step": 10 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Cmd Duration 9999", + "title": "Start TSO Wait Duration", "tooltip": { "msResolution": false, "shared": true, @@ -4684,7 +7312,22 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "PD Client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 148, + "panels": [ { "aliasColors": {}, "bars": false, @@ -4698,22 +7341,20 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 15 + "x": 0, + "y": 47 }, - "id": 18, + "id": 27, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, @@ -4729,24 +7370,25 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{instance}}", + "metric": "", "refId": "A", - "step": 40 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Cmd Duration 99", + "title": "Load Schema Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -4762,7 +7404,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -4778,22 +7420,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "KV Duration", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 146, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -4802,62 +7429,64 @@ "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 0, - "y": 9 + "w": 12, + "x": 12, + "y": 47 }, - "id": 5, + "id": 28, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "max", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*failed/", + "bars": true + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count[1m])) by (type)", + "expr": "sum(rate(tidb_domain_load_schema_total[1m])) by (instance,type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{instance}}-{{type}}", + "metric": "tidb_domain_load_schema_duration_count", "refId": "A", - "step": 40 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Cmd OPS", + "title": "Load Schema OPS", "tooltip": { - "msResolution": true, + "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -4871,7 +7500,7 @@ { "format": "short", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -4896,30 +7525,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 9 + "w": 12, + "x": 0, + "y": 54 }, - "id": 4, + "id": 29, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -4932,24 +7564,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_txn_total[1m])) by (instance)", + "expr": "sum(increase(tidb_session_schema_lease_error_total[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", + "metric": "tidb_server_", "refId": "A", - "step": 40 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Txn OPS", + "title": "Schema Lease Error OPM", "tooltip": { - "msResolution": true, + "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -4990,58 +7623,64 @@ "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 9 + "w": 12, + "x": 12, + "y": 54 }, - "id": 44, + "id": 157, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*failed/", + "bars": true + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.90, sum(rate(tidb_tikvclient_txn_regions_num_bucket[1m])) by (le, instance))", + "expr": "sum(rate(tidb_domain_load_privilege_total[1m])) by (instance,type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}-{{type}}", + "metric": "tidb_domain_load_schema_duration_count", "refId": "A", - "step": 40 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Txn Regions Num 90", + "title": "Load Privilege OPS", "tooltip": { - "msResolution": true, + "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5055,9 +7694,9 @@ { "format": "short", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -5073,7 +7712,22 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "Schema Load", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 149, + "panels": [ { "aliasColors": {}, "bars": false, @@ -5088,21 +7742,19 @@ "h": 7, "w": 8, "x": 0, - "y": 16 + "y": 111 }, - "id": 34, + "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "avg", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 2, @@ -5118,23 +7770,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bytes_bucket[1m])) by (le, instance))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "{{type}}", "refId": "A", - "step": 40 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Txn Write Size Bytes 100", + "title": "DDL Duration 95", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, + "sort": 0, "value_type": "cumulative" }, "type": "graph", @@ -5147,11 +7799,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5160,7 +7812,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -5174,34 +7826,31 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, - "y": 16 + "y": 111 }, - "id": 33, + "id": 63, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, - "sort": "avg", - "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5212,24 +7861,22 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_num_bucket[1m])) by (le, instance))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "100 {{instance}}", - "refId": "B", - "step": 40 + "legendFormat": "{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Txn Write KV Num 100", + "title": "Batch Add Index Duration 100", "tooltip": { - "msResolution": false, "shared": true, - "sort": 2, - "value_type": "cumulative" + "sort": 0, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5241,11 +7888,11 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5268,34 +7915,31 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 16 + "y": 111 }, - "id": 83, + "id": 62, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": false, - "max": true, + "hideEmpty": true, + "hideZero": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "avg", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5306,24 +7950,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{type=\"ok\"}[1m]))", + "expr": "tidb_ddl_waiting_jobs", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 40 + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Load Safepoint OPS", + "title": "DDL Waiting Jobs Count", "tooltip": { - "msResolution": false, "shared": true, - "sort": 2, - "value_type": "cumulative" + "sort": 0, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5335,11 +7977,11 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5355,39 +7997,21 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "KV Count", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 147, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, - "w": 8, + "w": 12, "x": 0, - "y": 10 + "y": 118 }, - "id": 20, + "id": 55, "legend": { "alignAsTable": true, "avg": false, @@ -5400,9 +8024,9 @@ "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5413,24 +8037,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type!=\"tso\"}[1m])) by (type)", + "expr": "increase(tidb_ddl_worker_operation_total[1m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "PD Client CMD OPS", + "title": "DDL META OPM", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5469,22 +8091,20 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 10 + "w": 12, + "x": 12, + "y": 118 }, - "id": 35, + "id": 56, "legend": { "alignAsTable": true, "avg": false, "current": false, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, "rightSide": true, @@ -5495,7 +8115,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5506,38 +8126,22 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!=\"tso\"}[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "999-{{type}}", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!=\"tso\"}[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99-{{type}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!=\"tso\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket[1m])) by (le, type, action, result))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "90-{{type}}", - "refId": "C" + "legendFormat": "{{type}}-{{action}}-{{result}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "PD Client CMD Duration", + "title": "DDL Worker Duration 99", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5553,7 +8157,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5576,17 +8180,14 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 10 + "x": 0, + "y": 125 }, - "id": 43, + "id": 64, "legend": { "alignAsTable": true, "avg": false, @@ -5599,9 +8200,9 @@ "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5612,24 +8213,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count[1m])) by (type)", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket[2m])) by (le, type, result))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 + "legendFormat": "{{type}}-{{result}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "PD Client CMD Fail OPS", + "title": "Deploy Syncer Duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5645,7 +8244,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -5668,23 +8267,21 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The duration of a client calling GetTSAsync until received the TS result.", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 17 + "x": 8, + "y": 125 }, - "id": 79, + "id": 65, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -5692,7 +8289,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5703,17 +8300,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type=\"tso\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "cmd", - "refId": "C" - }, - { - "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{type=\"tso\"}[1m]))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket[2m])) by (le, type, result))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "request", + "legendFormat": "{{type}}-{{result}}", "refId": "A" } ], @@ -5721,12 +8311,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "PD TSO OPS", + "title": "Owner Handle Syncer Duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5738,7 +8327,7 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -5765,23 +8354,21 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The duration of a client calling GetTSAsync until received the TS result.", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 17 + "x": 16, + "y": 125 }, - "id": 77, + "id": 66, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -5789,7 +8376,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5800,38 +8387,22 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "999", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket[2m])) by (le, result))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "90", - "refId": "C" + "legendFormat": "{{result}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "PD TSO Wait Duration", + "title": "Update Self Version Duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5863,25 +8434,36 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "DDL", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 150, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The duration of a client sending TSO request until received the response.", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 17 + "x": 0, + "y": 69 }, - "id": 78, + "id": 46, "legend": { "avg": false, "current": false, @@ -5894,7 +8476,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5905,38 +8487,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "999", + "legendFormat": "auto analyze duration", "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "90", - "refId": "C" + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "PD TSO RPC Duration", + "title": "Auto Analyze Duration 95", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5948,7 +8515,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -5968,46 +8535,26 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "PD Client", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 148, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 11 + "x": 8, + "y": 69 }, - "id": 27, + "id": 47, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6015,7 +8562,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6026,22 +8573,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket[1m])) by (le, instance))", + "expr": "sum(rate(tidb_statistics_auto_analyze_total[1m])) by (type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", + "legendFormat": "{{type}}", "refId": "A", - "step": 10 + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Load Schema Duration", + "title": "Auto Analyze QPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6056,11 +8601,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6083,24 +8628,19 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, + "fill": 1, "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 11 + "x": 16, + "y": 69 }, - "id": 28, + "id": 70, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6108,38 +8648,45 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*failed/", - "bars": true - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_domain_load_schema_total[1m])) by (instance,type)", + "expr": "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "metric": "tidb_domain_load_schema_duration_count", + "legendFormat": "99", "refId": "A", - "step": 10 + "step": 30 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Load Schema OPS", + "title": "Stats Inaccuracy Rate", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6156,9 +8703,9 @@ { "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6181,27 +8728,19 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, + "fill": 1, "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 11 + "x": 0, + "y": 76 }, - "id": 29, + "id": 71, "legend": { - "alignAsTable": true, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6209,7 +8748,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6220,22 +8759,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_session_schema_lease_error_total[1m])) by (instance)", + "expr": "sum(rate(tidb_statistics_pseudo_estimation_total[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tidb_server_", + "legendFormat": "Pseudo OPS", "refId": "A", - "step": 10 + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Schema Lease Error OPM", + "title": "Pseudo Estimation OPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6254,7 +8791,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6270,54 +8807,34 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Schema Load", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 149, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 12 + "x": 8, + "y": 76 }, - "id": 9, + "id": 92, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6328,24 +8845,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket[1m])) by (le, type))", + "expr": "sum(rate(tidb_statistics_dump_feedback_total[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", - "step": 10 + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "DDL Duration 95", + "title": "Dump Feedback OPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -6357,11 +8873,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6388,22 +8904,18 @@ "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 12 + "x": 16, + "y": 76 }, - "id": 63, + "id": 170, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, @@ -6419,18 +8931,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket[1m])) by (le, type))", + "expr": "sum(rate(tidb_statistics_store_query_feedback_total[1m])) by (type) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", - "refId": "A" + "refId": "A", + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Batch Add Index Duration 100", + "title": "Store Query Feedback QPS", "tooltip": { "shared": true, "sort": 0, @@ -6446,11 +8959,11 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6473,23 +8986,20 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "Counter of query feedback whose actual count is much different than calculated by current statistics", "fill": 1, "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 12 + "x": 0, + "y": 83 }, - "id": 62, + "id": 113, "legend": { - "alignAsTable": true, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6508,10 +9018,11 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_ddl_waiting_jobs", + "expr": "sum(rate(tidb_statistics_high_error_rate_feedback_total[1m]))", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "Significant Feedback", "refId": "A" } ], @@ -6519,7 +9030,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "DDL Waiting Jobs Count", + "title": "Significant Feedback", "tooltip": { "shared": true, "sort": 0, @@ -6535,11 +9046,11 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6565,18 +9076,16 @@ "fill": 1, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 19 + "w": 8, + "x": 8, + "y": 83 }, - "id": 55, + "id": 93, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6595,18 +9104,19 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tidb_ddl_worker_operation_total[1m])", + "expr": "sum(rate(tidb_statistics_update_stats_total[1m])) by (type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "refId": "A" + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "DDL META OPM", + "title": "Update Stats OPS", "tooltip": { "shared": true, "sort": 0, @@ -6649,31 +9159,34 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 19 + "w": 8, + "x": 16, + "y": 83 }, - "id": 56, + "id": 173, "legend": { "alignAsTable": true, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sort": "avg", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -6684,22 +9197,24 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket[1m])) by (le, type, action, result))", + "expr": "histogram_quantile(1, sum(rate(tidb_statistics_fast_analyze_status_bucket[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}-{{action}}-{{result}}", - "refId": "A" + "legendFormat": "{{type}}", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "DDL Worker Duration 99", + "title": "Fast Analyze Status 100", "tooltip": { + "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 2, + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -6711,11 +9226,11 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6724,43 +9239,61 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "Statistics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 161, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 26 + "y": 70 }, - "id": 64, + "id": 162, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -6771,22 +9304,24 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket[2m])) by (le, type, result))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket[1m])) by (le, instance, result))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}-{{result}}", - "refId": "A" + "legendFormat": "{{instance}}-{{result}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Deploy Syncer Duration", + "title": "New ETCD Session Duration 95", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -6798,11 +9333,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -6830,16 +9365,14 @@ "h": 7, "w": 8, "x": 8, - "y": 26 + "y": 70 }, - "id": 65, + "id": 163, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6858,18 +9391,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket[2m])) by (le, type, result))", + "expr": "sum(rate(tidb_owner_watch_owner_total[1m])) by (type, result, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}-{{result}}", - "refId": "A" + "legendFormat": "{{type}}-{{result}}-{{instance}}", + "refId": "A", + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Owner Handle Syncer Duration", + "title": "Owner Watcher OPS", "tooltip": { "shared": true, "sort": 0, @@ -6885,7 +9419,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -6917,16 +9451,14 @@ "h": 7, "w": 8, "x": 16, - "y": 26 + "y": 70 }, - "id": 66, + "id": 174, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -6945,18 +9477,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket[2m])) by (le, result))", + "expr": "sum(rate(tidb_owner_watch_owner_total[1m])) by (type, result, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{result}}", - "refId": "A" + "legendFormat": "{{type}}-{{result}}-{{instance}}", + "refId": "A", + "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Update Self Version Duration", + "title": "Owner Watcher OPS", "tooltip": { "shared": true, "sort": 0, @@ -6972,7 +9505,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -6994,8 +9527,7 @@ } } ], - "repeat": null, - "title": "DDL", + "title": "Owner", "type": "row" }, { @@ -7004,9 +9536,9 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 14 }, - "id": 150, + "id": 151, "panels": [ { "aliasColors": {}, @@ -7017,16 +9549,18 @@ "fill": 1, "gridPos": { "h": 7, - "w": 8, + "w": 12, "x": 0, - "y": 13 + "y": 79 }, - "id": 46, + "id": 50, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -7045,19 +9579,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_bucket[1m])) by (le))", + "expr": "sum(rate(tidb_autoid_operation_duration_seconds_count[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "auto analyze duration", - "refId": "A", - "step": 30 + "legendFormat": "AutoID QPS", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Auto Analyze Duration 95", + "title": "AutoID QPS", "tooltip": { "shared": true, "sort": 0, @@ -7103,16 +9636,18 @@ "fill": 1, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 13 + "w": 12, + "x": 12, + "y": 79 }, - "id": 47, + "id": 51, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -7131,19 +9666,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_auto_analyze_total[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 30 + "legendFormat": "99-{{type}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "80-{{type}}", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Auto Analyze QPS", + "title": "AutoID Duration", "tooltip": { "shared": true, "sort": 0, @@ -7159,11 +9700,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": null, + "min": "0.001", "show": true }, { @@ -7189,11 +9730,11 @@ "fill": 1, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 13 + "w": 12, + "x": 0, + "y": 86 }, - "id": 70, + "id": 164, "legend": { "avg": false, "current": false, @@ -7217,33 +9758,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))", + "expr": "sum(rate(tidb_tikvclient_region_cache_operations_total{result=\"err\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "99", + "legendFormat": "{{type}}", "refId": "A", "step": 30 - }, - { - "expr": "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Stats Inaccuracy Rate", + "title": "Region Cache Error OPS", "tooltip": { "shared": true, "sort": 0, @@ -7289,16 +9816,18 @@ "fill": 1, "gridPos": { "h": 7, - "w": 8, - "x": 0, - "y": 20 + "w": 12, + "x": 12, + "y": 86 }, - "id": 71, + "id": 52, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -7317,19 +9846,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_pseudo_estimation_total[1m]))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Pseudo OPS", - "refId": "A", - "step": 30 + "legendFormat": "{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pseudo Estimation OPS", + "title": "Meta Operations Duration 99", "tooltip": { "shared": true, "sort": 0, @@ -7345,11 +9873,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -7365,7 +9893,22 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "Meta", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 152, + "panels": [ { "aliasColors": {}, "bars": false, @@ -7376,15 +9919,17 @@ "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 20 + "x": 0, + "y": 16 }, - "id": 92, + "id": 85, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -7403,19 +9948,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_dump_feedback_total[1m])) by (type)", + "expr": "sum(increase(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", - "refId": "A", - "step": 30 + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Dump Feedback OPS", + "title": "Worker Action OPM", "tooltip": { "shared": true, "sort": 0, @@ -7458,17 +10002,20 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, "fill": 1, "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 20 + "x": 8, + "y": 16 }, - "id": 93, + "id": 86, "legend": { "avg": false, "current": false, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, "show": true, @@ -7478,7 +10025,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -7489,19 +10036,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_update_stats_total[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 30 + "legendFormat": "99", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Update Stats OPS", + "title": "Duration 99", "tooltip": { "shared": true, "sort": 0, @@ -7517,7 +10063,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -7541,18 +10087,18 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Counter of query feedback whose actual count is much different than calculated by current statistics", "fill": 1, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 27 + "x": 16, + "y": 16 }, - "id": 113, + "id": 87, "legend": { "avg": false, "current": false, @@ -7567,6 +10113,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 5, "points": false, "renderer": "flot", @@ -7576,11 +10123,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_high_error_rate_feedback_total[1m]))", + "expr": "max(tidb_tikvclient_gc_config) by (type)", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "Significant Feedback", + "legendFormat": "{{type}}", "refId": "A" } ], @@ -7588,7 +10134,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Significant Feedback", + "title": "Config", "tooltip": { "shared": true, "sort": 0, @@ -7604,7 +10150,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -7624,22 +10170,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Statistics", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 151, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -7651,16 +10182,14 @@ "h": 7, "w": 8, "x": 0, - "y": 14 + "y": 23 }, - "id": 50, + "id": 88, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -7679,10 +10208,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_autoid_operation_duration_seconds_count[1m]))", + "expr": "sum(increase(tidb_tikvclient_gc_failure[1m])) by (type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "AutoID QPS", + "legendFormat": "{{type}}", "refId": "A" } ], @@ -7690,7 +10219,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "AutoID QPS", + "title": "GC Failure OPM", "tooltip": { "shared": true, "sort": 0, @@ -7738,9 +10267,9 @@ "h": 7, "w": 8, "x": 8, - "y": 14 + "y": 23 }, - "id": 51, + "id": 158, "legend": { "alignAsTable": true, "avg": false, @@ -7766,25 +10295,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99-{{type}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le, type))", + "expr": "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures[1m])) by (type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "80-{{type}}", - "refId": "C" + "legendFormat": "{{type}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "AutoID Duration", + "title": "Delete Range Failure OPM", "tooltip": { "shared": true, "sort": 0, @@ -7800,11 +10322,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": "0.001", + "min": null, "show": true }, { @@ -7832,16 +10354,14 @@ "h": 7, "w": 8, "x": 16, - "y": 14 + "y": 23 }, - "id": 52, + "id": 90, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -7860,10 +10380,10 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket[1m])) by (le, type))", + "expr": "sum(increase(tidb_tikvclient_gc_region_too_many_locks[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "Locks Error OPM", "refId": "A" } ], @@ -7871,7 +10391,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Meta Operations Duration 99", + "title": "Too Many Locks Error OPM", "tooltip": { "shared": true, "sort": 0, @@ -7887,11 +10407,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -7907,22 +10427,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Meta", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 152, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -7934,16 +10439,14 @@ "h": 7, "w": 8, "x": 0, - "y": 15 + "y": 30 }, - "id": 85, + "id": 89, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -7962,7 +10465,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", + "expr": "sum(increase(tidb_tikvclient_gc_action_result[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -7973,7 +10476,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Worker Action OPM", + "title": "Action Result OPM", "tooltip": { "shared": true, "sort": 0, @@ -8016,25 +10519,27 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, - "y": 15 + "y": 30 }, - "id": 86, + "id": 181, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -8044,25 +10549,33 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (le))", + "expr": "sum(tidb_tikvclient_range_task_stats) by (type, result)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "99", - "refId": "A" + "legendFormat": "{{type}}-{{result}}", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Duration 99", + "title": "Delete Range Task Status", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -8077,11 +10590,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "2", + "min": "0", "show": true }, { @@ -8104,27 +10617,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 15 + "y": 30 }, - "id": 87, + "id": 182, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -8135,22 +10653,24 @@ "steppedLine": false, "targets": [ { - "expr": "max(tidb_tikvclient_gc_config) by (type)", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" + "legendFormat": "{{instance}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Config", + "title": "Push Task Duration 95", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -8166,7 +10686,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -8182,57 +10702,85 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "GC", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 178, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 22 + "y": 17 }, - "id": 88, + "id": 176, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "total", + "fill": 0, + "lines": false + } + ], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_failure[1m]))", + "expr": "sum(tidb_tikvclient_pending_batch_requests) by (store)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A" + "legendFormat": "{{store}}", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC Failure OPM", + "title": "Pending Request Count by TiKV", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -8251,7 +10799,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -8274,27 +10822,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, - "y": 22 + "y": 17 }, - "id": 89, + "id": 179, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -8305,22 +10858,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_action_result[1m])) by (type)", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_wait_duration_bucket[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" + "legendFormat": "{{instance}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Action Result OPM", + "title": "Wait Duration 95", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -8332,11 +10887,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -8359,27 +10914,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 22 + "y": 17 }, - "id": 90, + "id": 180, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -8390,22 +10950,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_region_too_many_locks[1m]))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Locks Error OPM", - "refId": "A" + "legendFormat": "{{instance}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Too Many Locks Error OPM", + "title": "Batch Client Unavailable Duration 95", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -8417,11 +10979,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -8439,12 +11001,11 @@ } } ], - "repeat": null, - "title": "GC", + "title": "Batch Client", "type": "row" } ], - "refresh": "30s", + "refresh": false, "schemaVersion": 18, "style": "dark", "tags": [], @@ -8481,7 +11042,7 @@ ] }, "timezone": "browser", - "title": "Test-Cluster-TiDB1", + "title": "Test-Cluster-TiDB", "uid": "000000011", "version": 2 }