diff --git a/scripts/tikv_details.json b/scripts/tikv_details.json index d27ef65a4..8c1202e4f 100644 --- a/scripts/tikv_details.json +++ b/scripts/tikv_details.json @@ -14,25 +14,31 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "5.4.3" + "version": "6.1.6" }, { "type": "panel", "id": "graph", "name": "Graph", - "version": "5.0.0" + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", - "version": "5.0.0" + "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", - "version": "5.0.0" + "version": "" } ], "annotations": { @@ -52,7 +58,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1558420051540, + "iteration": 1562919981465, "links": [ { "icon": "doc", @@ -7950,7 +7956,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 24, + "maxPerRow": 1, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8048,7 +8054,7 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": 12, + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8175,7 +8181,7 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": 12, + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8288,7 +8294,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 24, + "maxPerRow": 1, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8392,7 +8398,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8510,7 +8515,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8625,7 +8629,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8740,7 +8743,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8855,7 +8857,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8954,7 +8955,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9053,7 +9053,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9152,7 +9151,6 @@ "lines": true, "linewidth": 2, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -9227,73 +9225,10 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 15 }, - "id": 2757, + "id": 2758, "panels": [ - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#5195ce", - "colorScale": "linear", - "colorScheme": "interpolateBlues", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "heatmap": {}, - "highlightCards": true, - "id": 3062, - "legend": { - "show": true - }, - "links": [], - "targets": [ - { - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Request duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, @@ -9301,35 +9236,31 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, + "x": 0, "y": 19 }, - "id": 16, + "id": 26, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -9342,18 +9273,35 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", + "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "E" + "legendFormat": " max", + "metric": "", + "refId": "A", + "step": 4 }, { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " 95%", + "metric": "", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_mvcc_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_versions_count{instance=~\"$instance\"}[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", "step": 4 } ], @@ -9361,11 +9309,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Request duration", + "title": "MVCC versions", "tooltip": { "msResolution": false, "shared": true, - "sort": 1, + "sort": 2, "value_type": "cumulative" }, "type": "graph", @@ -9378,9 +9326,8 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", - "label": null, + "format": "short", + "label": "", "logBase": 1, "max": null, "min": "0", @@ -9392,7 +9339,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -9414,10 +9361,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 27 + "x": 12, + "y": 19 }, - "id": 74, + "id": 559, "legend": { "alignAsTable": true, "avg": false, @@ -9426,14 +9373,12 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -9446,28 +9391,49 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (req)", - "format": "time_series", - "interval": "", + "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}", - "metric": "tikv_coprocessor_request_error", + "legendFormat": " max", + "metric": "", "refId": "A", "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " 95%", + "metric": "", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_gc_delete_versions_count{instance=~\"$instance\"}[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total Requests", + "title": "MVCC delete versions", "tooltip": { "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 2, + "value_type": "cumulative" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -9478,8 +9444,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "ops", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -9492,7 +9457,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -9507,17 +9472,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 27 + "x": 0, + "y": 26 }, - "id": 3128, + "id": 121, "legend": { "alignAsTable": true, "avg": false, @@ -9526,9 +9488,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -9546,28 +9506,48 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)", + "expr": "sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\"$instance\"}[1m])) by (task)", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "{{reason}}", - "metric": "tikv_coprocessor_request_error", + "legendFormat": "total-{{task}}", + "metric": "tikv_storage_command_total", "refId": "A", "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_gc_skipped_counter{instance=~\"$instance\"}[1m])) by (task)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "skipped-{{task}}", + "metric": "tikv_storage_gc_skipped_counter", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\"$instance\"}[1m])) by (task)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "failed-{{task}}", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_gc_worker_too_busy{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "gcworker-too-busy", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total Request Errors", + "title": "GC tasks", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -9578,12 +9558,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -9592,7 +9571,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -9607,17 +9586,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 34 + "x": 12, + "y": 26 }, - "id": 52, + "id": 2224, "legend": { "alignAsTable": true, "avg": false, @@ -9626,9 +9602,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -9646,20 +9620,44 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\"$instance\"}[1m])) by (req)", + "expr": "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{req}}", - "refId": "D" - } - ], + "legendFormat": "max-{{task}}", + "metric": "tikv_storage_command_total", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%-{{task}}", + "metric": "tikv_storage_gc_skipped_counter", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%-{{task}}", + "refId": "C" + }, + { + "expr": "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\"$instance\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\"$instance\"}[1m])) by (task)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average-{{task}}", + "refId": "D" + } + ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total KV Cursor Operations", + "title": "GC tasks duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9674,12 +9672,11 @@ }, "yaxes": [ { - "decimals": 0, - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -9688,7 +9685,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -9703,17 +9700,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "editable": true, - "error": false, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 34 + "x": 0, + "y": 33 }, - "id": 3129, + "id": 2225, "legend": { "alignAsTable": true, "avg": false, @@ -9722,9 +9716,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -9742,27 +9734,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "100%-{{req}}", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", + "expr": "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"write\"}[1m])) by (tag)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "99%-{{req}}", - "refId": "A" + "legendFormat": "{{tag}}", + "metric": "tikv_storage_command_total", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV Cursor Operations", + "title": "GC keys (write CF)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9777,7 +9763,6 @@ }, "yaxes": [ { - "decimals": 0, "format": "short", "label": null, "logBase": 1, @@ -9791,7 +9776,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -9805,8 +9790,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", + "decimals": 2, "editable": true, "error": false, "fill": 1, @@ -9814,10 +9798,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 33 }, - "id": 2118, + "id": 966, "legend": { "alignAsTable": true, "avg": false, @@ -9826,38 +9810,30 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "key_skipped", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m])) by (req)", + "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "delete_skipped-{{req}}", - "metric": "scan_details", - "refId": "B", + "legendFormat": "{{type}}", + "metric": "", + "refId": "A", "step": 4 } ], @@ -9865,12 +9841,12 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total RocksDB Perf Statistics", + "title": "TiDB GC worker actions", "tooltip": { "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 2, + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -9882,22 +9858,20 @@ }, "yaxes": [ { - "decimals": 1, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { - "decimals": null, "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -9911,7 +9885,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -9919,26 +9892,25 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 41 + "x": 0, + "y": 40 }, - "id": 551, + "id": 969, "legend": { "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sort": null, + "sortDesc": null, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -9951,26 +9923,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_response_bytes{instance=~\"$instance\"}[1m]))", + "expr": "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (instance, le))", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "size", - "metric": "", + "legendFormat": "{{instance}}", "refId": "A", - "step": 4 + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total Response Size", + "title": "TiDB GC seconds", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -9982,12 +9952,11 @@ }, "yaxes": [ { - "decimals": 0, - "format": "decbytes", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -9996,29 +9965,14 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Coprocessor Overview", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 3197, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -10026,6 +9980,7 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "description": "keys / second", "editable": true, "error": false, "fill": 1, @@ -10033,10 +9988,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 20 + "x": 12, + "y": 40 }, - "id": 113, + "id": 2589, "legend": { "alignAsTable": true, "avg": false, @@ -10045,13 +10000,12 @@ "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -10064,30 +10018,22 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{req}}-100%", + "legendFormat": "keys/s", "refId": "E" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Handle duration", + "title": "GC speed", "tooltip": { "msResolution": false, "shared": true, - "sort": 1, + "sort": 2, "value_type": "cumulative" }, "type": "graph", @@ -10100,8 +10046,7 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "format": "short", "label": "", "logBase": 1, "max": null, @@ -10109,13 +10054,12 @@ "show": true }, { - "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -10125,22 +10069,19 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, + "decimals": 0, "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 20 + "x": 0, + "y": 47 }, - "id": 117, + "id": 2819, "legend": { "alignAsTable": true, "avg": false, @@ -10149,13 +10090,11 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -10165,15 +10104,18 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", + "expr": "sum(max_over_time(tikv_gcworker_autogc_status{instance=~\"$instance\", state=\"working\"}[1m])) by (instance)", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", - "refId": "B", + "legendFormat": "{{instance}}", + "metric": "tikv_storage_command_total", + "refId": "A", "step": 4 } ], @@ -10181,12 +10123,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "95% Handle duration by store", + "title": "TiKV AutoGC Working", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -10198,12 +10139,12 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "decimals": 0, + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10226,18 +10167,16 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, + "decimals": 0, + "description": "Progress of ResolveLocks, the first phase of GC", "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 27 + "x": 12, + "y": 47 }, - "id": 111, + "id": 2823, "legend": { "alignAsTable": true, "avg": false, @@ -10246,36 +10185,31 @@ "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "expr": "max(tidb_tikvclient_range_task_stats{type=~\"resolve-locks.*\"}) by (result)", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{req}}-99%", + "legendFormat": "{{result}}", + "metric": "tikv_storage_command_total", "refId": "A", "step": 4 } @@ -10284,12 +10218,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Wait duration", + "title": "ResolveLocks Progress", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -10301,12 +10234,12 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "decimals": 0, + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10329,28 +10262,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, + "decimals": 0, + "description": "Progress of TiKV's GC", "fill": 1, - "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 27 + "x": 0, + "y": 54 }, - "id": 116, + "id": 2821, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, @@ -10368,11 +10298,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", + "expr": "sum(tikv_gcworker_autogc_processed_regions{instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", - "refId": "B", + "legendFormat": "{{instance}}", + "metric": "tikv_storage_command_total", + "refId": "A", "step": 4 } ], @@ -10380,12 +10313,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "95% Wait duration by store", + "title": "TiKV Auto GC Progress", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -10397,16 +10329,15 @@ }, "yaxes": [ { - "decimals": 1, - "format": "s", + "decimals": null, + "format": "percentunit", "label": null, "logBase": 1, - "max": null, - "min": "0", + "max": "1.1", + "min": null, "show": true }, { - "decimals": 1, "format": "short", "label": null, "logBase": 1, @@ -10426,36 +10357,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, + "decimals": 0, + "description": "SafePoint used for TiKV's Auto GC", + "fill": 0, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 34 + "x": 12, + "y": 54 }, - "id": 3195, + "id": 2822, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -10466,12 +10393,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_dag_request_count{instance=~\"$instance\"}[1m])) by (vec_type)", + "expr": "max(tikv_gcworker_autogc_safe_point) by (instance) / (2^18)", "format": "time_series", + "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{vec_type}}", - "metric": "", + "legendFormat": "{{instance}}", + "metric": "tikv_storage_command_total", "refId": "A", "step": 4 } @@ -10480,9 +10408,8 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total DAG Requests", + "title": "TiKV Auto GC SafePoint", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -10497,12 +10424,12 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "decimals": null, + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10520,140 +10447,220 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, + "decimals": 0, "editable": true, "error": false, - "fill": 1, - "grid": {}, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 3264, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "w": 6, + "x": 0, + "y": 61 }, - "lines": true, - "linewidth": 1, + "id": 27, + "interval": null, "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", "targets": [ { - "expr": "sum(rate(tikv_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)", - "format": "time_series", + "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", "interval": "", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", "refId": "A", - "step": 4 + "step": 60 } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total DAG Executors", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, + "thresholds": "", + "title": "GC lifetime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "op": "=", + "text": "N/A", + "value": "null" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "valueName": "current" }, { - "aliasColors": {}, - "bars": false, + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 61 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "GC interval", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "title": "GC", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2759, + "panels": [ + { + "aliasColors": {}, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 9, - "w": 12, + "h": 7, + "w": 8, "x": 0, - "y": 41 + "y": 351 }, - "id": 552, + "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -10664,26 +10671,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag)", - "format": "time_series", - "interval": "", + "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"snapshot\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 + "legendFormat": " ", + "refId": "A", + "step": 60 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details (Table Scan)", + "title": "Rate snapshot message", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -10695,12 +10698,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "format": "opm", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10709,13 +10711,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10723,29 +10721,26 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 41 + "h": 7, + "w": 8, + "x": 8, + "y": 351 }, - "id": 3263, + "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -10763,26 +10758,36 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag)", - "format": "time_series", - "interval": "", + "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "scan_details", + "legendFormat": "send", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "apply", "refId": "B", - "step": 4 + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"generate\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "generate", + "refId": "C", + "step": 60 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details (Index Scan)", + "title": "99% Handle snapshot duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -10794,12 +10799,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10808,13 +10812,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10822,29 +10822,26 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 50 + "h": 7, + "w": 8, + "x": 16, + "y": 351 }, - "id": 122, + "id": 38, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -10856,33 +10853,29 @@ "pointradius": 5, "points": false, "renderer": "flot", - "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag,cf)", - "format": "time_series", - "interval": "", + "expr": "sum(tikv_raftstore_snapshot_traffic_total{instance=~\"$instance\"}) by (type)", "intervalFactor": 2, - "legendFormat": "{{cf}}-{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 + "legendFormat": "{{type}}", + "metric": "", + "refId": "A", + "step": 60 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details by CF (Table Scan)", + "title": "Snapshot state count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -10894,12 +10887,11 @@ }, "yaxes": [ { - "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10908,13 +10900,9 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -10922,63 +10910,55 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 9, + "h": 7, "w": 12, - "x": 12, - "y": 50 + "x": 0, + "y": 358 }, - "id": 554, + "id": 44, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "repeat": "cf", - "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag,cf)", - "format": "time_series", - "interval": "", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{cf}}-{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 + "legendFormat": "size", + "metric": "tikv_snapshot_size_bucket", + "refId": "A", + "step": 40 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Total Ops Details by CF (Index Scan)", + "title": "99.99% Snapshot size", "tooltip": { "msResolution": false, "shared": true, @@ -10995,12 +10975,11 @@ }, "yaxes": [ { - "decimals": 1, - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -11009,35 +10988,16 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "title": "Coprocessor Detail", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 2758, - "panels": [ + ] + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -11045,24 +11005,24 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 19 + "x": 12, + "y": 358 }, - "id": 26, + "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -11075,48 +11035,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\"$instance\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": " max", - "metric": "", + "legendFormat": "count", + "metric": "tikv_snapshot_kv_count_bucket", "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": " 95%", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_mvcc_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_versions_count{instance=~\"$instance\"}[1m])) ", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 4 + "step": 40 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "MVCC versions", + "title": "99.99% Snapshot KV count", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, - "value_type": "cumulative" + "sort": 0, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -11129,10 +11064,10 @@ "yaxes": [ { "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -11143,12 +11078,23 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, + ] + } + ], + "repeat": null, + "title": "Snapshot", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 2760, + "panels": [ { "aliasColors": {}, "bars": false, @@ -11161,28 +11107,30 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 12, - "y": 19 + "x": 0, + "y": 359 }, - "id": 559, + "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": 400, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -11193,48 +11141,22 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", + "expr": "sum(rate(tikv_worker_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", "intervalFactor": 2, - "legendFormat": " max", - "metric": "", + "legendFormat": "{{name}}", "refId": "A", "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": " 95%", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_storage_mvcc_gc_delete_versions_count{instance=~\"$instance\"}[1m])) ", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "MVCC delete versions", + "title": "Worker handled tasks", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, - "value_type": "cumulative" + "sort": 0, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -11246,8 +11168,8 @@ }, "yaxes": [ { - "format": "short", - "label": "", + "format": "ops", + "label": null, "logBase": 1, "max": null, "min": "0", @@ -11261,11 +11183,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -11274,30 +11192,35 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 0, - "y": 26 + "x": 12, + "y": 359 }, - "id": 121, + "id": 1395, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -11308,44 +11231,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", + "expr": "sum(rate(tikv_worker_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", "intervalFactor": 2, - "legendFormat": "total-{{task}}", - "metric": "tikv_storage_command_total", + "legendFormat": "{{name}}", "refId": "A", "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_gc_skipped_counter{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "skipped-{{task}}", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed-{{task}}", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_gc_worker_too_busy{instance=~\"$instance\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "gcworker-too-busy", - "refId": "D" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC tasks", + "title": "Worker pending tasks", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11364,7 +11262,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -11375,11 +11273,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -11388,30 +11282,35 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 12, - "y": 26 - }, - "id": 2224, + "x": 0, + "y": 367 + }, + "id": 1876, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -11422,44 +11321,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", + "expr": "sum(rate(tikv_futurepool_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "max-{{task}}", - "metric": "tikv_storage_command_total", + "legendFormat": "{{name}}", "refId": "A", "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%-{{task}}", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%-{{task}}", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\"$instance\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\"$instance\"}[1m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average-{{task}}", - "refId": "D" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC tasks duration", + "title": "FuturePool handled tasks", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11474,11 +11349,11 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -11489,11 +11364,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -11502,30 +11373,35 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 0, - "y": 33 + "x": 12, + "y": 367 }, - "id": 2225, + "id": 1877, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -11536,21 +11412,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"write\"}[1m])) by (tag)", + "expr": "sum(rate(tikv_futurepool_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "tikv_storage_command_total", + "legendFormat": "{{name}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC keys (write CF)", + "title": "FuturePool pending tasks", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11580,11 +11455,87 @@ "min": null, "show": true } + ] + } + ], + "repeat": null, + "title": "Task", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 2757, + "panels": [ + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#5195ce", + "colorScale": "linear", + "colorScheme": "interpolateBlues", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 3062, + "legend": { + "show": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Request duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -11592,32 +11543,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, + "decimals": 1, + "description": "", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 22 }, - "id": 966, + "id": 16, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -11630,11 +11585,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", + "legendFormat": "{{req}}-100%", + "refId": "E" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 } @@ -11643,11 +11604,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiDB GC worker actions", + "title": "Request duration", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, + "sort": 1, "value_type": "cumulative" }, "type": "graph", @@ -11660,8 +11621,9 @@ }, "yaxes": [ { - "format": "short", - "label": "", + "decimals": 1, + "format": "s", + "label": null, "logBase": 1, "max": null, "min": "0", @@ -11673,7 +11635,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -11687,6 +11649,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -11695,24 +11658,25 @@ "h": 7, "w": 12, "x": 0, - "y": 40 + "y": 30 }, - "id": 969, + "id": 74, "legend": { "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, - "sort": null, - "sortDesc": null, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -11725,24 +11689,26 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (instance, le))", + "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (req)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "{{req}}", + "metric": "tikv_coprocessor_request_error", "refId": "A", - "step": 40 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiDB GC seconds", + "title": "Total Requests", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -11754,11 +11720,12 @@ }, "yaxes": [ { - "format": "s", - "label": null, + "decimals": 1, + "format": "ops", + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -11767,7 +11734,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -11782,7 +11749,6 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "keys / second", "editable": true, "error": false, "fill": 1, @@ -11791,9 +11757,9 @@ "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 30 }, - "id": 2589, + "id": 3128, "legend": { "alignAsTable": true, "avg": false, @@ -11802,12 +11768,14 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, @@ -11820,23 +11788,26 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m]))", + "expr": "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "keys/s", - "refId": "E" + "legendFormat": "{{reason}}", + "metric": "tikv_coprocessor_request_error", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC speed", + "title": "Total Request Errors", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, - "value_type": "cumulative" + "sort": 0, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -11848,8 +11819,9 @@ }, "yaxes": [ { - "format": "short", - "label": "", + "decimals": 1, + "format": "ops", + "label": null, "logBase": 1, "max": null, "min": "0", @@ -11861,7 +11833,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -11871,19 +11843,22 @@ }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, + "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 37 }, - "id": 2819, + "id": 52, "legend": { "alignAsTable": true, "avg": false, @@ -11892,41 +11867,40 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": false, + "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(max_over_time(tikv_gcworker_autogc_status{instance=~\"$instance\", state=\"working\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\"$instance\"}[1m])) by (req)", "format": "time_series", - "instant": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 + "legendFormat": "{{req}}", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiKV AutoGC Working", + "title": "Total KV Cursor Operations", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11942,11 +11916,11 @@ "yaxes": [ { "decimals": 0, - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -11969,16 +11943,18 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "description": "Progress of ResolveLocks, the first phase of GC", + "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 47 + "y": 37 }, - "id": 2823, + "id": 3129, "legend": { "alignAsTable": true, "avg": false, @@ -11987,41 +11963,47 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(tidb_tikvclient_range_task_stats{type=~\"resolve-locks.*\"}) by (result)", + "expr": "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", "format": "time_series", - "instant": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "{{result}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 + "legendFormat": "100%-{{req}}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%-{{req}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "ResolveLocks Progress", + "title": "KV Cursor Operations", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -12037,11 +12019,11 @@ "yaxes": [ { "decimals": 0, - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -12064,50 +12046,59 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "description": "Progress of TiKV's GC", + "decimals": 1, + "description": "", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 54 + "y": 44 }, - "id": 2821, + "id": 2118, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "key_skipped", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(tikv_gcworker_autogc_processed_regions{instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)", + "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m])) by (req)", "format": "time_series", - "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_storage_command_total", - "refId": "A", + "legendFormat": "delete_skipped-{{req}}", + "metric": "scan_details", + "refId": "B", "step": 4 } ], @@ -12115,8 +12106,9 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiKV Auto GC Progress", + "title": "Total RocksDB Perf Statistics", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -12131,17 +12123,18 @@ }, "yaxes": [ { - "decimals": null, - "format": "percentunit", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, - "max": "1.1", - "min": null, + "max": null, + "min": "0", "show": true }, { + "decimals": null, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, @@ -12159,32 +12152,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "description": "SafePoint used for TiKV's Auto GC", - "fill": 0, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 54 + "y": 44 }, - "id": 2822, + "id": 551, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 250, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -12195,13 +12192,12 @@ "steppedLine": false, "targets": [ { - "expr": "max(tikv_gcworker_autogc_safe_point) by (instance) / (2^18)", + "expr": "sum(rate(tikv_coprocessor_response_bytes{instance=~\"$instance\"}[1m]))", "format": "time_series", - "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_storage_command_total", + "legendFormat": "size", + "metric": "", "refId": "A", "step": 4 } @@ -12210,8 +12206,9 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiKV Auto GC SafePoint", + "title": "Total Response Size", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -12226,12 +12223,12 @@ }, "yaxes": [ { - "decimals": null, - "format": "dateTimeAsIso", + "decimals": 0, + "format": "decbytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -12247,1099 +12244,92 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "Coprocessor Overview", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 3197, + "panels": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, + "decimals": 1, "editable": true, "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, + "fill": 1, + "grid": {}, "gridPos": { "h": 7, - "w": 6, + "w": 12, "x": 0, - "y": 61 + "y": 20 }, - "id": 27, - "interval": null, + "id": 113, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, "links": [], - "mappingType": 1, - "mappingTypes": [ + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "name": "value to text", - "value": 1 + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{req}}-100%", + "refId": "E" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "null", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", - "interval": "", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "thresholds": "", - "title": "GC lifetime", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "editable": true, - "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 61 - }, - "id": 28, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "null", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "thresholds": "", - "title": "GC interval", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "title": "GC", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 2759, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 351 - }, - "id": 35, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"snapshot\"}[1m]))", - "intervalFactor": 2, - "legendFormat": " ", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Rate snapshot message", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 351 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "send", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "apply", - "refId": "B", - "step": 60 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"generate\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "generate", - "refId": "C", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "99% Handle snapshot duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 351 - }, - "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum(tikv_raftstore_snapshot_traffic_total{instance=~\"$instance\"}) by (type)", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Snapshot state count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 358 - }, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "size", - "metric": "tikv_snapshot_size_bucket", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "99.99% Snapshot size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 358 - }, - "id": 43, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\"$instance\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "count", - "metric": "tikv_snapshot_kv_count_bucket", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "99.99% Snapshot KV count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "title": "Snapshot", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 2760, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 359 - }, - "id": 59, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 400, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_worker_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Worker handled tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 359 - }, - "id": 1395, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 400, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_worker_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Worker pending tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 367 - }, - "id": 1876, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 400, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_futurepool_handled_task_total{instance=~\"$instance\"}[1m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "FuturePool handled tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 367 - }, - "id": 1877, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 400, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_futurepool_pending_task_total{instance=~\"$instance\"}[1m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "FuturePool pending tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "title": "Task", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 2761, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 368 - }, - "id": 2108, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance, state)", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{state}}", + "intervalFactor": 2, + "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 - }, - { - "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-total", - "refId": "B" } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Threads state", + "title": "Handle duration", "tooltip": { + "msResolution": false, "shared": true, - "sort": 0, - "value_type": "individual" + "sort": 1, + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13351,22 +12341,28 @@ }, "yaxes": [ { - "format": "none", - "label": null, + "decimals": 1, + "format": "s", + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { + "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13375,14 +12371,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 368 + "y": 20 }, - "id": 2258, + "id": 117, "legend": { "alignAsTable": true, "avg": false, @@ -13391,8 +12390,8 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", + "sideWidth": null, + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -13402,8 +12401,8 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13411,24 +12410,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_threads_io_bytes_total{instance=~\"$instance\"}[30s])) by (name, io) > 1024", + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{name}}-{{io}}", - "refId": "A", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{req}}", + "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Threads IO", + "title": "95% Handle duration by store", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13440,11 +12439,12 @@ }, "yaxes": [ { - "format": "Bps", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13453,9 +12453,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13464,14 +12468,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 375 + "y": 27 }, - "id": 2660, + "id": 111, "legend": { "alignAsTable": true, "avg": false, @@ -13480,8 +12487,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -13489,10 +12495,10 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13500,24 +12506,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_voluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} - {{name}}", + "intervalFactor": 2, + "legendFormat": "{{req}}-100%", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Thread Voluntary Context Switches", + "title": "Wait duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13529,11 +12542,12 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13542,9 +12556,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13553,14 +12571,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 375 + "y": 27 }, - "id": 2661, + "id": 116, "legend": { "alignAsTable": true, "avg": false, @@ -13569,8 +12590,7 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 250, - "sort": "max", + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -13580,8 +12600,8 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 2, - "points": true, + "pointradius": 5, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -13589,24 +12609,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_nonvoluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} - {{name}}", - "refId": "A", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{req}}", + "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Thread Nonvoluntary Context Switches", + "title": "95% Wait duration by store", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -13618,38 +12638,29 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { + "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] - } - ], - "repeat": null, - "title": "Threads", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 2762, - "panels": [ + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -13657,14 +12668,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 376 + "y": 34 }, - "id": 138, + "id": 3195, "legend": { "alignAsTable": true, "avg": false, @@ -13673,73 +12687,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, + "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "memtable", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", + "expr": "sum(rate(tikv_coprocessor_dag_request_count{instance=~\"$instance\"}[1m])) by (vec_type)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "block_cache", + "legendFormat": "{{vec_type}}", "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "l0", "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "l1", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "l2_and_up", - "refId": "F", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Get operations", + "title": "Total DAG Requests", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -13754,11 +12738,12 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13767,9 +12752,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13778,14 +12767,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 376 + "y": 34 }, - "id": 82, + "id": 3264, "legend": { "alignAsTable": true, "avg": false, @@ -13794,69 +12786,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", + "expr": "sum(rate(tikv_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "{{type}}", + "metric": "", "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Get duration", + "title": "Total DAG Executors", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -13871,11 +12837,12 @@ }, "yaxes": [ { - "format": "µs", + "decimals": 1, + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13884,9 +12851,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -13895,14 +12866,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 0, - "y": 384 + "y": 41 }, - "id": 129, + "id": 552, "legend": { "alignAsTable": true, "avg": false, @@ -13911,84 +12885,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "seek", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "seek_found", - "metric": "", + "legendFormat": "{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "next", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "next_found", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "prev", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "prev_found", - "metric": "", - "refId": "F", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Seek operations", + "title": "Total Ops Details (Table Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14003,11 +12936,12 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14016,9 +12950,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -14027,14 +12965,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, - "y": 384 + "y": 41 }, - "id": 125, + "id": 3263, "legend": { "alignAsTable": true, "avg": false, @@ -14043,69 +12984,43 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Seek duration", + "title": "Total Ops Details (Index Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14120,11 +13035,12 @@ }, "yaxes": [ { - "format": "µs", + "decimals": 1, + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14133,9 +13049,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -14144,14 +13064,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 0, - "y": 392 + "y": 50 }, - "id": 139, + "id": 122, "legend": { "alignAsTable": true, "avg": false, @@ -14160,7 +13083,9 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -14172,45 +13097,30 @@ "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, + "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "done", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag,cf)", + "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "timeout", + "legendFormat": "{{cf}}-{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "with_wal", - "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Write operations", + "title": "Total Ops Details by CF (Table Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14225,11 +13135,12 @@ }, "yaxes": [ { - "format": "ops", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14238,9 +13149,13 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -14249,14 +13164,17 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, - "y": 392 + "y": 50 }, - "id": 126, + "id": 554, "legend": { "alignAsTable": true, "avg": false, @@ -14265,69 +13183,45 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, + "repeat": "cf", + "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", + "expr": "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag,cf)", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{cf}}-{{tag}}", + "metric": "scan_details", "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Write duration", + "title": "Total Ops Details by CF (Index Scan)", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -14342,11 +13236,12 @@ }, "yaxes": [ { - "format": "µs", + "decimals": 1, + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14355,10 +13250,28 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } - ] - }, + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Coprocessor Detail", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 2761, + "panels": [ { "aliasColors": {}, "bars": false, @@ -14368,12 +13281,12 @@ "decimals": 1, "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 400 + "y": 368 }, - "id": 137, + "id": 2108, "legend": { "alignAsTable": true, "avg": false, @@ -14382,7 +13295,9 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "current", + "sortDesc": false, "total": false, "values": true }, @@ -14391,34 +13306,34 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_wal_file_synced{instance=~\"$instance\", db=\"$db\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "sync", - "metric": "", + "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance, state)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{state}}", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-total", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "WAL sync operations", + "title": "Threads state", "tooltip": { "shared": true, "sort": 0, @@ -14434,7 +13349,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -14458,14 +13373,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 400 + "y": 368 }, - "id": 135, + "id": 2258, "legend": { "alignAsTable": true, "avg": false, @@ -14474,64 +13389,40 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", - "intervalFactor": 2, - "legendFormat": "max", + "expr": "sum(rate(tikv_threads_io_bytes_total{instance=~\"$instance\"}[30s])) by (name, io) > 1024", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}-{{io}}", "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "WAL sync duration", + "title": "Threads IO", "tooltip": { "shared": true, "sort": 0, @@ -14547,11 +13438,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "Bps", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -14573,12 +13464,12 @@ "decimals": 1, "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 408 + "y": 375 }, - "id": 128, + "id": 2660, "legend": { "alignAsTable": true, "avg": false, @@ -14587,7 +13478,9 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, @@ -14596,34 +13489,29 @@ "links": [], "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_event_total{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_engine_event_total", - "refId": "B", - "step": 10 + "expr": "sum(rate(tikv_thread_voluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} - {{name}}", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction operations", + "title": "Thread Voluntary Context Switches", "tooltip": { "shared": true, "sort": 0, @@ -14639,7 +13527,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -14663,14 +13551,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 408 + "y": 375 }, - "id": 136, + "id": 2661, "legend": { "alignAsTable": true, "avg": false, @@ -14679,69 +13567,40 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": 300, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": null, "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, + "pointradius": 2, + "points": true, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", + "expr": "sum(rate(tikv_thread_nonvoluntary_context_switches{instance=~\"$instance\"}[30s])) by (instance, name) > 0", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} - {{name}}", "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction duration", + "title": "Thread Nonvoluntary Context Switches", "tooltip": { "shared": true, "sort": 0, @@ -14757,9 +13616,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -14773,7 +13632,22 @@ "show": true } ] - }, + } + ], + "repeat": null, + "title": "Threads", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 2762, + "panels": [ { "aliasColors": {}, "bars": false, @@ -14786,9 +13660,9 @@ "h": 8, "w": 12, "x": 0, - "y": 416 + "y": 376 }, - "id": 140, + "id": 138, "legend": { "alignAsTable": true, "avg": false, @@ -14822,42 +13696,47 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", + "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "memtable", "metric": "", - "refId": "A", + "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "block_cache", "metric": "", - "refId": "B", + "refId": "E", "step": 10 }, { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", + "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", + "legendFormat": "l0", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "l1", "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", + "expr": "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", + "legendFormat": "l2_and_up", + "refId": "F", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "SST read duration", + "title": "Get operations", "tooltip": { "shared": true, "sort": 0, @@ -14873,9 +13752,9 @@ }, "yaxes": [ { - "format": "ms", + "format": "ops", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -14897,14 +13776,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 416 + "y": 376 }, - "id": 87, + "id": 82, "legend": { "alignAsTable": true, "avg": false, @@ -14938,34 +13817,34 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "max", - "metric": "", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", - "metric": "", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", - "metric": "", "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", + "expr": "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", - "metric": "", "refId": "D", "step": 10 } @@ -14973,7 +13852,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Write stall duration", + "title": "Get duration", "tooltip": { "shared": true, "sort": 0, @@ -14989,9 +13868,9 @@ }, "yaxes": [ { - "format": "ms", + "format": "µs", "label": null, - "logBase": 10, + "logBase": 2, "max": null, "min": null, "show": true @@ -15018,9 +13897,9 @@ "h": 8, "w": 12, "x": 0, - "y": 424 + "y": 384 }, - "id": 103, + "id": 129, "legend": { "alignAsTable": true, "avg": false, @@ -15054,17 +13933,58 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_memory_bytes{instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "seek", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "seek_found", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{cf}}", - "refId": "A", + "legendFormat": "next", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "next_found", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "prev", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "prev_found", + "metric": "", + "refId": "F", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Memtable size", + "title": "Seek operations", "tooltip": { "shared": true, "sort": 0, @@ -15080,7 +14000,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -15109,9 +14029,9 @@ "h": 8, "w": 12, "x": 12, - "y": 424 + "y": 384 }, - "id": 88, + "id": 125, "legend": { "alignAsTable": true, "avg": false, @@ -15127,8 +14047,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": null, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -15146,23 +14065,47 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))", + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "hit", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Memtable hit", + "title": "Seek duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -15173,20 +14116,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -15202,9 +14145,9 @@ "h": 8, "w": 12, "x": 0, - "y": 432 + "y": 392 }, - "id": 102, + "id": 139, "legend": { "alignAsTable": true, "avg": false, @@ -15238,17 +14181,31 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_block_cache_size_bytes{instance=~\"$instance\", db=\"$db\"}) by(cf)", + "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{cf}}", + "legendFormat": "done", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "timeout", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "with_wal", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache size", + "title": "Write operations", "tooltip": { "shared": true, "sort": 0, @@ -15264,7 +14221,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -15293,9 +14250,9 @@ "h": 8, "w": 12, "x": 12, - "y": 432 + "y": 392 }, - "id": 80, + "id": 126, "legend": { "alignAsTable": true, "avg": false, @@ -15311,8 +14268,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -15330,56 +14286,47 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_miss\"}[1m])))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "all", - "metric": "", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", - "intervalFactor": 2, - "legendFormat": "data", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "filter", - "metric": "", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "index", - "metric": "", + "legendFormat": "95%", "refId": "C", "step": 10 }, { - "expr": "sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", + "expr": "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "bloom prefix", - "metric": "", - "refId": "E", + "legendFormat": "avg", + "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache hit", + "title": "Write duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -15390,20 +14337,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -15414,15 +14361,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 440 + "y": 400 }, - "height": "", - "id": 467, + "id": 137, "legend": { "alignAsTable": true, "avg": false, @@ -15430,104 +14376,44 @@ "max": true, "min": false, "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total_read", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total_written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "data_insert", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_insert", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_evict", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "index_insert", - "metric": "", - "refId": "F", - "step": 10 - }, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", - "hide": false, - "interval": "", + "expr": "sum(rate(tikv_engine_wal_file_synced{instance=~\"$instance\", db=\"$db\"}[1m]))", "intervalFactor": 2, - "legendFormat": "index_evict", + "legendFormat": "sync", "metric": "", - "refId": "G", + "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache flow", + "title": "WAL sync operations", "tooltip": { "shared": true, "sort": 0, @@ -15543,19 +14429,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "ops", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15567,14 +14453,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 440 + "y": 400 }, - "id": 468, + "id": 135, "legend": { "alignAsTable": true, "avg": false, @@ -15590,6 +14476,7 @@ "lines": true, "linewidth": 1, "links": [], + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -15608,50 +14495,38 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "intervalFactor": 2, - "legendFormat": "total_add", - "metric": "", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "data_add", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "intervalFactor": 2, - "legendFormat": "filter_add", - "metric": "", - "refId": "D", + "legendFormat": "99%", + "refId": "B", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "intervalFactor": 2, - "legendFormat": "index_add", - "metric": "", - "refId": "E", + "legendFormat": "95%", + "refId": "C", "step": 10 }, { - "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", "intervalFactor": 2, - "legendFormat": "add_failures", - "metric": "", - "refId": "B", + "legendFormat": "avg", + "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Block cache operations", + "title": "WAL sync duration", "tooltip": { "shared": true, "sort": 0, @@ -15667,11 +14542,11 @@ }, "yaxes": [ { - "format": "ops", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -15691,15 +14566,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 448 + "y": 408 }, - "height": "", - "id": 132, + "id": 128, "legend": { "alignAsTable": true, "avg": false, @@ -15733,38 +14607,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", - "hide": false, - "interval": "", + "expr": "sum(rate(tikv_engine_event_total{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "read", + "legendFormat": "{{type}}", + "metric": "tikv_engine_event_total", "refId": "B", "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\"$instance\", db=\"$db\"}[1m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "corrupt", - "metric": "", - "refId": "A", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Keys flow", + "title": "Compaction operations", "tooltip": { "shared": true, "sort": 0, @@ -15784,7 +14638,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15792,7 +14646,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15804,14 +14658,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 448 + "y": 408 }, - "id": 131, + "id": 136, "legend": { "alignAsTable": true, "avg": false, @@ -15845,19 +14699,43 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_estimate_num_keys{instance=~\"$instance\", db=\"$db\"}) by (cf)", - "hide": false, + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_estimate_num_keys", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total keys", + "title": "Compaction duration", "tooltip": { "shared": true, "sort": 0, @@ -15873,11 +14751,11 @@ }, "yaxes": [ { - "format": "short", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15885,7 +14763,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -15897,15 +14775,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 456 + "y": 416 }, - "height": "", - "id": 85, + "id": 140, "legend": { "alignAsTable": true, "avg": false, @@ -15939,28 +14816,42 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", - "hide": false, - "interval": "", + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "intervalFactor": 2, - "legendFormat": "get", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", - "hide": false, - "interval": "", + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "intervalFactor": 2, - "legendFormat": "scan", + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", "refId": "C", "step": 10 + }, + { + "expr": "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Read flow", + "title": "SST read duration", "tooltip": { "shared": true, "sort": 0, @@ -15976,11 +14867,11 @@ }, "yaxes": [ { - "format": "Bps", + "format": "ms", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15988,7 +14879,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16000,14 +14891,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 456 + "y": 416 }, - "id": 133, + "id": 87, "legend": { "alignAsTable": true, "avg": false, @@ -16023,7 +14914,6 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -16042,30 +14932,34 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "intervalFactor": 2, "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", + "metric": "", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", + "metric": "", "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", + "expr": "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", "intervalFactor": 2, "legendFormat": "avg", + "metric": "", "refId": "D", "step": 10 } @@ -16073,7 +14967,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Bytes / Read", + "title": "Write stall duration", "tooltip": { "shared": true, "sort": 0, @@ -16089,11 +14983,11 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "ms", "label": null, "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16118,10 +15012,9 @@ "h": 8, "w": 12, "x": 0, - "y": 464 + "y": 424 }, - "height": "", - "id": 86, + "id": 103, "legend": { "alignAsTable": true, "avg": false, @@ -16155,18 +15048,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "wal", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", - "hide": false, + "expr": "avg(tikv_engine_memory_bytes{instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", "intervalFactor": 2, - "legendFormat": "write", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 } @@ -16174,7 +15058,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Write flow", + "title": "Memtable size", "tooltip": { "shared": true, "sort": 0, @@ -16190,11 +15074,11 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16202,7 +15086,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16219,9 +15103,9 @@ "h": 8, "w": 12, "x": 12, - "y": 464 + "y": 424 }, - "id": 134, + "id": 88, "legend": { "alignAsTable": true, "avg": false, @@ -16237,8 +15121,7 @@ "lines": true, "linewidth": 1, "links": [], - "minSpan": 12, - "nullPointMode": "null", + "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, @@ -16256,43 +15139,23 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", + "expr": "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "hit", "refId": "A", "step": 10 - }, - { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Bytes / Write", + "title": "Memtable hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -16303,20 +15166,20 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "percentunit", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": "0", "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] }, @@ -16332,9 +15195,9 @@ "h": 8, "w": 12, "x": 0, - "y": 472 + "y": 432 }, - "id": 90, + "id": 102, "legend": { "alignAsTable": true, "avg": false, @@ -16368,34 +15231,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", - "hide": false, + "expr": "avg(tikv_engine_block_cache_size_bytes{instance=~\"$instance\", db=\"$db\"}) by(cf)", "intervalFactor": 2, - "legendFormat": "read", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "flushed", - "refId": "B", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction flow", + "title": "Block cache size", "tooltip": { "shared": true, "sort": 0, @@ -16411,19 +15257,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16435,14 +15281,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 472 + "y": 432 }, - "id": 127, + "id": 80, "legend": { "alignAsTable": true, "avg": false, @@ -16458,7 +15304,8 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "maxPerRow": 2, + "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, @@ -16476,24 +15323,56 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf)", - "hide": false, + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_miss\"}[1m])))", "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_pending_compaction_bytes", + "legendFormat": "all", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "data", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "filter", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "index", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "bloom prefix", + "metric": "", + "refId": "E", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction pending bytes", + "title": "Block cache hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -16504,7 +15383,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -16512,12 +15391,12 @@ "show": true }, { - "format": "Bps", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", - "show": true + "min": null, + "show": false } ] }, @@ -16528,14 +15407,15 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 480 + "y": 440 }, - "id": 518, + "height": "", + "id": 467, "legend": { "alignAsTable": true, "avg": false, @@ -16569,19 +15449,78 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", + "legendFormat": "total_read", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total_written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "data_insert", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_insert", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_evict", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_insert", + "metric": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_evict", + "metric": "", + "refId": "G", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Read amplication", + "title": "Block cache flow", "tooltip": { "shared": true, "sort": 0, @@ -16597,15 +15536,15 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -16626,9 +15565,9 @@ "h": 8, "w": 12, "x": 12, - "y": 480 + "y": 440 }, - "id": 863, + "id": 468, "legend": { "alignAsTable": true, "avg": false, @@ -16662,19 +15601,50 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_compression_ratio{instance=~\"$instance\", db=\"$db\"}) by (level)", - "hide": false, + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "total_add", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "data_add", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", "intervalFactor": 2, - "legendFormat": "level - {{level}}", + "legendFormat": "filter_add", "metric": "", - "refId": "A", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "index_add", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "add_failures", + "metric": "", + "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compression ratio", + "title": "Block cache operations", "tooltip": { "shared": true, "sort": 0, @@ -16690,11 +15660,11 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16702,7 +15672,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -16714,14 +15684,15 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 488 + "y": 448 }, - "id": 516, + "height": "", + "id": 132, "legend": { "alignAsTable": true, "avg": false, @@ -16755,10 +15726,29 @@ "steppedLine": false, "targets": [ { - "expr": "tikv_engine_num_snapshots{instance=~\"$instance\", db=\"$db\"}", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "read", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\"$instance\", db=\"$db\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "corrupt", "metric": "", "refId": "A", "step": 10 @@ -16767,7 +15757,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Number of snapshots", + "title": "Keys flow", "tooltip": { "shared": true, "sort": 0, @@ -16783,7 +15773,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -16812,9 +15802,9 @@ "h": 8, "w": 12, "x": 12, - "y": 488 + "y": 448 }, - "id": 517, + "id": 131, "legend": { "alignAsTable": true, "avg": false, @@ -16848,11 +15838,11 @@ "steppedLine": false, "targets": [ { - "expr": "tikv_engine_oldest_snapshot_duration{instance=~\"$instance\", db=\"$db\"}", + "expr": "sum(tikv_engine_estimate_num_keys{instance=~\"$instance\", db=\"$db\"}) by (cf)", "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_engine_oldest_snapshot_duration", + "legendFormat": "{{cf}}", + "metric": "tikv_engine_estimate_num_keys", "refId": "A", "step": 10 } @@ -16860,7 +15850,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Oldest snapshots duration", + "title": "Total keys", "tooltip": { "shared": true, "sort": 0, @@ -16876,7 +15866,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -16899,22 +15889,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 1, + "decimals": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 496 + "y": 456 }, - "id": 2002, + "height": "", + "id": 85, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -16939,17 +15932,28 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_num_files_at_level{instance=~\"$instance\", db=\"$db\"}) by (cf, level)", - "format": "time_series", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "cf-{{cf}}, level-{{level}}", - "refId": "A" + "legendFormat": "get", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "scan", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Number files at each level", + "title": "Read flow", "tooltip": { "shared": true, "sort": 0, @@ -16965,11 +15969,11 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -16977,7 +15981,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -16988,28 +15992,31 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 1, + "decimals": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 496 + "y": 456 }, - "id": 2003, + "id": 133, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -17028,24 +16035,38 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", - "format": "time_series", + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", + "intervalFactor": 2, + "legendFormat": "max", + "refId": "A", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", - "refId": "A" + "refId": "B", + "step": 10 }, { - "expr": "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\"$instance\"}[1m]))", - "format": "time_series", + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "intervalFactor": 2, - "legendFormat": "average", - "refId": "B" + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Ingest SST duration seconds", + "title": "Bytes / Read", "tooltip": { "shared": true, "sort": 0, @@ -17061,11 +16082,11 @@ }, "yaxes": [ { - "format": "s", + "format": "decbytes", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17084,23 +16105,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 504 + "y": 464 }, - "id": 2381, + "height": "", + "id": 86, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideZero": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17125,17 +16148,26 @@ "steppedLine": false, "targets": [ { - "expr": "tikv_engine_stall_conditions_changed{instance=~\"$instance\", db=\"$db\"}", - "format": "time_series", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}-{{type}}", - "refId": "B" + "legendFormat": "wal", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "write", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Stall conditions changed of each CF", + "title": "Write flow", "tooltip": { "shared": true, "sort": 0, @@ -17151,11 +16183,11 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17163,7 +16195,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -17175,14 +16207,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 504 + "y": 464 }, - "id": 2451, + "id": 134, "legend": { "alignAsTable": true, "avg": false, @@ -17198,6 +16230,7 @@ "lines": true, "linewidth": 1, "links": [], + "maxPerRow": 2, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -17216,20 +16249,38 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_compaction_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", - "format": "time_series", - "hide": false, + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "intervalFactor": 2, - "legendFormat": "{{cf}} - {{reason}}", - "metric": "", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction reason", + "title": "Bytes / Write", "tooltip": { "shared": true, "sort": 0, @@ -17245,9 +16296,9 @@ }, "yaxes": [ { - "format": "short", + "format": "decbytes", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -17257,48 +16308,35 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } - ] - } - ], - "repeat": "db", - "title": "RocksDB - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 3301, - "panels": [ + ] + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 26 + "y": 472 }, - "id": 3414, + "id": 90, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, + "avg": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17306,44 +16344,51 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "hide": false, "intervalFactor": 2, - "legendFormat": "key_size_avg", - "refId": "A" + "legendFormat": "read", + "refId": "A", + "step": 10 }, { - "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "key_size_0.95", - "refId": "C" + "expr": "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "written", + "refId": "C", + "step": 10 }, { - "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", "hide": false, - "intervalFactor": 1, - "legendFormat": "key_size_0.99", - "refId": "D" + "intervalFactor": 2, + "legendFormat": "flushed", + "refId": "B", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob key size", + "title": "Compaction flow", "tooltip": { "shared": true, "sort": 0, @@ -17359,26 +16404,22 @@ }, "yaxes": [ { - "format": "bytes", + "format": "Bps", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17386,22 +16427,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 26 + "h": 8, + "w": 12, + "x": 12, + "y": 472 }, - "id": 3446, + "id": 127, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, + "avg": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17409,42 +16452,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size", - "refId": "A" - }, - { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size_0.95", - "refId": "B" - }, - { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size_0.99", - "refId": "C" + "expr": "sum(rate(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{cf}}", + "metric": "tikv_engine_pending_compaction_bytes", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob value size", + "title": "Compaction pending bytes", "tooltip": { "shared": true, "sort": 0, @@ -17462,24 +16499,20 @@ { "format": "bytes", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17487,22 +16520,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 32 + "y": 480 }, - "id": 3412, + "id": 518, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17510,42 +16545,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", + "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", + "hide": false, "intervalFactor": 2, - "legendFormat": "seek_micros", - "refId": "A" - }, - { - "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "prev_micros", - "refId": "B" - }, - { - "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "next_micros", - "refId": "C" + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob seek duration", + "title": "Read amplication", "tooltip": { "shared": true, "sort": 0, @@ -17561,11 +16590,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17573,14 +16602,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17588,22 +16613,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 480 }, - "id": 3410, + "id": 863, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, + "avg": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17611,42 +16638,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ - { - "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "blob_gc_micros", - "refId": "A" - }, - { - "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.95", - "refId": "B" - }, - { - "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.99", - "refId": "C" + { + "expr": "avg(tikv_engine_compression_ratio{instance=~\"$instance\", db=\"$db\"}) by (level)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "level - {{level}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC duration", + "title": "Compression ratio", "tooltip": { "shared": true, "sort": 0, @@ -17662,11 +16683,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17674,14 +16695,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17689,22 +16706,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 38 + "y": 488 }, - "id": 3342, + "id": 516, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17712,50 +16731,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "file_bytes_written", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "file_bytes_read", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "keys_read", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", - "format": "time_series", + "expr": "tikv_engine_num_snapshots{instance=~\"$instance\", db=\"$db\"}", + "hide": false, "intervalFactor": 2, - "legendFormat": "keys_written", - "refId": "D" + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob flow", + "title": "Number of snapshots", "tooltip": { "shared": true, "sort": 0, @@ -17771,11 +16776,11 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17783,14 +16788,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17798,22 +16799,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 38 + "h": 8, + "w": 12, + "x": 12, + "y": 488 }, - "id": 3408, + "id": 517, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -17821,49 +16824,36 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "A" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "C" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" + "expr": "tikv_engine_oldest_snapshot_duration{instance=~\"$instance\", db=\"$db\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "tikv_engine_oldest_snapshot_duration", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "GC file write duration", + "title": "Oldest snapshots duration", "tooltip": { "shared": true, "sort": 0, @@ -17879,11 +16869,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "s", "label": null, - "logBase": 2, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17891,14 +16881,10 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -17908,57 +16894,55 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 44 + "y": 496 }, - "id": 3344, + "id": 2002, "legend": { "alignAsTable": true, - "avg": true, - "current": false, - "max": false, - "min": false, + "avg": false, + "current": true, + "max": true, + "min": true, "rightSide": true, "show": true, - "total": true, + "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "gc_num_files", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", + "expr": "avg(tikv_engine_num_files_at_level{instance=~\"$instance\", db=\"$db\"}) by (cf, level)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "gc_num_new_files", - "refId": "C" + "legendFormat": "cf-{{cf}}, level-{{level}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob GC file", + "title": "Number files at each level", "tooltip": { "shared": true, "sort": 0, @@ -17989,11 +16973,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -18003,18 +16983,18 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 44 + "h": 8, + "w": 12, + "x": 12, + "y": 496 }, - "id": 3338, + "id": 2003, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, "max": true, - "min": false, + "min": true, "rightSide": true, "show": true, "total": false, @@ -18024,47 +17004,41 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "seek", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "prev", - "refId": "B" + "expr": "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A" }, { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", + "expr": "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\"$instance\"}[1m]))", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "next", - "refId": "C" + "intervalFactor": 2, + "legendFormat": "average", + "refId": "B" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob seek operations", + "title": "Ingest SST duration seconds", "tooltip": { "shared": true, "sort": 0, @@ -18080,7 +17054,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -18095,11 +17069,7 @@ "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -18109,18 +17079,19 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 6, - "w": 10, + "h": 8, + "w": 12, "x": 0, - "y": 50 + "y": 504 }, - "id": 3340, + "id": 2381, "legend": { "alignAsTable": true, - "avg": true, - "current": false, + "avg": false, + "current": true, + "hideZero": true, "max": true, - "min": false, + "min": true, "rightSide": true, "show": true, "total": false, @@ -18130,64 +17101,34 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_written", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "expr": "tikv_engine_stall_conditions_changed{instance=~\"$instance\", db=\"$db\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "bytes_read", + "legendFormat": "{{instance}}-{{cf}}-{{type}}", "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_overwritten", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_relocated", - "refId": "D" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "keys_relocated", - "refId": "E" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "keys_overwritten", - "refId": "F" } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Blob GC flows", + "title": "Stall conditions changed of each CF", "tooltip": { "shared": true, "sort": 0, @@ -18203,7 +17144,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -18211,18 +17152,14 @@ "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + ] }, { "aliasColors": {}, @@ -18230,22 +17167,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 51 + "h": 8, + "w": 12, + "x": 12, + "y": 504 }, - "id": 3523, + "id": 2451, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -18253,9 +17192,8 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "scopedVars": { @@ -18266,22 +17204,25 @@ } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_compaction_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "live blob size", - "refId": "A" + "legendFormat": "{{cf}} - {{reason}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Live blob size", + "title": "Compaction reason", "tooltip": { "shared": true, "sort": 0, @@ -18297,11 +17238,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -18309,15 +17250,26 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, + ] + } + ], + "repeat": "db", + "title": "RocksDB - $db", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 3301, + "panels": [ { "aliasColors": {}, "bars": false, @@ -18327,16 +17279,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 57 + "y": 27 }, - "id": 3555, + "id": 3414, "legend": { "alignAsTable": true, "avg": true, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, @@ -18360,30 +17312,39 @@ } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "live blob file num", + "legendFormat": "key_size_avg", "refId": "A" }, { - "expr": "avg(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "key_size_0.95", + "refId": "C" + }, + { + "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "obsolete blob file num", - "refId": "B" + "legendFormat": "key_size_0.99", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob file", + "title": "Blob key size", "tooltip": { "shared": true, "sort": 0, @@ -18399,9 +17360,9 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -18429,16 +17390,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 57 + "w": 12, + "x": 12, + "y": 27 }, - "id": 3557, + "id": 3446, "legend": { "alignAsTable": true, "avg": true, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, "rightSide": true, "show": true, @@ -18462,29 +17423,37 @@ } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "live blob file size", + "intervalFactor": 1, + "legendFormat": "value_size", "refId": "A" }, { - "expr": "avg(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "obsolete blob file size", + "intervalFactor": 1, + "legendFormat": "value_size_0.95", "refId": "B" + }, + { + "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "value_size_0.99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob file size", + "title": "Blob value size", "tooltip": { "shared": true, "sort": 0, @@ -18502,47 +17471,25 @@ { "format": "bytes", "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": "db", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "title": "Titan - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 3512, - "panels": [ + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -18552,16 +17499,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 28 + "y": 33 }, - "id": 3513, + "id": 3412, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, @@ -18577,49 +17524,45 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3414, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "key_size_avg", + "legendFormat": "seek_micros", "refId": "A" }, { - "expr": "quantile(0.95, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "key_size_0.95", - "refId": "C" + "legendFormat": "prev_micros", + "refId": "B" }, { - "expr": "quantile(0.99, tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "key_size_0.99", - "refId": "D" + "legendFormat": "next_micros", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob key size", + "title": "Blob seek duration", "tooltip": { "shared": true, "sort": 0, @@ -18635,7 +17578,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "µs", "label": null, "logBase": 2, "max": null, @@ -18665,11 +17608,11 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 28 + "w": 12, + "x": 12, + "y": 33 }, - "id": 3514, + "id": 3410, "legend": { "alignAsTable": true, "avg": true, @@ -18690,39 +17633,37 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3446, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "value_size", + "intervalFactor": 2, + "legendFormat": "blob_gc_micros", "refId": "A" }, { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "value_size_0.95", + "legendFormat": "blob_gc_micros_0.95", "refId": "B" }, { - "expr": "quantile(0.95, tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$db\"})", + "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "value_size_0.99", + "legendFormat": "blob_gc_micros_0.99", "refId": "C" } ], @@ -18730,7 +17671,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob value size", + "title": "GC duration", "tooltip": { "shared": true, "sort": 0, @@ -18746,7 +17687,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "µs", "label": null, "logBase": 2, "max": null, @@ -18776,11 +17717,11 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 34 + "y": 39 }, - "id": 3515, + "id": 3342, "legend": { "alignAsTable": true, "avg": true, @@ -18801,47 +17742,52 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3412, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "seek_micros", + "legendFormat": "file_bytes_written", "refId": "A" }, { - "expr": "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "prev_micros", + "intervalFactor": 2, + "legendFormat": "file_bytes_read", "refId": "B" }, { - "expr": "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "next_micros", + "intervalFactor": 2, + "legendFormat": "keys_read", "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "keys_written", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob seek duration", + "title": "Blob flow", "tooltip": { "shared": true, "sort": 0, @@ -18857,9 +17803,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "Bps", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -18887,16 +17833,16 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 34 + "w": 12, + "x": 12, + "y": 39 }, - "id": 3516, + "id": 3408, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, @@ -18912,47 +17858,52 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3410, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "blob_gc_micros", + "intervalFactor": 1, + "legendFormat": "avg", "refId": "A" }, { - "expr": "quantile(0.95, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.95", + "legendFormat": "99%", "refId": "B" }, { - "expr": "quantile(0.99, tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$db\"})", + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", "format": "time_series", "intervalFactor": 1, - "legendFormat": "blob_gc_micros_0.99", + "legendFormat": "95%", "refId": "C" + }, + { + "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC duration", + "title": "GC file write duration", "tooltip": { "shared": true, "sort": 0, @@ -18998,20 +17949,20 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 40 + "y": 45 }, - "id": 3517, + "id": 3344, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "total": false, + "total": true, "values": true }, "lines": true, @@ -19023,14 +17974,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3342, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19039,39 +17987,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "file_bytes_written", - "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "file_bytes_read", + "legendFormat": "gc_num_files", "refId": "B" }, { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "keys_read", + "legendFormat": "gc_num_new_files", "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "keys_written", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob flow", + "title": "Blob GC file", "tooltip": { "shared": true, "sort": 0, @@ -19087,7 +18021,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -19117,11 +18051,11 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 40 + "w": 12, + "x": 12, + "y": 45 }, - "id": 3518, + "id": 3338, "legend": { "alignAsTable": true, "avg": true, @@ -19142,54 +18076,49 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3408, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_average\"})", + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "seek", "refId": "A" }, { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile99\"})", + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "99%", + "legendFormat": "prev", "refId": "B" }, { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_percentile95\"})", + "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "95%", + "legendFormat": "next", "refId": "C" - }, - { - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$db\", type=\"blob_file_write_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC file write duration", + "title": "Blob seek operations", "tooltip": { "shared": true, "sort": 0, @@ -19205,9 +18134,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "ops", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -19235,20 +18164,20 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 46 + "y": 51 }, - "id": 3519, + "id": 3340, "legend": { "alignAsTable": true, "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, - "total": true, + "total": false, "values": true }, "lines": true, @@ -19260,14 +18189,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3344, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19276,25 +18202,53 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_input_files_count\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "gc_num_files", + "legendFormat": "bytes_written", + "refId": "A" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_read", "refId": "B" }, { - "expr": "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$db\", type=\"gc_output_files_count\"}[2m]))", + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "gc_num_new_files", + "legendFormat": "bytes_overwritten", "refId": "C" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes_relocated", + "refId": "D" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_relocated", + "refId": "E" + }, + { + "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "keys_overwritten", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob GC file", + "title": "Blob GC flows", "tooltip": { "shared": true, "sort": 0, @@ -19310,7 +18264,7 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -19318,7 +18272,7 @@ "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -19340,15 +18294,15 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, - "x": 10, - "y": 46 + "w": 12, + "x": 12, + "y": 51 }, - "id": 3520, + "id": 3523, "legend": { "alignAsTable": true, "avg": true, - "current": false, + "current": true, "max": true, "min": false, "rightSide": true, @@ -19365,14 +18319,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3338, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19381,36 +18332,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_seek\"}[2m]))", + "expr": "avg(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "seek", + "legendFormat": "live blob size", "refId": "A" - }, - { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_prev\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "prev", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$db\", type=\"number_blob_next\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "next", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob seek operations", + "title": "Live blob size", "tooltip": { "shared": true, "sort": 0, @@ -19426,7 +18359,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -19456,15 +18389,15 @@ "fill": 1, "gridPos": { "h": 6, - "w": 10, + "w": 12, "x": 0, - "y": 52 + "y": 57 }, - "id": 3521, + "id": 3555, "legend": { "alignAsTable": true, "avg": true, - "current": false, + "current": true, "max": true, "min": false, "rightSide": true, @@ -19481,14 +18414,11 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatIteration": 1562643407086, - "repeatPanelId": 3340, - "repeatedByRow": true, "scopedVars": { "db": { "selected": false, - "text": "raft", - "value": "raft" + "text": "kv", + "value": "kv" } }, "seriesOverrides": [], @@ -19497,53 +18427,26 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", + "expr": "avg(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "bytes_written", + "legendFormat": "live blob file num", "refId": "A" }, { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_read", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_overwritten\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_overwritten", - "refId": "C" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bytes_relocated", - "refId": "D" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_relocated\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "keys_relocated", - "refId": "E" - }, - { - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_overwritten\"}[2m]))", + "expr": "avg(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "keys_overwritten", - "refId": "F" + "legendFormat": "obsolete blob file num", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Blob GC flows", + "title": "Blob file", "tooltip": { "shared": true, "sort": 0, @@ -19559,7 +18462,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -19567,7 +18470,7 @@ "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -19579,31 +18482,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "repeatIteration": 1562643407086, - "repeatPanelId": 3301, - "scopedVars": { - "db": { - "selected": false, - "text": "raft", - "value": "raft" - } - }, - "title": "Titan - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 2763, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -19612,18 +18491,18 @@ "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "gridPos": { - "h": 7, + "h": 6, "w": 12, - "x": 0, - "y": 505 + "x": 12, + "y": 57 }, - "id": 2696, + "id": 3557, "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, "total": false, @@ -19633,28 +18512,43 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "tikv_allocator_stats{instance=~\"$instance\"}", + "expr": "avg(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$db\"})", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "live blob file size", "refId": "A" + }, + { + "expr": "avg(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$db\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "obsolete blob file size", + "refId": "B" } ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Allocator Stats", + "title": "Blob file size", "tooltip": { "shared": true, "sort": 0, @@ -19670,7 +18564,7 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -19685,11 +18579,15 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "repeat": null, - "title": "Memory", + "repeat": "db", + "title": "Titan - $db", "type": "row" }, { @@ -19698,7 +18596,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 23 }, "id": 2820, "panels": [ @@ -20191,10 +19089,108 @@ ], "title": "Lock manager", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 2763, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 505 + }, + "id": 2696, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "tikv_allocator_stats{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Allocator Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "Memory", + "type": "row" } ], "refresh": "1m", - "schemaVersion": 16, + "schemaVersion": 18, "style": "dark", "tags": [], "templating": { @@ -20298,6 +19294,6 @@ }, "timezone": "browser", "title": "Test-Cluster-TiKV-Details", - "uid": "RDVQiEzZz", + "uid": "RDVQiEzZz1", "version": 2 -} +} \ No newline at end of file