diff --git a/assets/monitoring/grafana/v1alpha1/dashboards-platform.cm.yaml b/assets/monitoring/grafana/v1alpha1/dashboards-platform.cm.yaml deleted file mode 100644 index e20da7ab119..00000000000 --- a/assets/monitoring/grafana/v1alpha1/dashboards-platform.cm.yaml +++ /dev/null @@ -1,21563 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: '{{ .scyllaDBMonitoringName }}-grafana-scylladb-dashboards' -data: - alternator.json: |- - {{`{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_restart", - "datasource": "prometheus", - "enable": true, - "expr": "resets(scylla_gossip_heart_beat[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "node_restart", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "restart", - "type": "tags" - }, - { - "class": "annotation_stall", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_stall_detector_reported[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "stall detector", - "showIn": 0, - "tagKeys": "dc,instance,shard", - "tags": [], - "titleFormat": "Stall found", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "originalTitle": "Scylla Cluster Metrics", - "overwrite": true, - "panels": [ - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
\n [[cluster]]
  The Alternator dashboard is only valid when using Alternator, check the documentation for more information.
", - "mode": "html" - }, - "span": 12, - "title": "", - "transparent": true, - "type": "text" - }, - { - "cacheTimeout": null, - "class": "single_stat_panel", - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 0, - "y": 4 - }, - "id": 2, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 1, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(scylla_scylladb_current_version{job=~\"$cluster|$^\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 40 - } - ], - "thresholds": "", - "title": "Total Nodes", - "transparent": true, - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "class": "single_stat_panel_fail", - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(250, 113, 0, 0.89)", - "rgba(255, 0, 0, 0.9)" - ], - "datasource": "prometheus", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 2, - "y": 4 - }, - "id": 3, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 1, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(scrape_samples_scraped{job=~\"$cluster|$^\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Unreachable", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "title": "Unreachable", - "transparent": true, - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "class": "single_stat_panel_fail", - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(250, 113, 0, 0.89)", - "rgba(255, 0, 0, 0.9)" - ], - "datasource": "prometheus", - "description": "Number of nodes that reported their status as Starting or Joining", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 4, - "y": 4 - }, - "id": 4, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 1, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(scylla_node_operation_mode<=2)OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Joining", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "title": "Joining", - "transparent": true, - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "class": "single_stat_panel_fail", - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(250, 113, 0, 0.89)", - "rgba(255, 0, 0, 0.9)" - ], - "datasource": "prometheus", - "description": "Number of nodes that reported their status as Leaving, Decommissioned, Draining or Drained", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 6, - "y": 4 - }, - "id": 5, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 1, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(scylla_node_operation_mode>3)OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Leaving", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "title": "Leaving", - "transparent": true, - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "class": "percent_panel", - "datasource": "prometheus", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 4 - }, - "id": 6, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "title": "Load", - "type": "timeseries" - }, - { - "class": "small_nodes_table", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 85 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "lcd-gauge" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 101 - }, - { - "id": "displayName", - "value": "Load" - }, - { - "id": "custom.width", - "value": 120 - }, - { - "id": "custom.filterable", - "value": false - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "svr" - }, - "properties": [ - { - "id": "custom.width", - "value": 85 - }, - { - "id": "displayName", - "value": "Version" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 105 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "custom.width", - "value": 105 - }, - { - "id": "displayName", - "value": "Status" - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - }, - { - "color": "rgb(87, 128, 193)", - "value": 3 - }, - { - "color": "red", - "value": 4 - } - ] - } - }, - { - "id": "custom.align", - "value": "center" - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "Starting", - "to": "", - "type": 1, - "value": "1" - }, - { - "from": "", - "id": 2, - "text": "Joining", - "to": "", - "type": 1, - "value": "2" - }, - { - "from": "", - "id": 3, - "text": "Normal", - "to": "", - "type": 1, - "value": "3" - }, - { - "from": "", - "id": 4, - "text": "Leaving", - "to": "", - "type": 1, - "value": "4" - }, - { - "from": "", - "id": 5, - "text": "Decommissioned", - "to": "", - "type": 1, - "value": "5" - }, - { - "from": "", - "id": 6, - "text": "Draining", - "to": "", - "type": 1, - "value": "6" - }, - { - "from": "", - "id": 7, - "text": "Drained", - "to": "", - "type": 1, - "value": "7" - }, - { - "from": "", - "id": 8, - "text": "Moving", - "to": "", - "type": 1, - "value": "8" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "CQL Information Dashboard. Warning indicates that there are potential issues in the CQL Optimization", - "url": "/d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 90 - }, - { - "id": "custom.filterable", - "value": false - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "CQL Info", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "1", - "id": 1, - "text": "CQL Warnings", - "to": "20", - "type": 2, - "value": "" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(87, 128, 193)", - "value": null - }, - { - "color": "dark-orange", - "value": 1 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "displayName", - "value": "CQL" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "OS Information Dashboard, an Error indicates there are OS related errors", - "url": "/d/OS-[[dash_version]]/os-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 90 - }, - { - "id": "custom.filterable", - "value": true - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "OS Info", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "0", - "id": 2, - "text": "OS Errors", - "to": "100000", - "type": 2, - "value": "0" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(87, 128, 193)", - "value": null - }, - { - "color": "dark-orange", - "value": 0.001 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "displayName", - "value": "OS" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "Cordinator and Replica node errors", - "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 80 - }, - { - "id": "custom.filterable", - "value": true - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "", - "id": 2, - "text": "Errors", - "to": "", - "type": 1, - "value": "1" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(87, 128, 193)", - "value": null - }, - { - "color": "dark-orange", - "value": 0.001 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "displayName", - "value": " " - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "Detailed view", - "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 4 - }, - "id": 7, - "scopedVars": { - "dc": { - "selected": false, - "text": "datacenter1", - "value": "datacenter1" - } - }, - "span": 4, - "targets": [ - { - "expr": "0*scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"} + on (instance) group_left() scylla_node_operation_mode{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - }, - { - "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} ) by (instance)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "expr": "sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "expr": "sum(scylla_errors:nodes_total{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 0", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "expr": "(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - } - ], - "title": "Nodes", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "instance", - "svr", - "Value #A", - "Value #B", - "Value #C", - "Value #D" - ] - } - } - }, - { - "id": "seriesToColumns", - "options": { - "byField": "instance" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Value #A": 4, - "Value #B": 5, - "Value #C": 2, - "Value #D": 1, - "instance": 0, - "svr": 3 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 8, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 4, - "targets": [ - { - "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Size by $by", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of Alternator Actions", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 9, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Total Actions", - "type": "timeseries" - }, - { - "class": "alert_table", - "columns": [], - "datasource": "alertmanager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 10, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 4, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": true, - "linkTooltip": "Jump to the see the node", - "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_4}&from=${__cell_0}", - "pattern": "Time", - "type": "date" - }, - { - "class": "hidden_column", - "pattern": "severity", - "type": "hidden" - }, - { - "class": "hidden_column", - "pattern": "alertname", - "type": "hidden" - }, - { - "class": "hidden_column", - "pattern": "cluster", - "type": "hidden" - }, - { - "class": "hidden_column", - "pattern": "monitor", - "type": "hidden" - }, - { - "class": "hidden_column", - "pattern": "summary", - "type": "hidden" - }, - { - "alias": "Instance", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the see the node", - "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell}", - "mappingType": 1, - "pattern": "instance", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "active": true, - "annotations": true, - "filters": "job!=\"scylla_manager\",advisor=\"\"", - "legendFormat": "{{description}}", - "refId": "A", - "target": "Query" - } - ], - "title": "Active Alerts", - "transform": "table", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Time", - "summary", - "instance" - ] - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Time": 0, - "instance": 1, - "summary": 2 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 11, - "panels": [], - "repeat": "", - "title": "Data Plane Actions", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Data Plane Actions

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 12, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 13, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetItem\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "GetItem by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 14, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"PutItem\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "PutItem by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 15, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"UpdateItem\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "UpdateItem by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 25 - }, - "id": 16, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteItem\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "DeleteItem by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 25 - }, - "id": 17, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchWriteItem\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "BatchWriteItem by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "dashversion": [ - ">4.4", - ">2021.1" - ], - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 18, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchGetItem\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "BatchGetItem by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 31 - }, - "id": 19, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Query\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Query by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 31 - }, - "id": 20, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Scan\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Scan by [[by]]", - "type": "timeseries" - }, - { - "class": "plain_text", - "content": "

Data Plane Latencies

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 21, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 22, - "panels": [], - "repeat": "alternator_latency_ops", - "title": "$alternator_latency_ops", - "type": "row" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 40 - }, - "id": 23, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Completed $alternator_latency_ops", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 40 - }, - "id": 24, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]]) + 1)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average $alternator_latency_ops latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 40 - }, - "id": 25, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile $alternator_latency_ops latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 40 - }, - "id": 26, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile $alternator_latency_ops latency by [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 27, - "panels": [], - "repeat": "", - "title": "Streams", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Streams Actions

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 28, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 49 - }, - "id": 29, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListStreams\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "ListStreams by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 49 - }, - "id": 30, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeStream\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "DescribeStream by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 49 - }, - "id": 31, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetShardIterator\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "GetShardIterator by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 55 - }, - "id": 32, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetRecords\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "GetRecords by [[by]]", - "type": "timeseries" - }, - { - "class": "plain_text", - "content": "

Streams Latencies

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 33, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 63 - }, - "id": 34, - "panels": [], - "repeat": "alternator_streams_latency_ops", - "title": "$alternator_streams_latency_ops", - "type": "row" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 64 - }, - "id": 35, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Completed $alternator_streams_latency_ops", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 64 - }, - "id": 36, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]]) + 1)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average $alternator_streams_latency_ops latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 64 - }, - "id": 37, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile $alternator_streams_latency_ops latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 64 - }, - "id": 38, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile $alternator_streams_latency_ops latency by [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 39, - "panels": [], - "repeat": "", - "title": "Control plane", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Control Plane Actions

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 71 - }, - "id": 40, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 73 - }, - "id": 41, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"CreateTable\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "CreateTable by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 73 - }, - "id": 42, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteTable\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "DeleteTable by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 73 - }, - "id": 43, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeTable\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "DescribeTable by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 79 - }, - "id": 44, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListTables\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "ListTables by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 79 - }, - "id": 45, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeEndpoints\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "DescribeEndpoints by [[by]]", - "type": "timeseries" - }, - { - "class": "text_panel", - "content": "## ", - "dashversion": [ - "<5.1", - "<2022.2" - ], - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 79 - }, - "id": 46, - "isNew": true, - "links": [], - "mode": "markdown", - "options": {}, - "span": 4, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "plain_text", - "content": "

Cache

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 12, - "x": 0, - "y": 85 - }, - "id": 47, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 6, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "plain_text", - "content": "

Timeouts

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 12, - "x": 12, - "y": 85 - }, - "id": 48, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 6, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of rows that were read from the cache, without needing to be fetched from storage.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 87 - }, - "id": 49, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cache Hits", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of rows that were not present in the cache, and had to be fetched from storage.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 87 - }, - "id": 50, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cache Misses", - "type": "timeseries" - }, - { - "class": "wpm_panel", - "datasource": "prometheus", - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:writes/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 87 - }, - "id": 51, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Write Timeouts/Minutes by [[by]]", - "type": "timeseries" - }, - { - "class": "rpm_panel", - "datasource": "prometheus", - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/m" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 87 - }, - "id": 52, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Read Timeouts/Minutes by [[by]]", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 93 - }, - "id": 53, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Your Panels

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 95 - }, - "id": 54, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 95 - }, - "id": 55, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 101 - }, - "id": 56, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - 4.2.1
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "5.0" - ], - "templating": { - "list": [ - { - "allValue": null, - "class": "by_template_var", - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "error": null, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "instance,shard", - "value": "instance,shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(scylla_reactor_utilization, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "node", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "shard", - "multi": true, - "name": "shard", - "options": [], - "query": "label_values(scylla_reactor_utilization,shard)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "aggregation_function", - "current": { - "tags": [], - "text": "sum", - "value": "sum" - }, - "hide": 0, - "includeAll": false, - "label": "Function", - "multi": false, - "name": "func", - "options": [ - { - "selected": true, - "text": "sum", - "value": "sum" - }, - { - "selected": false, - "text": "avg", - "value": "avg" - }, - { - "selected": false, - "text": "max", - "value": "max" - }, - { - "selected": false, - "text": "min", - "value": "min" - }, - { - "selected": false, - "text": "stddev", - "value": "stddev" - }, - { - "selected": false, - "text": "stdvar", - "value": "stdvar" - } - ], - "query": "sum,avg,max,min,stddev,stdvar", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5-0", - "value": "5-0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "dash_version", - "options": [ - { - "selected": true, - "text": "5-0", - "value": "5-0" - } - ], - "query": "5-0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5.0", - "value": "5.0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "scylla_version", - "options": [ - { - "selected": true, - "text": "5.0", - "value": "5.0" - } - ], - "query": "5.0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "4.2.1", - "value": "4.2.1" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "4.2.1", - "value": "4.2.1" - } - ], - "query": "4.2.1", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "All", - "value": "$__all" - }, - "error": null, - "hide": 2, - "includeAll": true, - "label": null, - "multi": true, - "name": "alternator_latency_ops", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "GetItem", - "value": "GetItem" - }, - { - "selected": false, - "text": "PutItem", - "value": "PutItem" - }, - { - "selected": false, - "text": "UpdateItem", - "value": "UpdateItem" - }, - { - "selected": false, - "text": "DeleteItem", - "value": "DeleteItem" - } - ], - "query": "GetItem,PutItem,UpdateItem,DeleteItem", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "All", - "value": "$__all" - }, - "error": null, - "hide": 2, - "includeAll": true, - "label": null, - "multi": true, - "name": "alternator_streams_latency_ops", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "GetRecords", - "value": "GetRecords" - } - ], - "query": "GetRecords", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "datasource": "prometheus", - "definition": "scylla_alternator_total_operations{cluster=~\"$cluster|$^\"}", - "description": null, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "no_alternator", - "options": [], - "query": { - "query": "scylla_alternator_total_operations{cluster=~\"$cluster|$^\"}", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/^(scylla_alternator_total_operations)/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Alternator", - "uid": "alternator-5-0", - "version": 1}`}} - advanced.json: |- - {{`{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_restart", - "datasource": "prometheus", - "enable": true, - "expr": "resets(scylla_gossip_heart_beat[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "node_restart", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "restart", - "type": "tags" - }, - { - "class": "annotation_stall", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_stall_detector_reported[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "stall detector", - "showIn": 0, - "tagKeys": "dc,instance,shard", - "tags": [], - "titleFormat": "Stall found", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "overwrite": true, - "panels": [ - { - "class": "text_panel", - "dashproductreject": "no-version-check", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 9, - "x": 0, - "y": 0 - }, - "id": 1, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "", - "mode": "html" - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 4, - "x": 9, - "y": 0 - }, - "id": 2, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "# " - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 4, - "x": 13, - "y": 0 - }, - "id": 3, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "# " - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 7, - "x": 17, - "y": 0 - }, - "id": 4, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "# " - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The number of nodes configured in the cluster.", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 0, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "count(scylla_scylladb_current_version{job=~\"$cluster|$^\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "# Nodes", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The number of unreachable nodes.\nUsually because a machine is down or unreachable.", - "fieldConfig": { - "defaults": { - "custom": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - } - } - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 2, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "(count(scrape_samples_scraped{job=~\"$cluster|$^\", cluster=~\"$cluster|$^\"}==0) OR vector(0))", - "intervalFactor": 1, - "legendFormat": "Offline ", - "refId": "A", - "step": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Unreachable", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The number of joining and leaving nodes.\nThe number of nodes that are up but not actively part of the cluster, either because they are still joining or because they are leaving.", - "fieldConfig": { - "defaults": { - "custom": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - } - } - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 4, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "count(scylla_node_operation_mode!=3)OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Offline ", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "Inactive", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "from": "", - "id": 0, - "text": "Backup", - "to": "", - "type": 1, - "value": "2" - }, - { - "from": "", - "id": 1, - "text": "Repair", - "to": "", - "type": 1, - "value": "1" - }, - { - "from": "", - "id": 2, - "text": "Online", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "", - "id": 2, - "text": "Offline", - "to": "", - "type": 1, - "value": "-1" - } - ], - "noValue": " Offline", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - } - } - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 6, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "(sum(scylla_manager_task_active_count{type=~\"repair\",cluster=~\"$cluster|$^\"}) or on() vector(0)) + (sum(scylla_manager_task_active_count{type=~\"backup\",cluster=~\"$cluster|$^\"})*2 or on() vector(0)) + (sum(scylla_manager_server_current_version{}) or on() vector(-1))", - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Manager", - "type": "stat" - }, - { - "class": "vertical_lcd", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 1, - "x": 8, - "y": 1 - }, - "id": 9, - "options": { - "displayMode": "lcd", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "(sum(avg(scylla_manager_repair_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"repair\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0)) + (sum(avg(scylla_manager_backup_percent_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"backup\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0))", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "", - "type": "bargauge" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "Average Write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 9, - "y": 1 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0) or on() (sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 11, - "y": 1 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "99% Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "Average Read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 13, - "y": 1 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0) or on() (sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 15, - "y": 1 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "99% Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "si:" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 17, - "y": 1 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests/s", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 20, - "y": 1 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Load", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 1 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Timeouts", - "type": "stat" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 17, - "panels": [], - "repeat": "", - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "content": "

IO Queue Information

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 18, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 19, - "panels": [], - "repeat": "classes", - "title": "$classes", - "type": "row" - }, - { - "class": "seconds_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 9 - }, - "id": 20, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "IO queue {{[[by]]}}", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "$classes I/O Queue delay by [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 9 - }, - "id": 21, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "max(scylla_io_queue_queue_length{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "queue length {{[[by]]}}", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "$classes Queue length by [[by]]", - "type": "timeseries" - }, - { - "class": "bps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 9 - }, - "id": 22, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "$classes I/O Queue bandwidth by [[by]]", - "type": "timeseries" - }, - { - "class": "iops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "iops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 9 - }, - "id": 23, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "$classes I/O Queue IOPS by [[by]]", - "type": "timeseries" - }, - { - "class": "seconds_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 15 - }, - "id": 24, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Disk {{[[by]]}}", - "metric": "scylla_io_queue_total_exec_sec", - "refId": "A", - "step": 30 - } - ], - "title": "Disk $classes I/O Queue delay by [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 15 - }, - "id": 25, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "max(scylla_io_queue_disk_queue_length{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "disk queue length {{[[by]]}}", - "metric": "scylla_io_queue_disk_queue_length", - "refId": "A", - "step": 30 - } - ], - "title": "DISK $classes Queue length by [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 26, - "panels": [], - "repeat": "", - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Information by Task Group

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 27, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 28, - "panels": [], - "repeat": "group", - "title": "$group", - "type": "row" - }, - { - "class": "percentunit_panel", - "datasource": "prometheus", - "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. This graph shows how much time was spent in $group group", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 25 - }, - "id": 29, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Time used by [[by]] - $group", - "type": "timeseries" - }, - { - "class": "percentunit_panel", - "datasource": "prometheus", - "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer. This may cause latency issues", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 25 - }, - "id": 30, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Time spent in task quota violations by [[by]] - $group", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Shares assigned to the $group. Shares determine how Scylla reactor distributes the task quotas between groups (Higher share gets more quotas)", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 31, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 4, - "targets": [ - { - "expr": "$func(scylla_scheduler_shares{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Scheduler shares [[by]] - $group", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 32, - "panels": [], - "repeat": "", - "title": "Internal node errors", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Internal node Errors - $cluster

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 33, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "rps_panel", - "datasource": "prometheus", - "description": "Number of Read requests that failed due to an 'unavailable' error", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 34 - }, - "id": 34, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 4, - "targets": [ - { - "expr": "avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Local Reads Error by [[by]]", - "type": "timeseries" - }, - { - "class": "wps_panel", - "datasource": "prometheus", - "description": "Number of write requests that failed due to an 'unavailable' error", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:writes/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 34 - }, - "id": 35, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 4, - "targets": [ - { - "expr": "avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Local Write Error by [[by]]", - "type": "timeseries" - }, - { - "class": "text_panel", - "content": "## ", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 34 - }, - "id": 36, - "isNew": true, - "links": [], - "mode": "markdown", - "options": {}, - "span": 4, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "rps_panel", - "datasource": "prometheus", - "description": "Number of Read requests that failed due to an 'unavailable' error", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 40 - }, - "id": 37, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 4, - "targets": [ - { - "expr": "avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Reads Unavailable Error by [[by]]", - "type": "timeseries" - }, - { - "class": "wps_panel", - "datasource": "prometheus", - "description": "Number of write requests that failed on a local Node", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:writes/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 40 - }, - "id": 38, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 4, - "targets": [ - { - "expr": "avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Write Unavailable Error by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of write requests that failed on a local Node", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 40 - }, - "id": 39, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Range Unavailable Error by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of AIO Errors", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 46 - }, - "id": 40, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "AIO Error by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Total number of abandoned failed futures, futures destroyed while still containing an exception.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 46 - }, - "id": 41, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Ignored Future By [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of C++ exceptions thrown.\n\n An exception by itself does not indicate a problem", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 46 - }, - "id": 42, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 4, - "targets": [ - { - "expr": "sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "C++ Exceptions [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 43, - "panels": [], - "repeat": "", - "title": "Commit Log", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Commit log Information

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 53 - }, - "id": 44, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "description": "Holds the size of disk space in bytes reserved for data so far. A too high value indicates that we have some bottleneck in the writing to sstables path", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 55 - }, - "id": 45, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_disk_total_bytes{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Avg reserved disk space by [[by]]", - "type": "timeseries" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "description": "Holds the size of disk space in bytes used for data so far. A too high value indicates that we have some bottleneck in the writing to sstables path", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 55 - }, - "id": 46, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_disk_active_bytes{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Avg used disk space by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Counts a number of times the flush() method was called for a file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 55 - }, - "id": 47, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(rate(scylla_commitlog_flush{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Avg flush by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Holds the current number of segments", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 55 - }, - "id": 48, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_segments{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Segments by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Counts a number of times a flush limit was exceeded. A non-zero value indicates that there are too many pending flush operations (see pending_flushes) and some of them will be blocked till the total amount of pending flush operations drops below 5.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 61 - }, - "id": 49, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(rate(scylla_commitlog_flush_limit_exceeded{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Avg flush limit exceeded by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Holds the number of currently pending allocations. A non-zero value indicates that we have a bottleneck in the disk write flow.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 61 - }, - "id": 50, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_pending_allocations{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Pending allocations by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Counts a number of requests blocked due to memory pressure. A non-zero value indicates that the commitlog memory quota is not enough to serve the required amount of requests.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 61 - }, - "id": 51, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_pending_flushes{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Pending flush by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Holds the current number of unused segments. A non-zero value indicates that the disk write path became temporary slow.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 61 - }, - "id": 52, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_unused_segments{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Unused segments by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Holds the number of not closed segments that still have some free space. This value should not get too high.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 67 - }, - "id": 53, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "avg(scylla_commitlog_allocating_segments{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Allocating segments by [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 73 - }, - "id": 54, - "panels": [], - "repeat": "", - "title": "Your panels", - "type": "row" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 55, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Your Panels

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 56, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 57, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 82 - }, - "id": 58, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - 4.2.1
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "5.0" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "Shard", - "value": "instance,shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(scylla_reactor_utilization, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "node", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "shard", - "multi": true, - "name": "shard", - "options": [], - "query": "label_values(scylla_reactor_utilization,shard)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 2, - "includeAll": true, - "label": "classes", - "multi": true, - "name": "classes", - "options": [], - "query": "label_values(scylla_io_queue_delay,class)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 2, - "includeAll": true, - "label": "group", - "multi": true, - "name": "group", - "options": [], - "query": "label_values(scylla_scheduler_time_spent_on_task_quota_violations_ms,group)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "class": "adhoc_filter", - "datasource": "prometheus", - "error": null, - "filters": [], - "hide": 0, - "label": "ad hoc", - "name": "adhoc", - "skipUrlSync": false, - "type": "adhoc" - }, - { - "allValue": null, - "class": "aggregation_function", - "current": { - "tags": [], - "text": "avg", - "value": "avg" - }, - "hide": 0, - "includeAll": false, - "label": "Function", - "multi": false, - "name": "func", - "options": [ - { - "selected": true, - "text": "sum", - "value": "sum" - }, - { - "selected": false, - "text": "avg", - "value": "avg" - }, - { - "selected": false, - "text": "max", - "value": "max" - }, - { - "selected": false, - "text": "min", - "value": "min" - }, - { - "selected": false, - "text": "stddev", - "value": "stddev" - }, - { - "selected": false, - "text": "stdvar", - "value": "stdvar" - } - ], - "query": "sum,avg,max,min,stddev,stdvar", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5.0", - "value": "5.0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "scylla_version", - "options": [ - { - "selected": true, - "text": "5.0", - "value": "5.0" - } - ], - "query": "5.0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "4.2.1", - "value": "4.2.1" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "4.2.1", - "value": "4.2.1" - } - ], - "query": "4.2.1", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Advanced", - "uid": "advanced-5-0", - "version": 5}`}} - cql.json: |- - {{`{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_restart", - "datasource": "prometheus", - "enable": true, - "expr": "resets(scylla_gossip_heart_beat[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "node_restart", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "restart", - "type": "tags" - }, - { - "class": "annotation_stall", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_stall_detector_reported[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "stall detector", - "showIn": 0, - "tagKeys": "dc,instance,shard", - "tags": [], - "titleFormat": "Stall found", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "originalTitle": "CQL", - "overwrite": true, - "panels": [ - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "repeat": "", - "title": "", - "type": "row" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
\n\n [[cluster]]
", - "mode": "html" - }, - "span": 12, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 3, - "panels": [], - "repeat": "", - "title": "CQL By User", - "type": "row" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 4, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

CQL By User - Coordinator

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL INSERT commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 7 - }, - "id": 5, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Insert", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL SELECT commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 7 - }, - "id": 6, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Reads", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL DELETE commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 7 - }, - "id": 7, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Deletes", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL UPDATE commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 7 - }, - "id": 8, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Updates", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "amount of CQL connections currently established", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 13 - }, - "id": 9, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(scylla_transport_current_connections{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Client CQL connections by [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL batches command, each batched command is counted once", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 13 - }, - "id": 10, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Batches by [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL command batched. Each batch would add the number of commands inside the batch", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 13 - }, - "id": 11, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Command In Batches by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Counts the number of SELECT statements with BYPASS CACHE option", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 13 - }, - "id": 12, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "BYPASS CACHE", - "type": "timeseries" - }, - { - "class": "graph_panel", - "dashversion": [ - ">4.4", - ">2021.1" - ], - "datasource": "prometheus", - "description": "CQL errors by type, only active errors are shown", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 19 - }, - "id": 13, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_transport_cql_errors_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]],type) >0", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Errors [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL row reads", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 19 - }, - "id": 14, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Row Reads [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of reads using secondary indexes", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 19 - }, - "id": 15, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Secondary indexes Reads [[by]]", - "type": "timeseries" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 16, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "CQL System tables
The following information is based on Scylla Plugin configuration, check the documentation for more details on how to enable it.", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "collapsible_row_panel", - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 17, - "panels": [ - { - "class": "single_value_table", - "columns": [ - { - "text": "Avg", - "value": "avg" - } - ], - "datasource": "scylla-datasource", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 18, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 12, - "targets": [ - { - "dashversion": [ - ">4.4", - ">2020.1" - ], - "queryHost": "$node", - "queryText": "select address, port, shard_id, connection_stage, client_type, ssl_enabled, username, driver_name, driver_version, protocol_version from system.clients", - "refId": "A" - } - ], - "title": "Connection Table", - "transform": "table", - "type": "table" - } - ], - "repeat": "", - "title": "Connection", - "type": "row" - }, - { - "class": "collapsible_row_panel", - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 19, - "panels": [ - { - "class": "single_value_table", - "columns": [ - { - "text": "Avg", - "value": "avg" - } - ], - "datasource": "scylla-datasource", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "row_size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 20, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 12, - "targets": [ - { - "queryHost": "$node", - "queryText": "select keyspace_name, table_name,partition_key, clustering_key, row_size from system.large_rows", - "refId": "A" - } - ], - "title": "Large Rows", - "transform": "table", - "type": "table" - } - ], - "repeat": "", - "title": "Large Rows", - "type": "row" - }, - { - "class": "collapsible_row_panel", - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 21, - "panels": [ - { - "class": "single_value_table", - "columns": [ - { - "text": "Avg", - "value": "avg" - } - ], - "datasource": "scylla-datasource", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "cell_size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 22, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 12, - "targets": [ - { - "queryHost": "$node", - "queryText": "select keyspace_name, table_name,partition_key, clustering_key, column_name,cell_size from system.large_cells", - "refId": "A" - } - ], - "title": "Large Cells", - "transform": "table", - "type": "table" - } - ], - "repeat": "", - "title": "Large Cells", - "type": "row" - }, - { - "class": "collapsible_row_panel", - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 23, - "panels": [ - { - "class": "single_value_table", - "columns": [ - { - "text": "Avg", - "value": "avg" - } - ], - "datasource": "scylla-datasource", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "partition_size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 24, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 12, - "targets": [ - { - "queryHost": "$node", - "queryText": "select keyspace_name, table_name,partition_key, partition_size from system.large_partitions", - "refId": "A" - } - ], - "title": "Large Partitions", - "transform": "table", - "type": "table" - } - ], - "repeat": "", - "title": "Large Partitions", - "type": "row" - }, - { - "class": "collapsible_row_panel", - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 25, - "panels": [ - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 57 - }, - "id": 26, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

CQL Internal - Coordinator

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL INSERT commands generated by intenal operations", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 59 - }, - "id": 27, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Internal Insert", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL SELECT commands generated by intenal operations", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 59 - }, - "id": 28, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Internal Reads", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL DELETE commands generated by intenal operations", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 59 - }, - "id": 29, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Deletes", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL UPDATE commands generated by intenal operations", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 59 - }, - "id": 30, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Updates", - "type": "timeseries" - } - ], - "repeat": "", - "title": "CQL Internal", - "type": "row" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 31, - "panels": [], - "repeat": "", - "title": "LWT", - "type": "row" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 66 - }, - "id": 32, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

LWT

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 68 - }, - "id": 33, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Insert", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 68 - }, - "id": 34, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Deletes", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 68 - }, - "id": 35, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Updates", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL batches command, each batched command is counted once", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 68 - }, - "id": 36, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Batches by [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 37, - "panels": [], - "repeat": "", - "title": "Optimization", - "type": "row" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 75 - }, - "id": 38, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Optimization

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "All of the requests should be prepared\n\nPrepared statements remove the overhead of parsing the query every time and allow optimal routing of requests from client to server", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 0, - "y": 77 - }, - "id": 39, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "floor(100 *sum(cql:non_system_prepared1m)/ (sum(cql:all_shardrate1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) - sum(cql:all_system_shardrate1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "CQL Non-Prepared Statements", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "All of the requests should be prepared\n", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 2, - "y": 77 - }, - "id": 40, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(cql:non_system_prepared1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "CQL Non-Prepared Statements", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "All requests should be paged\n\nNon Paged request sources:\n- Client modifying the fetch size\n\nNon Paged requests require reading all the results and returning them in a single request.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 8, - "y": 77 - }, - "id": 41, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "100 * ((sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))/sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Non-Paged CQL Reads", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Non-Paged requests require reading all the results and returning them in a single request", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 10, - "y": 77 - }, - "id": 42, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Non-Paged CQL Reads", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "All of the requests should be Token Aware\n\nNon Token Aware requests sources:\n* Non-Prepared Stamements\n* Client not using a Token Aware load balancing policy\n\nTokenAware requests are sent to a Scylla node that is also a replica. Token Un-Aware requests require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 16, - "y": 77 - }, - "id": 43, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "scalar(sum(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) <=bool 0)*scalar(sum(rate(scylla_storage_proxy_replica_received_counter_updates{cluster=~\"$cluster|$^\"}[1m]))<=bool 0) *(100 - clamp_max(100*(sum(rate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) + sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/(sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) + 1,100)) OR vector(0)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Non-Token Aware", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Requests that are not token aware indicates that requests are not routed to the right node, which require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 77 - }, - "id": 44, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "scalar(sum(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) <=bool 0)*scalar(sum(rate(scylla_storage_proxy_replica_received_counter_updates{cluster=~\"$cluster|$^\"}[1m]))<=bool 0) *clamp_min(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) + rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) + rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]),0)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Non-Token Aware Queries", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "Reversed CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with ORDER BY that is different from the \"CLUSTERING ORDER BY\" of the table\nAlternatives:\n\n* Denormalize your data (use a Materialized View)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 0, - "y": 83 - }, - "id": 45, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "100 * sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Reversed CQL Reads", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Reversed CQL Reads entail additional processing on server side and should be avoided", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 2, - "y": 83 - }, - "id": 46, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Reversed CQL Reads", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "ALLOW FILTERING CQL Reads, the percentage of read requests with 'ALLOW FILTERING'\n\nALLOW FILTERING CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned - check \n\"ALLOW FILTERING CQL Read Filtered Rows to check what percentage of the data is used\"\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 8, - "y": 83 - }, - "id": 47, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "100 * sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "ALLOW FILTERING CQL Reads", - "transparent": true, - "type": "gauge" - }, - { - "class": "rps_panel", - "datasource": "prometheus", - "description": "Read requests with ALLOW FILTERING\n\nALLOW FILTERING CQL Reads entail additional processing on server side and should be avoided", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 10, - "y": 83 - }, - "id": 48, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "ALLOW FILTERING CQL Reads", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "ALLOW FILTERING Filtered rows, the percentage of rows that were read and then filtered.\n\nALLOW FILTERING CQL Reads entail additional processing on server side. \nReading a row and then filter it is a waste of resources.\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 16, - "y": 83 - }, - "id": 49, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "100 * sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) /sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "ALLOW FILTERING Filtered Rows", - "transparent": true, - "type": "gauge" - }, - { - "class": "rps_panel", - "datasource": "prometheus", - "description": "CQL Queries with ALLOW FILTERING should be avoided.\nDropped rows are rows that were read but were filtered by the server.\nWhen dropped rows is relatively high you should consider the alternatives", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 83 - }, - "id": 50, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "rows read $node $shard", - "refId": "A" - }, - { - "expr": "sum(rate(scylla_cql_filtered_rows_matched_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "rows matched $node $shard", - "refId": "B" - }, - { - "expr": "sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "rows dropped $node $shard", - "refId": "C" - } - ], - "title": "ALLOW FILTERING CQL Read Filtering", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "Range scans should typically by pass the cache.\n\n Add BYPASS CACHE to your select queries.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 0, - "y": 89 - }, - "id": 51, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "floor(100 *sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))/(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) )) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Range Scans", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Range scans should typically by pass the cache.\n\n Add BYPASS CACHE to your select queries.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 2, - "y": 89 - }, - "id": 52, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Range Scans Without BYPASS CACHE ", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 8, - "y": 89 - }, - "id": 53, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "CQL ANY Queries", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 10, - "y": 89 - }, - "id": 54, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "CQL ANY CL Queries", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 16, - "y": 89 - }, - "id": 55, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "CQL ALL CL Queries", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 89 - }, - "id": 56, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "CQL ALL CL Queries", - "type": "timeseries" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 57, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Cross DC Information

This section is relevant only if you have more than one DC
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "How many Queries use Consistency level ONE\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 0, - "y": 98 - }, - "id": 58, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "CQL ONE Queries", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "How many Queries use Consistency level ONE\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 2, - "y": 98 - }, - "id": 59, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "CQL ONE CL Queries", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "How many Queries use Consistency level QUORUM\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 8, - "y": 98 - }, - "id": 60, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "span": 1, - "targets": [ - { - "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "CQL QUORUM CL Queries", - "transparent": true, - "type": "gauge" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "How many Queries use Consistency level QUORUM\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 10, - "y": 98 - }, - "id": 61, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "CQL QUORUM CL Queries", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "class": "gauge_errors_panel", - "datasource": "prometheus", - "description": "Cross DC traffic may cause additional latencies and network loads and in most cases, should be avoided.\n\nCross DC Read requests sources:\n- Consistency Level that is not LOCAL_XXX\n- Tables with read_repair_chance > 0\n\nNote:\n- If requests are supposed to be DC local - verify client is using a DCAware policy and a LOCAL_XX consistency level", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 4 - }, - { - "color": "#d44a3a", - "value": 10 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 104 - }, - "id": 62, - "links": [], - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.3.5", - "repeat": "dc", - "span": 2, - "targets": [ - { - "expr": "100*(sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) - sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", datacenter=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Cross DC read requests $dc", - "transparent": true, - "type": "gauge" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 110 - }, - "id": 63, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - 4.2.1
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "5.0" - ], - "templating": { - "list": [ - { - "allValue": null, - "class": "by_template_var", - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "error": null, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "instance,shard", - "value": "instance,shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(scylla_reactor_utilization, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "node", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "shard", - "multi": true, - "name": "shard", - "options": [], - "query": "label_values(scylla_reactor_utilization,shard)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5-0", - "value": "5-0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "dash_version", - "options": [ - { - "selected": true, - "text": "5-0", - "value": "5-0" - } - ], - "query": "5-0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5.0", - "value": "5.0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "scylla_version", - "options": [ - { - "selected": true, - "text": "5.0", - "value": "5.0" - } - ], - "query": "5.0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "4.2.1", - "value": "4.2.1" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "4.2.1", - "value": "4.2.1" - } - ], - "query": "4.2.1", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Scylla CQL", - "uid": "cql-5-0", - "version": 1}`}} - detailed.json: |- - {{`{ - "annotations": { - "class": "default_annotations", - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_restart", - "datasource": "prometheus", - "enable": true, - "expr": "resets(scylla_gossip_heart_beat[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "node_restart", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "restart", - "type": "tags" - }, - { - "class": "annotation_stall", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_stall_detector_reported[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "stall detector", - "showIn": 0, - "tagKeys": "dc,instance,shard", - "tags": [], - "titleFormat": "Stall found", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - }, - { - "class": "annotation_manager_task", - "datasource": "prometheus", - "enable": true, - "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", - "hide": false, - "iconColor": "#73BF69", - "limit": 100, - "name": "Task", - "showIn": 0, - "tagKeys": "type", - "tags": [], - "titleFormat": "Running", - "type": "tags" - }, - { - "class": "annotation_hints_writes", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_hints_manager_written[$__rate_interval])>0", - "hide": false, - "iconColor": "rgb(255, 176, 0, 128)", - "limit": 100, - "name": "Hints Write", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Hints write", - "type": "tags" - }, - { - "class": "annotation_hints_sent", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_hints_manager_written[$__rate_interval])>0", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "Hints Sent", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Hints Sent", - "type": "tags" - }, - { - "class": "mv_building", - "datasource": "prometheus", - "enable": true, - "expr": "sum(scylla_view_builder_builds_in_progress)>0", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "MV", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Materialized View built", - "type": "tags" - }, - { - "class": "ops_annotation", - "datasource": "prometheus", - "enable": true, - "expr": "10*min(scylla_node_ops_finished_percentage) by (ops, dc,instance) < 10", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "ops", - "showIn": 0, - "tagKeys": "ops,dc,instance", - "tags": [], - "titleFormat": "Operation", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "overwrite": true, - "panels": [ - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 1, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
[[cluster]]
Keyspace and table level metrics are not available. To enable, set enable-keyspace-column-family-metrics to true in scylla.yaml. Note this has a significant effect on the monitoring stack sizing.
", - "mode": "html" - }, - "span": 12, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "collapsible_row_panel", - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 2, - "panels": [ - { - "class": "wps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:writes/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 3, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_column_family_write_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 4 - }, - "id": 4, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "wlatencyaks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average write latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 4 - }, - "id": 5, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "wlatencyp95ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile write latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 4 - }, - "id": 6, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "wlatencyp99ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile write latency by [[by]]", - "type": "timeseries" - }, - { - "class": "rps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 10 - }, - "id": 7, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_column_family_read_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 10 - }, - "id": 8, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "rlatencyaks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average read latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 10 - }, - "id": 9, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "rlatencyp95ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile read latency by [[by]]", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 10 - }, - "id": 10, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "rlatencyp99ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile read latency by [[by]]", - "type": "timeseries" - } - ], - "repeat": "table", - "title": "Latencies - $ks:$table", - "type": "row" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 11, - "panels": [], - "repeat": "", - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 12, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Your Panels

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 13, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 14, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 15, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - 4.2.1
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "5.0" - ], - "templating": { - "list": [ - { - "allValue": null, - "class": "by_template_var", - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "error": null, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "instance,shard", - "value": "instance,shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(scylla_reactor_utilization, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "node", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "shard", - "multi": true, - "name": "shard", - "options": [], - "query": "label_values(scylla_reactor_utilization,shard)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "ks", - "multi": false, - "name": "ks", - "options": [], - "query": "label_values(scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"},ks)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "table", - "multi": true, - "name": "table", - "options": [], - "query": "label_values(scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\", ks=\"$ks\"},cf)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "aggregation_function", - "current": { - "tags": [], - "text": "sum", - "value": "sum" - }, - "hide": 0, - "includeAll": false, - "label": "Function", - "multi": false, - "name": "func", - "options": [ - { - "selected": true, - "text": "sum", - "value": "sum" - }, - { - "selected": false, - "text": "avg", - "value": "avg" - }, - { - "selected": false, - "text": "max", - "value": "max" - }, - { - "selected": false, - "text": "min", - "value": "min" - }, - { - "selected": false, - "text": "stddev", - "value": "stddev" - }, - { - "selected": false, - "text": "stdvar", - "value": "stdvar" - } - ], - "query": "sum,avg,max,min,stddev,stdvar", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5.0", - "value": "5.0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "scylla_version", - "options": [ - { - "selected": true, - "text": "5.0", - "value": "5.0" - } - ], - "query": "5.0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "4.2.1", - "value": "4.2.1" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "4.2.1", - "value": "4.2.1" - } - ], - "query": "4.2.1", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "datasource": "prometheus", - "definition": "scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"}", - "description": null, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "no_ks", - "options": [], - "query": { - "query": "scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"}", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/^(scylla_column_family_write_latency_count)/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Keyspace", - "uid": "ks-5-0", - "version": 5}`}} - os.json: |- - {{`{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_restart", - "datasource": "prometheus", - "enable": true, - "expr": "resets(scylla_gossip_heart_beat[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "node_restart", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "restart", - "type": "tags" - }, - { - "class": "annotation_stall", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_stall_detector_reported[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "stall detector", - "showIn": 0, - "tagKeys": "dc,instance,shard", - "tags": [], - "titleFormat": "Stall found", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "overwrite": true, - "panels": [ - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "repeat": "", - "title": "", - "type": "row" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
\n\n [[cluster]]
", - "mode": "html" - }, - "span": 12, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 3, - "panels": [], - "repeat": "", - "title": "", - "type": "row" - }, - { - "aliasColors": {}, - "class": "piechart_panel_percent", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "decimals": 0, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 5 - }, - "height": "250px", - "id": 4, - "isNew": true, - "links": [], - "options": { - "displayLabels": [ - "name", - "value", - "percent" - ], - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single" - } - }, - "repeat": "node", - "targets": [ - { - "expr": "sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Free", - "metric": "", - "refId": "A", - "step": 7200 - }, - { - "expr": "(sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}))", - "instant": true, - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "B", - "step": 7200 - } - ], - "title": "Total Storage $node", - "type": "piechart" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 5, - "panels": [], - "repeat": "mount_point", - "title": "Partition $mount_point", - "type": "row" - }, - { - "class": "percentunit_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 14 - }, - "id": 6, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "1-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])/sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Used disk by $by", - "type": "timeseries" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 14 - }, - "id": 7, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Used Bytes by $by", - "type": "timeseries" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 14 - }, - "id": 8, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(node_filesystem_free_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Free Bytes by $by", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 14 - }, - "id": 9, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(node_filesystem_files{mountpoint=\"$mount_point\", instance=~\"$node\"}- node_filesystem_files_free{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Number of files by $by", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 10, - "panels": [], - "repeat": "monitor_disk", - "title": "Disk $monitor_disk", - "type": "row" - }, - { - "class": "text_panel", - "content": "

Disk $monitor_disk

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 11, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "wps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:writes/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 23 - }, - "id": 12, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_disk_writes_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Writes per $by", - "type": "timeseries" - }, - { - "class": "rps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:reads/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 23 - }, - "id": 13, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_disk_reads_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Reads per $by", - "type": "timeseries" - }, - { - "class": "bps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 23 - }, - "id": 14, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_disk_bytes_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Writes Bps per $by", - "type": "timeseries" - }, - { - "class": "bps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 23 - }, - "id": 15, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_disk_bytes_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Read Bps per $by", - "type": "timeseries" - }, - { - "class": "seconds_panel", - "datasource": "prometheus", - "description": "The average read time from disk", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 29 - }, - "id": 16, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "rate(node_disk_read_time_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])/rate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Read AWait per instance", - "type": "timeseries" - }, - { - "class": "seconds_panel", - "datasource": "prometheus", - "description": "The average write time to disk", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 29 - }, - "id": 17, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "rate(node_disk_write_time_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])/rate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Write AWait per instance", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 18, - "panels": [], - "repeat": "monitor_network_interface", - "title": "Network Interface $monitor_network_interface", - "type": "row" - }, - { - "class": "text_panel", - "content": "

Network $monitor_network_interface

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 19, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "pps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "pps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 20, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 6, - "targets": [ - { - "expr": "sum(rate(node_network_receive_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_network_receive_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Rx Packets", - "type": "timeseries" - }, - { - "class": "pps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "pps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 21, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 6, - "targets": [ - { - "expr": "sum(rate(node_network_transmit_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_network_transmit_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Tx Packets", - "type": "timeseries" - }, - { - "class": "bps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 22, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 6, - "targets": [ - { - "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_network_receive_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Rx Bps", - "type": "timeseries" - }, - { - "class": "bps_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 23, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 6, - "targets": [ - { - "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(rate(node_network_transmit_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Tx Bps", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 24, - "panels": [], - "title": "CPU and Memory", - "type": "row" - }, - { - "class": "bps_panel", - "datasource": "prometheus", - "description": "The available memory, note that in a production environment we expect this to be low, Scylla would use most of the available memory when possible", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 51 - }, - "id": 25, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Available memory", - "type": "timeseries" - }, - { - "class": "percentunit_panel", - "datasource": "prometheus", - "description": "Percent of available memory, note that in a production environment we expect this to be low, Scylla would use most of the available memory when possible", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 51 - }, - "id": 26, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])/sum(node_memory_MemTotal_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Available memory", - "type": "timeseries" - }, - { - "class": "percentunit_panel", - "datasource": "prometheus", - "description": "Percent of CPU used, note that in production Scylla would try to use most of the CPU and this is not a problem", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 51 - }, - "id": 27, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 3, - "targets": [ - { - "expr": "1-sum(rate(node_cpu_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", mode=\"idle\"}[3m])) by ([[by]])/count(node_cpu_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", mode=\"idle\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CPU used", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "CPU frequency should be set for performance.\n\n The current frequency should match the max frequency. If that is not the case, check your host configuration.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "links": [], - "unit": "hertz" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 51 - }, - "id": 28, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - { - "$$hashKey": "object:211", - "alias": "Max", - "color": "#F2495C" - } - ], - "span": 3, - "targets": [ - { - "expr": "max(node_cpu_scaling_frequency_max_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) or on() max(node_cpu_frequency_max_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"})", - "intervalFactor": 1, - "legendFormat": "Max", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "min(node_cpu_scaling_frequency_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]]) or on() min(node_cpu_frequency_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "CPU Frequency", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 57 - }, - "id": 29, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - 4.2.1
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "5.0" - ], - "templating": { - "list": [ - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(node_filesystem_avail_bytes, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "node", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - } - ], - "query": "Cluster,DC,Instance", - "type": "custom" - }, - { - "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": null, - "multi": true, - "name": "monitor_disk", - "options": [], - "query": "node_disk_read_bytes_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": null, - "multi": true, - "name": "monitor_network_interface", - "options": [], - "query": "node_network_receive_packets_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": true, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5.0", - "value": "5.0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "scylla_version", - "options": [ - { - "selected": true, - "text": "5.0", - "value": "5.0" - } - ], - "query": "5.0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "4.2.1", - "value": "4.2.1" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "4.2.1", - "value": "4.2.1" - } - ], - "query": "4.2.1", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "OS Metrics", - "uid": "OS-5-0", - "version": 5}`}} - overview.json: |- - {{`{ - "annotations": { - "class": "default_annotations", - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_restart", - "datasource": "prometheus", - "enable": true, - "expr": "resets(scylla_gossip_heart_beat[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "node_restart", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "restart", - "type": "tags" - }, - { - "class": "annotation_stall", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_stall_detector_reported[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "stall detector", - "showIn": 0, - "tagKeys": "dc,instance,shard", - "tags": [], - "titleFormat": "Stall found", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - }, - { - "class": "annotation_manager_task", - "datasource": "prometheus", - "enable": true, - "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", - "hide": false, - "iconColor": "#73BF69", - "limit": 100, - "name": "Task", - "showIn": 0, - "tagKeys": "type", - "tags": [], - "titleFormat": "Running", - "type": "tags" - }, - { - "class": "annotation_hints_writes", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_hints_manager_written[$__rate_interval])>0", - "hide": false, - "iconColor": "rgb(255, 176, 0, 128)", - "limit": 100, - "name": "Hints Write", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Hints write", - "type": "tags" - }, - { - "class": "annotation_hints_sent", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_hints_manager_written[$__rate_interval])>0", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "Hints Sent", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Hints Sent", - "type": "tags" - }, - { - "class": "mv_building", - "datasource": "prometheus", - "enable": true, - "expr": "sum(scylla_view_builder_builds_in_progress)>0", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "MV", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Materialized View built", - "type": "tags" - }, - { - "class": "ops_annotation", - "datasource": "prometheus", - "enable": true, - "expr": "10*min(scylla_node_ops_finished_percentage) by (ops, dc,instance) < 10", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "ops", - "showIn": 0, - "tagKeys": "ops,dc,instance", - "tags": [], - "titleFormat": "Operation", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "originalTitle": "Scylla Cluster Metrics", - "overwrite": true, - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Cluster overview $cluster", - "type": "row" - }, - { - "class": "text_panel", - "dashproductreject": "no-version-check", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 9, - "x": 0, - "y": 1 - }, - "id": 2, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "", - "mode": "html" - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 4, - "x": 9, - "y": 1 - }, - "id": 3, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "# " - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 4, - "x": 13, - "y": 1 - }, - "id": 4, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "# " - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "text_panel", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 7, - "x": 17, - "y": 1 - }, - "id": 5, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "# " - }, - "span": 12, - "title": "", - "transparent": false, - "type": "text" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The number of nodes configured in the cluster.", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 0, - "y": 2 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "count(scylla_scylladb_current_version{job=~\"$cluster|$^\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "# Nodes", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The number of unreachable nodes.\nUsually because a machine is down or unreachable.", - "fieldConfig": { - "defaults": { - "custom": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - } - } - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 2, - "y": 2 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "(count(scrape_samples_scraped{job=~\"$cluster|$^\", cluster=~\"$cluster|$^\"}==0) OR vector(0))", - "intervalFactor": 1, - "legendFormat": "Offline ", - "refId": "A", - "step": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Unreachable", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The number of joining and leaving nodes.\nThe number of nodes that are up but not actively part of the cluster, either because they are still joining or because they are leaving.", - "fieldConfig": { - "defaults": { - "custom": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - } - } - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 4, - "y": 2 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "count(scylla_node_operation_mode!=3)OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Offline ", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "Inactive", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "from": "", - "id": 0, - "text": "Backup", - "to": "", - "type": 1, - "value": "2" - }, - { - "from": "", - "id": 1, - "text": "Repair", - "to": "", - "type": 1, - "value": "1" - }, - { - "from": "", - "id": 2, - "text": "Online", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "", - "id": 2, - "text": "Offline", - "to": "", - "type": 1, - "value": "-1" - } - ], - "noValue": " Offline", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - } - } - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 6, - "y": 2 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "(sum(scylla_manager_task_active_count{type=~\"repair\",cluster=~\"$cluster|$^\"}) or on() vector(0)) + (sum(scylla_manager_task_active_count{type=~\"backup\",cluster=~\"$cluster|$^\"})*2 or on() vector(0)) + (sum(scylla_manager_server_current_version{}) or on() vector(-1))", - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Manager", - "type": "stat" - }, - { - "class": "vertical_lcd", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 1, - "x": 8, - "y": 2 - }, - "id": 10, - "options": { - "displayMode": "lcd", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "(sum(avg(scylla_manager_repair_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"repair\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0)) + (sum(avg(scylla_manager_backup_percent_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"backup\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0))", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "", - "type": "bargauge" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "Average Write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 9, - "y": 2 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0) or on() (sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 11, - "y": 2 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "99% Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "Average Read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 13, - "y": 2 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0) or on() (sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 15, - "y": 2 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "99% Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "si:" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 17, - "y": 2 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests/s", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 20, - "y": 2 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Load", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 2 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Timeouts", - "type": "stat" - }, - { - "class": "alert_table", - "columns": [], - "datasource": "alertmanager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 18, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 4, - "targets": [ - { - "active": true, - "annotations": true, - "filters": "job!=\"scylla_manager\",advisor=\"\"", - "legendFormat": "{{description}}", - "refId": "A", - "target": "Query" - } - ], - "title": "Active Alerts", - "transform": "table", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Time", - "summary", - "instance" - ] - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Time": 0, - "instance": 1, - "summary": 2 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 6 - }, - "id": 19, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 6 - }, - "id": 20, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(wlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B", - "step": 1 - } - ], - "title": "Write Latencies", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 6 - }, - "id": 21, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", - "intervalFactor": 1, - "legendFormat": "Reads", - "refId": "A", - "step": 1 - } - ], - "title": "Reads", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 6 - }, - "id": 22, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(rlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B", - "step": 1 - } - ], - "title": "Read Latencies", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 23, - "panels": [], - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Advisor

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 24, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 25, - "options": { - "content": "

Balance

\nAn Imbalance between shards or nodes may indicates a potential problem", - "mode": "html" - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "", - "type": "text" - }, - { - "class": "advisor_table", - "dashversion": ">4.1", - "datasource": "alertmanager", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "dashboard" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/${__data.fields.dashboard}-[[dash_version]]?refresh=30s&orgId=1&var-by=instance&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "advisor" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - }, - { - "id": "displayName", - "value": "Category" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "severity" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "Open an issue", - "url": "https://github.com/scylladb/scylla/issues/new?body=description%3D${__data.fields[4]}%0ASource%3DAdvisor%0AScylla-versions%3D${all_scyllas_versions}%0Ascylla-monitoring%3D${monitoring_version}%0Acluster%3D${count_dc}%0Aname%3D${cluster}%0A%0A" - } - ] - }, - { - "id": "mappings", - "value": [ - { - "from": "0", - "id": 1, - "text": "\ud83d\udd14", - "to": "10", - "type": 2, - "value": "" - } - ] - }, - { - "id": "displayName", - "value": "Report" - }, - { - "id": "custom.width", - "value": 65 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "summary" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "${__data.fields.description}\n\n click for more information", - "url": "https://monitoring.docs.scylladb.com/branch-4.2/use-monitoring/advisor/${__data.fields.alertname}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alertname" - }, - "properties": [ - { - "id": "displayName", - "value": "." - }, - { - "id": "custom.width", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "description" - }, - "properties": [ - { - "id": "displayName", - "value": "." - }, - { - "id": "custom.width", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 26, - "links": [], - "options": { - "showHeader": true - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "active": true, - "annotations": true, - "filters": "advisor!=\"\"", - "legendFormat": "{{description}}", - "refId": "A", - "target": "Query" - } - ], - "title": "", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "advisor", - "dashboard", - "description", - "severity", - "alertname", - "summary", - "Time" - ] - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Time": 1, - "advisor": 2, - "dashboard": 3, - "severity": 0, - "summary": 4 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 12, - "y": 17 - }, - "id": 27, - "links": [ - { - "title": "The number of connections per shard should be balanced", - "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "max(abs(sum(scylla_transport_current_connections{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by (instance,shard)-scalar(avg(scylla_transport_current_connections{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"})))) - 8", - "hide": false, - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Connections", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 14, - "y": 17 - }, - "id": 28, - "links": [ - { - "title": "Indicates that the number of CQL operations (inserts, updates, deletes, reads) is not balance between shards in one of the nodes", - "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "max(abs(rate(scylla_cql_updates{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_updates{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_updates{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", - "format": "time_series", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "max(abs(rate(scylla_cql_inserts{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_inserts{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_inserts{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "expr": "max(abs(rate(scylla_cql_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "expr": "max(abs(rate(scylla_cql_deletes{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_deletes{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_deletes{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "D" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CQL OPs", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 16, - "y": 17 - }, - "id": 29, - "links": [ - { - "title": "A single node with higher latency is an indication for a node related issue", - "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "((max(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "((max(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Node Latency", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 18, - "y": 17 - }, - "id": 30, - "links": [ - { - "title": "A single shard with high latency is an indication of a hot-partition, or a large row/cell/partition", - "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "((max(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "((max(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Shard Latency", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "A shard that reads more from the cache could be an indication for hot partition", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 20, - "y": 17 - }, - "id": 31, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))- scalar(avg(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))))/scalar(stddev(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))+100)-3", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Cache", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "A single shard that reads more from sstables, could be that a node is slow", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 17 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "max(abs(scylla_database_active_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} - scalar(avg(scylla_database_active_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"})))/scalar(stddev(scylla_database_active_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"})+0.001))-3", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "SSTable", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 33, - "panels": [], - "repeat": "dc", - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Information for $dc

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 34, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "vertical_lcd", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 85 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 1, - "x": 0, - "y": 26 - }, - "id": 35, - "options": { - "displayMode": "lcd", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Load", - "type": "bargauge" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 1, - "y": 26 - }, - "id": 36, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 5, - "targets": [ - { - "expr": "Avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])-avg(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Avg Usage {{[[by]]}}", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])", - "interval": "", - "legendFormat": "Size {{[[by]]}}", - "refId": "B" - } - ], - "title": "Disk Size by $by", - "type": "timeseries" - }, - { - "class": "graph_panel_int", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 26 - }, - "id": 37, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 2, - "targets": [ - { - "expr": "$func(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Running Compactions", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "The Hits and Misses", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 8, - "y": 26 - }, - "id": 38, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Hit {{[[by]]}}", - "refId": "A", - "step": 10 - }, - { - "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Misses {{[[by]]}}", - "refId": "B", - "step": 10 - } - ], - "title": "Cache Hits/Misses", - "type": "timeseries" - }, - { - "class": "small_nodes_table", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 85 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "lcd-gauge" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 101 - }, - { - "id": "displayName", - "value": "Load" - }, - { - "id": "custom.width", - "value": 120 - }, - { - "id": "custom.filterable", - "value": false - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "svr" - }, - "properties": [ - { - "id": "custom.width", - "value": 85 - }, - { - "id": "displayName", - "value": "Version" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 105 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "custom.width", - "value": 105 - }, - { - "id": "displayName", - "value": "Status" - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - }, - { - "color": "rgb(87, 128, 193)", - "value": 3 - }, - { - "color": "red", - "value": 4 - } - ] - } - }, - { - "id": "custom.align", - "value": "center" - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "Starting", - "to": "", - "type": 1, - "value": "1" - }, - { - "from": "", - "id": 2, - "text": "Joining", - "to": "", - "type": 1, - "value": "2" - }, - { - "from": "", - "id": 3, - "text": "Normal", - "to": "", - "type": 1, - "value": "3" - }, - { - "from": "", - "id": 4, - "text": "Leaving", - "to": "", - "type": 1, - "value": "4" - }, - { - "from": "", - "id": 5, - "text": "Decommissioned", - "to": "", - "type": 1, - "value": "5" - }, - { - "from": "", - "id": 6, - "text": "Draining", - "to": "", - "type": 1, - "value": "6" - }, - { - "from": "", - "id": 7, - "text": "Drained", - "to": "", - "type": 1, - "value": "7" - }, - { - "from": "", - "id": 8, - "text": "Moving", - "to": "", - "type": 1, - "value": "8" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "CQL Information Dashboard. Warning indicates that there are potential issues in the CQL Optimization", - "url": "/d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 90 - }, - { - "id": "custom.filterable", - "value": false - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "CQL Info", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "1", - "id": 1, - "text": "CQL Warnings", - "to": "20", - "type": 2, - "value": "" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(87, 128, 193)", - "value": null - }, - { - "color": "dark-orange", - "value": 1 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "displayName", - "value": "CQL" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "OS Information Dashboard, an Error indicates there are OS related errors", - "url": "/d/OS-[[dash_version]]/os-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 90 - }, - { - "id": "custom.filterable", - "value": true - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "OS Info", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "0", - "id": 2, - "text": "OS Errors", - "to": "100000", - "type": 2, - "value": "0" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(87, 128, 193)", - "value": null - }, - { - "color": "dark-orange", - "value": 0.001 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "displayName", - "value": "OS" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "Cordinator and Replica node errors", - "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 80 - }, - { - "id": "custom.filterable", - "value": true - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 1, - "text": "", - "to": "", - "type": 1, - "value": "0" - }, - { - "from": "", - "id": 2, - "text": "Errors", - "to": "", - "type": 1, - "value": "1" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(87, 128, 193)", - "value": null - }, - { - "color": "dark-orange", - "value": 0.001 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "displayName", - "value": " " - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "Detailed view", - "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 17, - "w": 10, - "x": 14, - "y": 26 - }, - "id": 39, - "scopedVars": { - "dc": { - "selected": false, - "text": "datacenter1", - "value": "datacenter1" - } - }, - "span": 8, - "targets": [ - { - "expr": "0*scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"} + on (instance) group_left() scylla_node_operation_mode{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - }, - { - "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} ) by (instance)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "expr": "sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "expr": "sum(scylla_errors:nodes_total{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 0", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "expr": "(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - } - ], - "title": "Nodes", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "instance", - "svr", - "Value #A", - "Value #B", - "Value #C", - "Value #D", - "Value #E", - "Value #F" - ] - } - } - }, - { - "id": "seriesToColumns", - "options": { - "byField": "instance" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Value #A": 5, - "Value #B": 7, - "Value #C": 3, - "Value #D": 1, - "Value #E": 2, - "Value #F": 6, - "instance": 0, - "svr": 4 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 43 - }, - "id": 40, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - { - "alias": "1 Day Ago", - "dashLength": 4, - "dashes": true - }, - { - "alias": "1 Week Ago", - "dashLength": 2, - "dashes": true - } - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Writes", - "refId": "A", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m] offset 1d))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "1 Day Ago", - "refId": "B", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m] offset 1w))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "1 Week Ago", - "refId": "C", - "step": 1 - } - ], - "title": "Writes", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 6, - "y": 43 - }, - "id": 41, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(wlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}>0) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "95% {{[[by]]}}", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(wlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}>0) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "99% {{[[by]]}}", - "refId": "B", - "step": 1 - } - ], - "title": "Write Latencies", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 10, - "y": 43 - }, - "id": 42, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Writes {{[[by]]}}", - "refId": "A", - "step": 10 - } - ], - "title": "Write Timeouts by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 43 - }, - "id": 43, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - { - "alias": "1 Day Ago", - "dashLength": 4, - "dashes": true - }, - { - "alias": "1 Week Ago", - "dashLength": 2, - "dashes": true - } - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Reads", - "refId": "A", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}[1m] offset 1d))", - "intervalFactor": 1, - "legendFormat": "1 Day Ago", - "refId": "B", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}[1m] offset 1w))", - "intervalFactor": 1, - "legendFormat": "1 Week Ago", - "refId": "C", - "step": 1 - } - ], - "title": "Reads", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 6, - "y": 43 - }, - "id": 44, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(rlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}>0) by([[by]])", - "intervalFactor": 1, - "legendFormat": "95% {{[[by]]}}", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(rlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name!=\"streaming\"}>0) by([[by]])", - "intervalFactor": 1, - "legendFormat": "99% {{[[by]]}}", - "refId": "B", - "step": 1 - } - ], - "title": "Read Latencies", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 10, - "y": 49 - }, - "id": 45, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])+rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])+rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Read {{[[by]]}}", - "refId": "A", - "step": 10 - } - ], - "title": "Read Timeouts by [[by]]", - "type": "timeseries" - }, - { - "class": "plain_text", - "dashproductreject": "no-version-check", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 10, - "x": 14, - "y": 49 - }, - "id": 46, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "plain_text", - "dashproduc": "no-version-check", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 10, - "x": 14, - "y": 49 - }, - "id": 47, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 55 - }, - "id": 48, - "panels": [], - "repeat": "", - "title": "Your panels", - "type": "row" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 49, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Your Panels

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 50, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 51, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 52, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - 4.2.1
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "5.0" - ], - "templating": { - "list": [ - { - "allValue": null, - "class": "by_template_var", - "current": { - "tags": [], - "text": "DC", - "value": "dc" - }, - "error": null, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": true, - "text": "DC", - "value": "dc" - }, - { - "selected": false, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "instance,shard", - "value": "instance,shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(scylla_reactor_utilization, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "node", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "shard", - "multi": true, - "name": "shard", - "options": [], - "query": "label_values(scylla_reactor_utilization,shard)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "aggregation_function", - "current": { - "tags": [], - "text": "sum", - "value": "sum" - }, - "hide": 0, - "includeAll": false, - "label": "Function", - "multi": false, - "name": "func", - "options": [ - { - "selected": true, - "text": "sum", - "value": "sum" - }, - { - "selected": false, - "text": "avg", - "value": "avg" - }, - { - "selected": false, - "text": "max", - "value": "max" - }, - { - "selected": false, - "text": "min", - "value": "min" - }, - { - "selected": false, - "text": "stddev", - "value": "stddev" - }, - { - "selected": false, - "text": "stdvar", - "value": "stdvar" - } - ], - "query": "sum,avg,max,min,stddev,stdvar", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5-0", - "value": "5-0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "dash_version", - "options": [ - { - "selected": true, - "text": "5-0", - "value": "5-0" - } - ], - "query": "5-0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 2, - "includeAll": true, - "multi": true, - "name": "all_scyllas_versions", - "options": [], - "query": "label_values(scylla_scylladb_current_version{cluster=~\"$cluster|$^\"}, version)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", - "error": null, - "hide": 2, - "includeAll": true, - "multi": true, - "name": "count_dc", - "options": [], - "query": { - "query": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/(?\\{dc=\"[^\"]+\".* \\d+) .*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_custom", - "current": { - "text": "5.0", - "value": "5.0" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "scylla_version", - "options": [ - { - "selected": true, - "text": "5.0", - "value": "5.0" - } - ], - "query": "5.0", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "4.2.1", - "value": "4.2.1" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "4.2.1", - "value": "4.2.1" - } - ], - "query": "4.2.1", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Overview", - "uid": "overview-5-0", - "version": 1}` }} - diff --git a/assets/monitoring/grafana/v1alpha1/dashboards-saas.cm.yaml b/assets/monitoring/grafana/v1alpha1/dashboards-saas.cm.yaml deleted file mode 100644 index ac7949e6bf7..00000000000 --- a/assets/monitoring/grafana/v1alpha1/dashboards-saas.cm.yaml +++ /dev/null @@ -1,4686 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: '{{ .scyllaDBMonitoringName }}-grafana-scylladb-dashboards' -data: - overview.json: |- - {{`{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }, - { - "class": "annotation_manager_task", - "datasource": "prometheus", - "enable": true, - "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", - "hide": false, - "iconColor": "#73BF69", - "limit": 100, - "name": "Task", - "showIn": 0, - "tagKeys": "type", - "tags": [], - "titleFormat": "Running", - "type": "tags" - }, - { - "class": "mv_building", - "datasource": "prometheus", - "enable": true, - "expr": "sum(scylla_view_builder_builds_in_progress)>0", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "MV", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "Materialized View built", - "type": "tags" - }, - { - "class": "ops_annotation", - "datasource": "prometheus", - "enable": true, - "expr": "10*min(scylla_node_ops_finished_percentage) by (ops, dc,instance) < 10", - "hide": false, - "iconColor": "rgb(50, 176, 0, 128)", - "limit": 100, - "name": "ops", - "showIn": 0, - "tagKeys": "ops,dc,instance", - "tags": [], - "titleFormat": "Operation", - "type": "tags" - }, - { - "class": "annotation_schema_changed", - "datasource": "prometheus", - "enable": false, - "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", - "hide": false, - "iconColor": "rgba(255, 96, 96, 1)", - "limit": 100, - "name": "Schema Changed", - "showIn": 0, - "tagKeys": "instance,dc,cluster", - "tags": [], - "titleFormat": "schema changed", - "type": "tags" - } - ] - }, - "class": "dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "hideControls": true, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "type": "dashboards" - } - ], - "originalTitle": "Scylla Cluster Metrics", - "overwrite": true, - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Cluster overview $cluster", - "type": "row" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "si:" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "sum(rate(scylla_transport_requests_served{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])) + (sum(rate(scylla_thrift_served{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])) or on() vector(0))", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests/s", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "Average Write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 5, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencyp95{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "95% Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% write Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 7, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "99% Write", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "Average Read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 9, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 11, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencyp95{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "95% Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "99% read Latency", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 13, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "99% Read", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 15, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"} )", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Load", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 17, - "y": 1 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "$func(rate(scylla_database_total_reads_failed{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "R Failed", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 19, - "y": 1 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "$func(rate(scylla_database_total_writes_failed{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "W Failed", - "type": "stat" - }, - { - "class": "small_stat", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 21, - "y": 1 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", - "instant": true, - "intervalFactor": 1, - "refId": "A", - "step": 40 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Timeouts", - "type": "stat" - }, - { - "class": "alert_table", - "columns": [], - "datasource": "alertmanager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 5 - }, - "id": 13, - "links": [], - "options": { - "showHeader": true - }, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "span": 4, - "targets": [ - { - "active": true, - "annotations": true, - "filters": "job!=\"scylla_manager\",advisor=\"\"", - "legendFormat": "{{description}}", - "refId": "A", - "target": "Query" - } - ], - "title": "Active Alerts", - "transform": "table", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Time", - "summary", - "instance" - ] - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Time": 0, - "instance": 1, - "summary": 2 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 14, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 5 - }, - "id": 15, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(wlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B", - "step": 1 - } - ], - "title": "Write Latencies", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 5 - }, - "id": 16, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))", - "intervalFactor": 1, - "legendFormat": "Reads", - "refId": "A", - "step": 1 - } - ], - "title": "Reads", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 5 - }, - "id": 17, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(rlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B", - "step": 1 - } - ], - "title": "Read Latencies", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 18, - "panels": [], - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Advisor

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 19, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 20, - "options": { - "content": "

Balance

\nAn Imbalance between shards or nodes may indicates a potential problem", - "mode": "html" - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "", - "type": "text" - }, - { - "class": "advisor_table", - "dashversion": ">4.1", - "datasource": "alertmanager", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "dashboard" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/${__data.fields.dashboard}-[[dash_version]]?refresh=30s&orgId=1&var-by=instance&from=${__from}&to=${__to}" - } - ] - }, - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "advisor" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - }, - { - "id": "displayName", - "value": "Category" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "severity" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "Open an issue", - "url": "https://github.com/scylladb/scylla/issues/new?body=description%3D${__data.fields[4]}%0ASource%3DAdvisor%0AScylla-versions%3D${all_scyllas_versions}%0Ascylla-monitoring%3D${monitoring_version}%0Acluster%3D${count_dc}%0Aname%3D${cluster}%0A%0A" - } - ] - }, - { - "id": "mappings", - "value": [ - { - "from": "0", - "id": 1, - "text": "\ud83d\udd14", - "to": "10", - "type": 2, - "value": "" - } - ] - }, - { - "id": "displayName", - "value": "Report" - }, - { - "id": "custom.width", - "value": 65 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "summary" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "${__data.fields.description}\n\n click for more information", - "url": "https://monitoring.docs.scylladb.com/branch-master/use-monitoring/advisor/${__data.fields.alertname}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "alertname" - }, - "properties": [ - { - "id": "displayName", - "value": "." - }, - { - "id": "custom.width", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "description" - }, - "properties": [ - { - "id": "displayName", - "value": "." - }, - { - "id": "custom.width", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 21, - "links": [], - "options": { - "showHeader": true - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "active": true, - "annotations": true, - "filters": "advisor!=\"\"", - "legendFormat": "{{description}}", - "refId": "A", - "target": "Query" - } - ], - "title": "", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "advisor", - "dashboard", - "description", - "severity", - "alertname", - "summary", - "Time" - ] - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Time": 1, - "advisor": 2, - "dashboard": 3, - "severity": 0, - "summary": 4 - }, - "renameByName": {} - } - } - ], - "type": "table" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 12, - "y": 16 - }, - "id": 22, - "links": [ - { - "title": "The number of connections per shard should be balanced" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "max(abs(sum(scylla_transport_current_connections{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by (instance,shard)-scalar(avg(scylla_transport_current_connections{cluster=~\"$cluster|$^\", dc=~\"$dc\"})))) - 8", - "hide": false, - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Connections", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 14, - "y": 16 - }, - "id": 23, - "links": [ - { - "title": "Indicates that the number of CQL operations (inserts, updates, deletes, reads) is not balance between shards in one of the nodes" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "max(abs(rate(scylla_cql_updates{conditional=\"no\", dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_updates{conditional=\"no\", dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_updates{conditional=\"no\", dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", - "format": "time_series", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "max(abs(rate(scylla_cql_inserts{conditional=\"no\", dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_inserts{conditional=\"no\", dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_inserts{conditional=\"no\", dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "expr": "max(abs(rate(scylla_cql_reads{ dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_reads{ dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_reads{ dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "expr": "max(abs(rate(scylla_cql_deletes{conditional=\"no\", dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_deletes{conditional=\"no\", dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_deletes{conditional=\"no\", dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "D" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CQL OPs", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 16, - "y": 16 - }, - "id": 24, - "links": [ - { - "title": "A single node with higher latency is an indication for a node related issue" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "((max(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "((max(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Node Latency", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 18, - "y": 16 - }, - "id": 25, - "links": [ - { - "title": "A single shard with high latency is an indication of a hot-partition, or a large row/cell/partition" - } - ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "((max(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "((max(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Shard Latency", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "A shard that reads more from the cache could be an indication for hot partition", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 20, - "y": 16 - }, - "id": 26, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "((rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]) - rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))- scalar(avg(rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]) - rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))))/scalar(stddev(rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]) - rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))+100)-3", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Cache", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "class": "small_stat_error", - "datasource": "prometheus", - "description": "A single shard that reads more from sstables, could be that a node is slow", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "-1000", - "id": 1, - "text": "\u2713", - "to": "0.001", - "type": 2, - "value": "" - }, - { - "from": "0.001", - "id": 2, - "text": "\u26a0", - "to": "10000", - "type": 2, - "value": "0.001" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.001 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 16 - }, - "id": 27, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "max(abs(scylla_database_active_reads{ dc=~\"$dc\"} - scalar(avg(scylla_database_active_reads{ dc=~\"$dc\"})))/scalar(stddev(scylla_database_active_reads{ dc=~\"$dc\"})+0.001))-3", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "SSTable", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "reducer": "max" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 28, - "panels": [], - "repeat": "dc", - "title": "", - "type": "row" - }, - { - "class": "plain_text", - "content": "

Information for $dc

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 29, - "isNew": true, - "links": [], - "mode": "html", - "options": {}, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "vertical_lcd", - "datasource": "prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 85 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 1, - "x": 0, - "y": 25 - }, - "id": 30, - "options": { - "displayMode": "lcd", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.3", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} )", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Load", - "type": "bargauge" - }, - { - "class": "bytes_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 1, - "y": 25 - }, - "id": 31, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 5, - "targets": [ - { - "expr": "Avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\"}) by ([[by]])-avg(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Avg Usage {{[[by]]}}", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\"}) by ([[by]])", - "interval": "", - "legendFormat": "Size {{[[by]]}}", - "refId": "B" - } - ], - "title": "Disk Size by $by", - "type": "timeseries" - }, - { - "class": "graph_panel_int", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 25 - }, - "id": 32, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "span": 2, - "targets": [ - { - "expr": "$func(scylla_compaction_manager_compactions{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Running Compactions", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "The Hits and Misses", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 8, - "y": 25 - }, - "id": 33, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_cache_row_hits{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Hit {{[[by]]}}", - "refId": "A", - "step": 10 - }, - { - "expr": "$func(rate(scylla_cache_row_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Misses {{[[by]]}}", - "refId": "B", - "step": 10 - } - ], - "title": "Cache Rows Hits/Misses", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 14, - "y": 25 - }, - "id": 34, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - { - "alias": "1 Day Ago", - "dashLength": 4, - "dashes": true - }, - { - "alias": "1 Week Ago", - "dashLength": 2, - "dashes": true - } - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Writes", - "refId": "A", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m] offset 1d))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "1 Day Ago", - "refId": "B", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m] offset 1w))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "1 Week Ago", - "refId": "C", - "step": 1 - } - ], - "title": "Writes", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 25 - }, - "id": 35, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(wlatencyp95{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "95% {{[[by]]}}", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(wlatencyp99{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "99% {{[[by]]}}", - "refId": "B", - "step": 1 - } - ], - "title": "Write Latencies", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 31 - }, - "id": 36, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Writes {{[[by]]}}", - "refId": "A", - "step": 10 - } - ], - "title": "Write Timeouts by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 31 - }, - "id": 37, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])+rate(scylla_storage_proxy_coordinator_cas_read_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])+rate(scylla_storage_proxy_coordinator_range_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Read {{[[by]]}}", - "refId": "A", - "step": 10 - } - ], - "title": "Read Timeouts by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "The Hits and Misses", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 8, - "y": 31 - }, - "id": 38, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])-$func(rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Hit {{[[by]]}}", - "refId": "A", - "step": 10 - }, - { - "expr": "$func(rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Misses {{[[by]]}}", - "refId": "B", - "step": 10 - } - ], - "title": "Cache Reads Hits/Misses", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 14, - "y": 31 - }, - "id": 39, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - { - "alias": "1 Day Ago", - "dashLength": 4, - "dashes": true - }, - { - "alias": "1 Week Ago", - "dashLength": 2, - "dashes": true - } - ], - "span": 3, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Reads", - "refId": "A", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}[1m] offset 1d))", - "intervalFactor": 1, - "legendFormat": "1 Day Ago", - "refId": "B", - "step": 1 - }, - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}[1m] offset 1w))", - "intervalFactor": 1, - "legendFormat": "1 Week Ago", - "refId": "C", - "step": 1 - } - ], - "title": "Reads", - "type": "timeseries" - }, - { - "class": "us_panel", - "datasource": "prometheus", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 31 - }, - "id": 40, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "class": "show_legend", - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 2, - "targets": [ - { - "expr": "avg(rlatencyp95{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by([[by]])", - "intervalFactor": 1, - "legendFormat": "95% {{[[by]]}}", - "refId": "A", - "step": 1 - }, - { - "expr": "avg(rlatencyp99{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by([[by]])", - "intervalFactor": 1, - "legendFormat": "99% {{[[by]]}}", - "refId": "B", - "step": 1 - } - ], - "title": "Read Latencies", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL INSERT commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 37 - }, - "id": 41, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_inserts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Insert", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL SELECT commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 37 - }, - "id": 42, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Reads", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL DELETE commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 37 - }, - "id": 43, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_deletes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Deletes", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Number of CQL UPDATE commands generated by the user", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 37 - }, - "id": 44, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_updates{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Updates", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "amount of CQL connections currently established", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 43 - }, - "id": 45, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(scylla_transport_current_connections{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Client CQL connections by [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL batches command, each batched command is counted once", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 43 - }, - "id": 46, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_batches{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Batches by [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL command batched. Each batch would add the number of commands inside the batch", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 43 - }, - "id": 47, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_statements_in_batches{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Command In Batches by [[by]]", - "type": "timeseries" - }, - { - "class": "ops_panel", - "datasource": "prometheus", - "description": "Counts the number of SELECT statements with BYPASS CACHE option", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 43 - }, - "id": 48, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_select_bypass_caches{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "BYPASS CACHE", - "type": "timeseries" - }, - { - "class": "graph_panel", - "dashversion": [ - ">4.4", - ">2021.1" - ], - "datasource": "prometheus", - "description": "CQL errors by type, only active errors are shown", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 49 - }, - "id": 49, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_transport_cql_errors_total{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]],type) >0", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Errors [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of CQL row reads", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 49 - }, - "id": 50, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_rows_read{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Row Reads [[by]]", - "type": "timeseries" - }, - { - "class": "graph_panel", - "datasource": "prometheus", - "description": "Number of reads using secondary indexes", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 49 - }, - "id": 51, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pointradius": 1, - "span": 3, - "targets": [ - { - "expr": "sum(rate(scylla_cql_secondary_index_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Secondary indexes Reads [[by]]", - "type": "timeseries" - }, - { - "class": "collapsible_row_panel", - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 55 - }, - "id": 52, - "panels": [], - "repeat": "", - "title": "Your panels", - "type": "row" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 53, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "

Your Panels

", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 54, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "user_panel", - "datasource": "prometheus", - "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "class": "fieldConfig_defaults", - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "class": "fieldConfig_defaults_custom", - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "si:ops/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 55, - "isNew": true, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "seriesOverrides": [ - {} - ], - "span": 6, - "title": "Your Graph here", - "type": "timeseries" - }, - { - "class": "plain_text", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 56, - "isNew": true, - "links": [], - "mode": "html", - "options": { - "content": "
Scylla Monitoring version - master
\n    
", - "mode": "html" - }, - "span": 12, - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "class": "by_template_var", - "current": { - "tags": [], - "text": "DC", - "value": "dc" - }, - "error": null, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": true, - "text": "DC", - "value": "dc" - } - ], - "query": "Cluster,DC,Instance,Shard", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(scylla_reactor_utilization, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": true, - "label": "dc", - "multi": true, - "name": "dc", - "options": [], - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_single", - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [ - { - "selected": true, - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - } - ], - "query": "/var/lib/scylla", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "custom", - "useTags": false - }, - { - "allValue": null, - "class": "aggregation_function", - "current": { - "tags": [], - "text": "sum", - "value": "sum" - }, - "hide": 0, - "includeAll": false, - "label": "Function", - "multi": false, - "name": "func", - "options": [ - { - "selected": true, - "text": "sum", - "value": "sum" - }, - { - "selected": false, - "text": "avg", - "value": "avg" - }, - { - "selected": false, - "text": "max", - "value": "max" - }, - { - "selected": false, - "text": "min", - "value": "min" - }, - { - "selected": false, - "text": "stddev", - "value": "stddev" - }, - { - "selected": false, - "text": "stdvar", - "value": "stdvar" - } - ], - "query": "sum,avg,max,min,stddev,stdvar", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "", - "error": null, - "hide": 2, - "includeAll": true, - "multi": true, - "name": "all_scyllas_versions", - "options": [], - "query": "label_values(scylla_scylladb_current_version{cluster=~\"$cluster|$^\"}, version)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "template_variable_all", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "prometheus", - "definition": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", - "error": null, - "hide": 2, - "includeAll": true, - "multi": true, - "name": "count_dc", - "options": [], - "query": { - "query": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/(?\\{dc=\"[^\"]+\".* \\d+) .*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "class": "monitor_version_var", - "current": { - "text": "master", - "value": "master" - }, - "error": null, - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitoring_version", - "options": [ - { - "selected": true, - "text": "master", - "value": "master" - } - ], - "query": "master", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "CQL Overview", - "uid": "cql-overview", - "version": 1 - }`}} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards.cm.yaml b/assets/monitoring/grafana/v1alpha1/dashboards.cm.yaml new file mode 100644 index 00000000000..bce7181cde7 --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards.cm.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ .scyllaDBMonitoringName }}-grafana-scylladb-dashboards' +data: +{{- range $key, $value := .dashboards }} + "{{ $key }}": "{{ $value }}" +{{ else }} +--- Can't read dashboards --- +{{ end }} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/advanced.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/advanced.json new file mode 100644 index 00000000000..9635d6c566b --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/advanced.json @@ -0,0 +1,4148 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "dashproductreject": "no-version-check", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 9, + "y": 0 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 13, + "y": 0 + }, + "id": 3, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 7, + "x": 17, + "y": 0 + }, + "id": 4, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of nodes configured in the cluster.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_scylladb_current_version{job=\"scylla\", cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "# Nodes", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of unreachable nodes.\nUsually because a machine is down or unreachable.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0))", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unreachable", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of joining and leaving nodes.\nThe number of nodes that are up but not actively part of the cluster, either because they are still joining or because they are leaving.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 4, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_node_operation_mode{cluster=~\"$cluster|$^\"}!=3)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Inactive", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 0, + "text": "Backup", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 1, + "text": "Repair", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Online", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Offline", + "to": "", + "type": 1, + "value": "-1" + } + ], + "noValue": " Offline", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 6, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(scylla_manager_task_active_count{type=~\"repair\",cluster=~\"$cluster|$^\"}) or on() vector(0)) + (sum(scylla_manager_task_active_count{type=~\"backup\",cluster=~\"$cluster|$^\"})*2 or on() vector(0)) + (sum(scylla_manager_server_current_version{}) or on() vector(-1))", + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Manager", + "type": "stat" + }, + { + "class": "vertical_lcd", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 1, + "x": 8, + "y": 1 + }, + "id": 9, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(avg(scylla_manager_repair_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"repair\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0)) + (sum(avg(scylla_manager_backup_percent_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"backup\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "bargauge" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "Average Write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 9, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 11, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "Average Read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 13, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 15, + "y": 1 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 17, + "y": 1 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests/s", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Timeouts", + "type": "stat" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 17, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

IO Queue Information

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 18, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [], + "repeat": "classes", + "title": "$classes", + "type": "row" + }, + { + "class": "seconds_panel", + "datasource": "prometheus", + "description": "seastar_io_queue_delay", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes I/O Queue delay by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of requests in the queue", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "max(scylla_io_queue_queue_length{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes Queue length by [[by]]", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "prometheus", + "description": "seastar_io_queue_delay", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes I/O Queue bandwidth by [[by]]", + "type": "timeseries" + }, + { + "class": "iops_panel", + "datasource": "prometheus", + "description": "Total operations passed in the queue", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 23, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes I/O Queue IOPS by [[by]]", + "type": "timeseries" + }, + { + "class": "seconds_panel", + "datasource": "prometheus", + "description": "Total time spent in disk", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 15 + }, + "id": 24, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "metric": "Total time spent in disk", + "refId": "A", + "step": 30 + } + ], + "title": "Disk $classes I/O Queue delay by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of requests in the disk", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 15 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "max(scylla_io_queue_disk_queue_length{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "metric": "Number of requests in the disk", + "refId": "A", + "step": 30 + } + ], + "title": "DISK $classes Queue length by [[by]]", + "type": "timeseries" + }, + { + "class": "seconds_panel", + "datasource": "prometheus", + "description": "The time the class waited for being dispatched with non-empty software queue.\n\nLarge IO delays coupled with small starvation time denotes that scheduler is doing its job properly, and it's upper layer that overflows disk capacity.\n\nLarge IO delays coupled with large starvation time denotes that there might be some problem on the scheduler level that it cannot deliver IO requests from that class into disk in timely manner or the disk is slow and cannot afford timely dispatching.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 15 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_io_queue_starvation_time_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "metric": "Total time spent starving for disk", + "refId": "A", + "step": 30 + } + ], + "title": "DISK $classes starvation time by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 27, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Information by Task Group

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 28, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 29, + "panels": [], + "repeat": "group", + "title": "$group", + "type": "row" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. This graph shows how much time was spent in $group group", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 25 + }, + "id": 30, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Time used by [[by]] - $group", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer. This may cause latency issues", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 25 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Time spent in task quota violations by [[by]] - $group", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer.\n\n This graph shows the amount of time the group was waiting to get CPU time.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 25 + }, + "id": 32, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_scheduler_starvetime_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[1m])/1000) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Starvation time by [[by]] - $group", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Shares assigned to the $group. Shares determine how Scylla reactor distributes the task quotas between groups (Higher share gets more quotas)", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 25 + }, + "id": 33, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_scheduler_shares{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Scheduler shares [[by]] - $group", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 34, + "panels": [], + "repeat": "", + "title": "Internal node errors", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Internal node Errors - $cluster

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 35, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of Read requests that failed due to an 'unavailable' error", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 36, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Local Reads Error by [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of write requests that failed due to an 'unavailable' error", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 34 + }, + "id": 37, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Local Write Error by [[by]]", + "type": "timeseries" + }, + { + "class": "text_panel", + "content": "## ", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 38, + "isNew": true, + "links": [], + "mode": "markdown", + "options": {}, + "span": 4, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of Read requests that failed due to an 'unavailable' error", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 39, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Reads Unavailable Error by [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of write requests that failed on a local Node", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 40 + }, + "id": 40, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Write Unavailable Error by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of write requests that failed on a local Node", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 40 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Range Unavailable Error by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of AIO Errors", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "AIO Error by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Total number of abandoned failed futures, futures destroyed while still containing an exception.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Ignored Future By [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of C++ exceptions thrown.\n\n An exception by itself does not indicate a problem", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "C++ Exceptions [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 45, + "panels": [], + "repeat": "", + "title": "Commit Log", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Commit log Information

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 46, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "Holds the size of disk space in bytes reserved for data so far. A too high value indicates that we have some bottleneck in the writing to sstables path", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 55 + }, + "id": 47, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_disk_total_bytes{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Avg reserved disk space by [[by]]", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "Holds the size of disk space in bytes used for data so far. A too high value indicates that we have some bottleneck in the writing to sstables path", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 55 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_disk_active_bytes{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Avg used disk space by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts a number of times the flush() method was called for a file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 55 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(rate(scylla_commitlog_flush{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Avg flush by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Holds the current number of segments", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 55 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_segments{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Segments by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts a number of times a flush limit was exceeded. A non-zero value indicates that there are too many pending flush operations (see pending_flushes) and some of them will be blocked till the total amount of pending flush operations drops below 5.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 61 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(rate(scylla_commitlog_flush_limit_exceeded{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Avg flush limit exceeded by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Holds the number of currently pending allocations. A non-zero value indicates that we have a bottleneck in the disk write flow.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 61 + }, + "id": 52, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_pending_allocations{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Pending allocations by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts a number of requests blocked due to memory pressure. A non-zero value indicates that the commitlog memory quota is not enough to serve the required amount of requests.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 61 + }, + "id": 53, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_pending_flushes{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Pending flush by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Holds the current number of unused segments. A non-zero value indicates that the disk write path became temporary slow.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 61 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_unused_segments{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Unused segments by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Holds the number of not closed segments that still have some free space. This value should not get too high.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 67 + }, + "id": 55, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_commitlog_allocating_segments{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Allocating segments by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "dashproductreject": "no-your-pannels", + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 56, + "panels": [], + "repeat": "", + "title": "Your panels", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 57, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 58, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 59, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 60, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "Shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "classes", + "multi": true, + "name": "classes", + "options": [], + "query": "label_values(scylla_io_queue_delay,class)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "group", + "multi": true, + "name": "group", + "options": [], + "query": "label_values(scylla_scheduler_time_spent_on_task_quota_violations_ms,group)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "avg", + "value": "avg" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Advanced", + "uid": "advanced-5-2", + "version": 5 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/alternator.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/alternator.json new file mode 100644 index 00000000000..9ea60696a85 --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/alternator.json @@ -0,0 +1,4897 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "Scylla Cluster Metrics", + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n [[cluster]]
  The Alternator dashboard is only valid when using Alternator, check the documentation for more information.
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel", + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 4 + }, + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scylla_scylladb_current_version{job=\"scylla\", cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "Total Nodes", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel_fail", + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(250, 113, 0, 0.89)", + "rgba(255, 0, 0, 0.9)" + ], + "datasource": "prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 2, + "y": 4 + }, + "id": 3, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Unreachable", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "title": "Unreachable", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel_fail", + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(250, 113, 0, 0.89)", + "rgba(255, 0, 0, 0.9)" + ], + "datasource": "prometheus", + "description": "Number of nodes that reported their status as Starting or Joining", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 4, + "y": 4 + }, + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scylla_node_operation_mode<=2)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Joining", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "title": "Joining", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel_fail", + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(250, 113, 0, 0.89)", + "rgba(255, 0, 0, 0.9)" + ], + "datasource": "prometheus", + "description": "Number of nodes that reported their status as Leaving, Decommissioned, Draining or Drained", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 6, + "y": 4 + }, + "id": 5, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scylla_node_operation_mode>3)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Leaving", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "title": "Leaving", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "class": "percent_panel", + "datasource": "prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "class": "small_nodes_table", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 101 + }, + { + "id": "displayName", + "value": "Load" + }, + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svr" + }, + "properties": [ + { + "id": "custom.width", + "value": 85 + }, + { + "id": "displayName", + "value": "Version" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + }, + { + "id": "displayName", + "value": "Status" + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "rgb(87, 128, 193)", + "value": 3 + }, + { + "color": "red", + "value": 4 + } + ] + } + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "Starting", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Joining", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 3, + "text": "Normal", + "to": "", + "type": 1, + "value": "3" + }, + { + "from": "", + "id": 4, + "text": "Leaving", + "to": "", + "type": 1, + "value": "4" + }, + { + "from": "", + "id": 5, + "text": "Decommissioned", + "to": "", + "type": 1, + "value": "5" + }, + { + "from": "", + "id": 6, + "text": "Draining", + "to": "", + "type": 1, + "value": "6" + }, + { + "from": "", + "id": 7, + "text": "Drained", + "to": "", + "type": 1, + "value": "7" + }, + { + "from": "", + "id": 8, + "text": "Moving", + "to": "", + "type": 1, + "value": "8" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "CQL Information Dashboard. Warning indicates that there are potential issues in the CQL Optimization", + "url": "/d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "CQL Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "1", + "id": 1, + "text": "CQL Warnings", + "to": "20", + "type": 2, + "value": "" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(87, 128, 193)", + "value": null + }, + { + "color": "dark-orange", + "value": 1 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "CQL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "OS Information Dashboard, an Error indicates there are OS related errors", + "url": "/d/OS-[[dash_version]]/os-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "OS Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "0", + "id": 2, + "text": "OS Errors", + "to": "100000", + "type": 2, + "value": "0" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(87, 128, 193)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "OS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Cordinator and Replica node errors", + "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Errors", + "to": "", + "type": 1, + "value": "1" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(87, 128, 193)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": " " + } + ] + }, + { + "dashversion": [ + ">5.1", + ">2022.2" + ], + "matcher": { + "id": "byName", + "options": "Value #F" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "How many live nodes this node sees", + "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "Live" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Detailed view", + "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + } + ] + }, + { + "dashversion": [ + ">5.2", + ">2023.1" + ], + "matcher": { + "id": "byName", + "options": "Value #G" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 101 + }, + { + "id": "displayName", + "value": "Streaming" + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 7, + "scopedVars": { + "dc": { + "selected": false, + "text": "datacenter1", + "value": "datacenter1" + } + }, + "span": 4, + "targets": [ + { + "expr": "0*scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"} + on (instance) group_left() scylla_node_operation_mode{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} ) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "sum(scylla_errors:nodes_total{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "expr": "(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "dashversion": [ + ">5.1", + ">2022.2" + ], + "expr": "sum(scylla_gossip_live{cluster=~\"$cluster|$^\"})by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "dashversion": [ + ">5.2", + ">2023.1" + ], + "expr": "(min(scylla_streaming_finished_percentage{cluster=~\"$cluster|$^\", dc=~\"$dc\"}*100) by (instance) < 100) or on (instance) 0*sum(scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + } + ], + "title": "Nodes", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "instance", + "svr", + "Value #A", + "Value #B", + "Value #C", + "Value #D" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 4, + "Value #B": 5, + "Value #C": 2, + "Value #D": 1, + "instance": 0, + "svr": 3 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Size by $by", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of Alternator Actions", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "title": "Total Actions", + "type": "timeseries" + }, + { + "class": "alert_table", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "options": { + "alertInstanceLabelFilter": "job!=\"scylla_manager\",advisor=\"\"", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 3, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + }, + "span": 4, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": true, + "linkTooltip": "Jump to the see the node", + "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_4}&from=${__cell_0}", + "pattern": "Time", + "type": "date" + }, + { + "class": "hidden_column", + "pattern": "severity", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "alertname", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "cluster", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "monitor", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "summary", + "type": "hidden" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Jump to the see the node", + "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell}", + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "title": "Active Alerts", + "type": "alertlist" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 10, + "panels": [], + "repeat": "", + "title": "Data Plane Actions", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Data Plane Actions

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 11, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "GetItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"PutItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "PutItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"UpdateItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "UpdateItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DeleteItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 16, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchWriteItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "BatchWriteItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "dashversion": [ + ">4.4", + ">2021.1" + ], + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 17, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchGetItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "BatchGetItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 18, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Query\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Query by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 19, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Scan\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Scan by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "content": "

Data Plane Latencies

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 20, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 21, + "panels": [], + "repeat": "alternator_latency_ops", + "title": "$alternator_latency_ops", + "type": "row" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 40 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Completed $alternator_latency_ops", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 40 + }, + "id": 23, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average $alternator_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 40 + }, + "id": 24, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile $alternator_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 40 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile $alternator_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 26, + "panels": [], + "repeat": "", + "title": "Streams", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Streams Actions

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 27, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 49 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListStreams\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "ListStreams by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 49 + }, + "id": 29, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeStream\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DescribeStream by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 49 + }, + "id": 30, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetShardIterator\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "GetShardIterator by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 55 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetRecords\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "GetRecords by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "content": "

Streams Latencies

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 32, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 33, + "panels": [], + "repeat": "alternator_streams_latency_ops", + "title": "$alternator_streams_latency_ops", + "type": "row" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 64 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Completed $alternator_streams_latency_ops", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 64 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average $alternator_streams_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 64 + }, + "id": 36, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile $alternator_streams_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 64 + }, + "id": 37, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile $alternator_streams_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 38, + "panels": [], + "repeat": "", + "title": "Control plane", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Control Plane Actions

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 39, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 73 + }, + "id": 40, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"CreateTable\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "CreateTable by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 73 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteTable\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DeleteTable by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 73 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeTable\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DescribeTable by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListTables\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "ListTables by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeEndpoints\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DescribeEndpoints by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "dashversion": [ + ">5.1", + ">2022.2" + ], + "datasource": "prometheus", + "description": "The number of items deleted by their TTL", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 45, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_expiration_items_deleted{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Expired Item Deleted by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "content": "

Cache

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 46, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 6, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "plain_text", + "content": "

Timeouts

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 47, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 6, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of rows that were read from the cache, without needing to be fetched from storage.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 87 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cache Hits", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of rows that were not present in the cache, and had to be fetched from storage.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 87 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cache Misses", + "type": "timeseries" + }, + { + "class": "wpm_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 87 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Write Timeouts/Minutes by [[by]]", + "type": "timeseries" + }, + { + "class": "rpm_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 87 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Read Timeouts/Minutes by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 52, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 53, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 95 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 55, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": false, + "name": "mount_point", + "options": [], + "query": "node_filesystem_avail_bytes", + "refresh": 2, + "regex": "/mountpoint=\"([^\"]*)\".*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5-2", + "value": "5-2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "dash_version", + "options": [ + { + "selected": true, + "text": "5-2", + "value": "5-2" + } + ], + "query": "5-2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "All", + "value": "$__all" + }, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "alternator_latency_ops", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "GetItem", + "value": "GetItem" + }, + { + "selected": false, + "text": "PutItem", + "value": "PutItem" + }, + { + "selected": false, + "text": "UpdateItem", + "value": "UpdateItem" + }, + { + "selected": false, + "text": "DeleteItem", + "value": "DeleteItem" + } + ], + "query": "GetItem,PutItem,UpdateItem,DeleteItem", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "All", + "value": "$__all" + }, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "alternator_streams_latency_ops", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "GetRecords", + "value": "GetRecords" + } + ], + "query": "GetRecords", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "datasource": "prometheus", + "definition": "scylla_alternator_total_operations{cluster=~\"$cluster|$^\"}", + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "no_alternator", + "options": [], + "query": { + "query": "scylla_alternator_total_operations{cluster=~\"$cluster|$^\"}", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/^(scylla_alternator_total_operations)/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Alternator", + "uid": "alternator-5-2", + "version": 1 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/cql.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/cql.json new file mode 100644 index 00000000000..e83baa839f1 --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/cql.json @@ -0,0 +1,4538 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "CQL", + "overwrite": true, + "panels": [ + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "panels": [], + "repeat": "", + "title": "CQL By User", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 4, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

CQL By User - Coordinator

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL INSERT requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 5, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL SELECT commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Reads", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL DELETE requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 7, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL UPDATE requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "amount of CQL connections currently established", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(scylla_transport_current_connections{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Client CQL connections by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL batches command, each batched command is counted once", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 10, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL command batched. Each batch would add the number of commands inside the batch", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 11, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Command In Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the number of SELECT statements with BYPASS CACHE option", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "BYPASS CACHE", + "type": "timeseries" + }, + { + "class": "graph_panel", + "dashversion": [ + ">4.4", + ">2021.1" + ], + "datasource": "prometheus", + "description": "CQL errors by type, only active errors are shown", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_transport_cql_errors_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]],type) >0", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Errors [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL row reads", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Row Reads [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of reads using secondary indexes", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Secondary indexes Reads [[by]]", + "type": "timeseries" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 16, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "CQL System tables
The following information is based on Scylla Plugin configuration, check the documentation for more details on how to enable it.", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 17, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 18, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "dashversion": [ + ">4.4", + ">2020.1" + ], + "queryHost": "$node", + "queryText": "select address, port, shard_id, connection_stage, client_type, ssl_enabled, username, driver_name, driver_version, protocol_version from system.clients", + "refId": "A" + } + ], + "title": "Connection Table", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Connection", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 19, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "row_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 20, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "queryHost": "$node", + "queryText": "select keyspace_name, table_name,partition_key, clustering_key, row_size from system.large_rows", + "refId": "A" + } + ], + "title": "Large Rows", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Large Rows", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 21, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cell_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 22, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "queryHost": "$node", + "queryText": "select keyspace_name, table_name,partition_key, clustering_key, column_name,cell_size from system.large_cells", + "refId": "A" + } + ], + "title": "Large Cells", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Large Cells", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 23, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "partition_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 24, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "queryHost": "$node", + "queryText": "select keyspace_name, table_name,partition_key, partition_size from system.large_partitions", + "refId": "A" + } + ], + "title": "Large Partitions", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Large Partitions", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 25, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 26, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

CQL Internal - Coordinator

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL INSERT commands generated by intenal operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 59 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Internal Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL SELECT commands generated by intenal operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 59 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Internal Reads", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL DELETE requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 59 + }, + "id": 29, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL UPDATE requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 59 + }, + "id": 30, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + } + ], + "repeat": "", + "title": "CQL Internal", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 31, + "panels": [], + "repeat": "", + "title": "LWT", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 32, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

LWT

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL INSERT requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 68 + }, + "id": 33, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL DELETE requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 68 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the total number of CQL UPDATE requests with/without conditions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 68 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL batches command, each batched command is counted once", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 68 + }, + "id": 36, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 37, + "panels": [], + "repeat": "", + "title": "Optimization", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 38, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Optimization

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "All of the requests should be prepared\n\nPrepared statements remove the overhead of parsing the query every time and allow optimal routing of requests from client to server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 77 + }, + "id": 39, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(cql:non_system_prepared1m)/ (sum(cql:all_shardrate1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) - sum(cql:all_system_shardrate1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL Non-Prepared Statements", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "All of the requests should be prepared\n", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 77 + }, + "id": 40, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(cql:non_system_prepared1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL Non-Prepared Statements", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "All requests should be paged\n\nNon Paged request sources:\n- Client modifying the fetch size\n\nNon Paged requests require reading all the results and returning them in a single request.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 77 + }, + "id": 41, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * ((sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))/sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Non-Paged CQL Reads", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Non-Paged requests require reading all the results and returning them in a single request", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 77 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Non-Paged CQL Reads", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "All of the requests should be Token Aware\n\nNon Token Aware requests sources:\n* Non-Prepared Stamements\n* Client not using a Token Aware load balancing policy\n\nTokenAware requests are sent to a Scylla node that is also a replica. Token Un-Aware requests require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 77 + }, + "id": 43, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "scalar(sum(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) <=bool 0)*scalar(sum(rate(scylla_storage_proxy_replica_received_counter_updates{cluster=~\"$cluster|$^\"}[1m]))<=bool 0) *(100 - clamp_max(100*(sum(rate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) + sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/(sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) + 1,100)) OR vector(0)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Non-Token Aware", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Requests that are not token aware indicates that requests are not routed to the right node, which require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 77 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "scalar(sum(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) <=bool 0)*scalar(sum(rate(scylla_storage_proxy_replica_received_counter_updates{cluster=~\"$cluster|$^\"}[1m]))<=bool 0) *clamp_min(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) + rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) + rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) - sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]),0)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Non-Token Aware Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "Reversed CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with ORDER BY that is different from the \"CLUSTERING ORDER BY\" of the table\nAlternatives:\n\n* Denormalize your data (use a Materialized View)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 83 + }, + "id": 45, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Reversed CQL Reads", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Reversed CQL Reads entail additional processing on server side and should be avoided", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 83 + }, + "id": 46, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Reversed CQL Reads", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "ALLOW FILTERING CQL Reads, the percentage of read requests with 'ALLOW FILTERING'\n\nALLOW FILTERING CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned - check \n\"ALLOW FILTERING CQL Read Filtered Rows to check what percentage of the data is used\"\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 83 + }, + "id": 47, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "ALLOW FILTERING CQL Reads", + "transparent": true, + "type": "gauge" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Read requests with ALLOW FILTERING\n\nALLOW FILTERING CQL Reads entail additional processing on server side and should be avoided", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 83 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "ALLOW FILTERING CQL Reads", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "ALLOW FILTERING Filtered rows, the percentage of rows that were read and then filtered.\n\nALLOW FILTERING CQL Reads entail additional processing on server side. \nReading a row and then filter it is a waste of resources.\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 83 + }, + "id": 49, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) /sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "ALLOW FILTERING Filtered Rows", + "transparent": true, + "type": "gauge" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "CQL Queries with ALLOW FILTERING should be avoided.\nDropped rows are rows that were read but were filtered by the server.\nWhen dropped rows is relatively high you should consider the alternatives", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 83 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "rows read $node $shard", + "refId": "A" + }, + { + "expr": "sum(rate(scylla_cql_filtered_rows_matched_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "rows matched $node $shard", + "refId": "B" + }, + { + "expr": "sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rows dropped $node $shard", + "refId": "C" + } + ], + "title": "ALLOW FILTERING CQL Read Filtering", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "Range scans should typically by pass the cache.\n\n Add BYPASS CACHE to your select queries.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 89 + }, + "id": 51, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))/(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[1m])) )) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Range Scans", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Range scans should typically by pass the cache.\n\n Add BYPASS CACHE to your select queries.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 89 + }, + "id": 52, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_select_partition_range_scan_no_bypass_cache{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Range Scans Without BYPASS CACHE ", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 89 + }, + "id": 53, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL ANY Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 89 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL ANY CL Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 89 + }, + "id": 55, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL ALL CL Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 89 + }, + "id": 56, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL ALL CL Queries", + "type": "timeseries" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 57, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Cross DC Information

This section is relevant only if you have more than one DC
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "How many Queries use Consistency level ONE\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 98 + }, + "id": 58, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL ONE Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "How many Queries use Consistency level ONE\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 98 + }, + "id": 59, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL ONE CL Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "How many Queries use Consistency level QUORUM\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 98 + }, + "id": 60, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[1m]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL QUORUM CL Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "How many Queries use Consistency level QUORUM\n\nThis is an issue when using multiple datacenters.\n\nUsing consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 98 + }, + "id": 61, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL QUORUM CL Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "prometheus", + "description": "Cross DC traffic may cause additional latencies and network loads and in most cases, should be avoided.\n\nCross DC Read requests sources:\n- Consistency Level that is not LOCAL_XXX\n- Tables with read_repair_chance > 0\n\nNote:\n- If requests are supposed to be DC local - verify client is using a DCAware policy and a LOCAL_XX consistency level", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 104 + }, + "id": 62, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "repeat": "dc", + "span": 2, + "targets": [ + { + "expr": "100*(sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) - sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", datacenter=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) OR vector(0)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cross DC read requests $dc", + "transparent": true, + "type": "gauge" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 110 + }, + "id": 63, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5-2", + "value": "5-2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "dash_version", + "options": [ + { + "selected": true, + "text": "5-2", + "value": "5-2" + } + ], + "query": "5-2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Scylla CQL", + "uid": "cql-5-2", + "version": 1 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/detailed.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/detailed.json new file mode 100644 index 00000000000..20c26e58bc8 --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/detailed.json @@ -0,0 +1,9132 @@ +{ + "annotations": { + "class": "default_annotations", + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_hints_writes", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgb(255, 176, 0, 128)", + "limit": 100, + "name": "Hints Write", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints write", + "type": "tags" + }, + { + "class": "annotation_hints_sent", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "Hints Sent", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints Sent", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress{cluster=~\"$cluster|$^\"})>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + }, + { + "class": "ops_annotation", + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_node_ops_finished_percentage{cluster=~\"$cluster|$^\"}) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "ops", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Operation", + "type": "tags" + }, + { + "class": "stream_annotation", + "dashversion": [ + ">5.2", + ">2023.1" + ], + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_streaming_finished_percentage{cluster=~\"$cluster|$^\"}) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "streaming", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Streaming", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "percent_panel", + "datasource": "prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 2, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Amount of requests served as the coordinator. Imbalances here represent dispersion at the connection level, not your data model.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 3, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + ($func(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or max(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by([[by]])*0)", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Requests Served per [[by]] - Coordinator", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Counts the total number of successful user reads on this shard.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 4, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads per [[by]] - Replica", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Counts the total number of successful write operations performed by this shard.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 5, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes per [[by]] - Replica", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "repeat": "", + "title": "reads and writes", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Reads and Writes - Coordinator

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "writes_panel", + "datasource": "prometheus", + "description": "Foreground writes are writes that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_foreground_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Foreground Writes per [[by]]", + "type": "timeseries" + }, + { + "class": "writes_panel", + "datasource": "prometheus", + "description": "Background writes are writes that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_background_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Background Writes per [[by]]", + "type": "timeseries" + }, + { + "class": "reads_panel", + "datasource": "prometheus", + "description": "Foreground reads are reads that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 10, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_foreground_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Foreground Reads per [[by]]", + "type": "timeseries" + }, + { + "class": "reads_panel", + "datasource": "prometheus", + "description": "Background reads are reads that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 11, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_background_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Background Reads per [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of successfully written hints.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Hints Written per [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of sent hints.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Hints sent per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of times a digest read was done on behalf of a speculative retry.\n\nSpeculative retry is a mechanism that causes the client or server to speculate that a request may fail, and send a new request.\n\nspeculative retry may reduce latency in exchange for system load, but only if there is little activity.\n\nA lot of speculative retries increases load and can harm latency more than helping.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_speculative_digest_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Speculative Digest Reads By [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of times a read was done on behalf of a speculative retry.\n\nSpeculative retry is a mechanism that causes the client or server to speculate that a request may fail, and send a new request.\n\nspeculative retry may reduce latency in exchange for system load, but only if there is little activity.\n\nA lot of speculative retries increases load and can harm latency more than helping.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_speculative_data_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Speculative Data Reads By [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 16, + "panels": [], + "repeat": "", + "title": "Timeouts and Errors", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Timeouts and Errors - Coordinator

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 17, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 18, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Write Timeouts/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Requests that Scylla did not even try to write because replicas that were needed to execute this write were unavailable. Unavailable writes are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 19, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Write Unavailable/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) + rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])+rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Read Timeouts/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Requests that Scylla did not even try to read because replicas that were needed to execute this write were unavailable. Unavailable reads are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "title": "Read Unavailable/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 22, + "panels": [], + "repeat": "", + "title": "Replica", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 23, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "reads_panel", + "datasource": "prometheus", + "description": "The number of currently active read operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 24, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_active_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Active reads", + "type": "timeseries" + }, + { + "class": "reads_panel", + "datasource": "prometheus", + "description": "number of currently queued read operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_queued_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Queued reads", + "type": "timeseries" + }, + { + "class": "writes_panel", + "datasource": "prometheus", + "description": "The current number of requests blocked due to reaching the memory quota. Non-zero value indicates that our bottleneck is memory", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_requests_blocked_memory_current{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes currently blocked on dirty", + "type": "timeseries" + }, + { + "class": "writes_panel", + "datasource": "prometheus", + "description": "number of currently pending allocations. A non-zero value indicates that we have a bottleneck in the disk write flow.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_commitlog_pending_allocations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes currently blocked on commitlog", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Reciprocal Miss Rate is a score in the range of 1 to 100 that is used to decide the fraction of read requests to send to each replica - a replica with twice the RMR value of another replica will serve twice the number of read requests.\n\nRMR is calculated on a table level, this is an aggregate estimation of that score.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "clamp_max(1 + sum((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) by ([[by]])/(sum(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 0.00001),100)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reciprocal Miss Rate (HWLB)", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Counts the total number of failed user read operations. Add the total_reads to this value to get the total amount of reads issued on this shard.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 29, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads failed", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Holds the current number of requests blocked due to reaching the memory quota (15566635008B). Non-zero value indicates that our bottleneck is memory and more specifically - the memory quota allocated for the \"database\" component.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 30, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes blocked on dirty", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Counts number of requests blocked due to memory pressure. A non-zero value indicates that the commitlog memory quota is not enough to serve the required amount of requests.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes blocked on commitlog", + "type": "timeseries" + }, + { + "class": "requestsps_panel", + "datasource": "prometheus", + "description": "Holds an incrementing counter with the requests that were shed due to overload (threshold configured via max_concurrent_requests_per_shard). The first derivative of this value shows how often we shed requests due to overload in the \"CQL transport\" component.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:requests/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 32, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_transport_requests_shed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Requests Shed", + "type": "timeseries" + }, + { + "class": "text_panel", + "content": "", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 48 + }, + "id": 33, + "isNew": true, + "links": [], + "mode": "markdown", + "options": {}, + "span": 3, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Counts the total number of failed write operations. A sum of this value plus total_writes represents a total amount of writes attempted on this shard.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 48 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes failed", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Counts write operations failed due to a timeout. A positive value is a sign of storage being overloaded.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 48 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes timed out", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 36, + "panels": [], + "repeat": "", + "title": "Cache", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Cache - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 37, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "number of reads that were served from the cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 38, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads with no misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "number of reads which had to read from sstables", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 39, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads with misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "total number of rows needed by reads and found in cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 40, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Hits", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "number of partitions needed by reads and found in cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Hits", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "total number of rows needed by reads and missing in cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 63 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "number of partitions needed by reads and missing in cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 63 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "total number of rows added to cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 69 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Insertions", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "total number of partitions added to cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 69 + }, + "id": 45, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Insertions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "total number of rows evicted from cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 69 + }, + "id": 46, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Evictions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "total number of evicted partitions", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 69 + }, + "id": 47, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Evictions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "total number of rows in memtables which were merged with existing rows during cache update on memtable flush", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 75 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Merges", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "total number of partitions merged", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 75 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Merges", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "total number of invalidated rows", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 75 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Removals", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "total number of invalidated partitions", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 75 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Removals", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "total number of cached rows", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 81 + }, + "id": 52, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_rows{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Rows", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "total number of cached partitions", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 81 + }, + "id": 53, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_partitions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partitions", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "current bytes used by the cache out of the total size of memory", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 81 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_bytes_used{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Used Bytes", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "total size of memory for the cache", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 81 + }, + "id": 55, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Total Bytes", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Counts the number of prepared statements cache entries evictions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 87 + }, + "id": 56, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cql_prepared_cache_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Prepared Statements Cache Eviction", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Counts the number of authenticated prepared statements cache entries evictions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 87 + }, + "id": 57, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cql_authorized_prepared_statements_cache_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Authorized Prepared Statements Cache Eviction", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 58, + "panels": [ + { + "class": "text_panel", + "content": "

Materialized Views - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 59, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of view update locally", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 96 + }, + "id": 60, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "View Local Update", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of view update remotely", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 96 + }, + "id": 61, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "View Remote Update", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "Size in bytes of the view update backlog at each base replica.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 96 + }, + "id": 62, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_view_update_backlog{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "View Update Backlog", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of dropped view updates due to an excessive view update backlog.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 96 + }, + "id": 63, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Dropped View Updates", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of hints sent for view.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 102 + }, + "id": 64, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Hints for view", + "type": "timeseries" + }, + { + "class": "writes_panel", + "datasource": "prometheus", + "description": "Currently throttled base writes, as a consequence of the respective view update backlog.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 102 + }, + "id": 65, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_current_throttled_base_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Throttled Base Writes", + "type": "timeseries" + } + ], + "repeat": "", + "title": "Materialized Views", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "dashversion": [ + ">4.5", + ">2021.1" + ], + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 66, + "panels": [ + { + "class": "rps_panel", + "dashversion": [ + ">4.6", + ">2021.1" + ], + "datasource": "prometheus", + "description": "Amount of range tombstones processed during read.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 109 + }, + "id": 67, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_sstables_range_tombstone_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Range Tombstones reads", + "type": "timeseries" + }, + { + "class": "rps_panel", + "dashversion": [ + ">4.6", + ">2021.1" + ], + "datasource": "prometheus", + "description": "Amount of range tombstones processed during read.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 109 + }, + "id": 68, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_range_tombstone_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cache Range Tombstones Read", + "type": "timeseries" + }, + { + "class": "rps_panel", + "dashversion": [ + ">4.6", + ">2021.1" + ], + "datasource": "prometheus", + "description": "Amount of row tombstones read", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 109 + }, + "id": 69, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_sstables_row_tombstone_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Row Tombstones reads", + "type": "timeseries" + }, + { + "class": "rps_panel", + "dashversion": [ + ">4.6", + ">2021.1" + ], + "datasource": "prometheus", + "description": "Amount of cache row tombstones read", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 109 + }, + "id": 70, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_tombstone_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cache Row Tombstones reads", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Amount of tombstones writes.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 115 + }, + "id": 71, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_sstables_tombstone_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Tombstones Writes", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Amount of range tombstones writes.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 115 + }, + "id": 72, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_sstables_range_tombstone_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Range Tombstones Writes", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Amount of Cell Tombstones Writes.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 115 + }, + "id": 73, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_sstables_cell_tombstone_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cell Tombstones Writes", + "type": "timeseries" + } + ], + "repeat": "", + "title": "Tombstones", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 74, + "panels": [ + { + "class": "text_panel", + "content": "

LWT - Coordinator

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 75, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "LWT read rate.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 124 + }, + "id": 76, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "description": "LWT Avrage Read latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 124 + }, + "id": 77, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(casrlatencya{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"} or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Avrage Read latency", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "description": "LWT 95% Read latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 124 + }, + "id": 78, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "casrlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "95% latency", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "LWT Read Timeouts", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 124 + }, + "id": 79, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Timeouts", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "LWT write rate.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 130 + }, + "id": 80, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on ([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "description": "LWT Avrage Write latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 130 + }, + "id": 81, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(caswlatencya{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", by=\"[[by]]\"} or on([[by]]) ($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 1))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Avrage Write latency", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "description": "LWT 95% write latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 130 + }, + "id": 82, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "caswlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "95% latency", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "LWT Write Timeouts", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 130 + }, + "id": 83, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Timeouts", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "A single Read/Write LWT will result in multiple paxos operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 136 + }, + "id": 84, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Paxos operations", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "How many paxos operations that did not yet produce a result are running", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 136 + }, + "id": 85, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_cas_foreground{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Paxos Foreground operations", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "How many paxos operations are still running after a result was alredy returned", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 136 + }, + "id": 86, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_cas_background{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Paxos Background operations", + "type": "timeseries" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 136 + }, + "id": 87, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 3, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "An LWT INSERT, UPDATE or DELETE command that involves a condition will be rejected if the condition is not met.\n\nWhile it is ok, a high value may indicate that there is a potential problem with data distribution", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 142 + }, + "id": 88, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_condition_not_met{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Condition-Not-Met", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of times some INSERT, UPDATE or DELETE request with conditions had to retry because there was a concurrent conditional statement against the same key. Each retry is performed after a randomized sleep interval, so it can lead to statement timing out completely.\n\nIt can indicates contention over a hot row or key", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 142 + }, + "id": 89, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_contention_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Contention", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of times some SELECT with SERIAL consistency had to retry because there was a concurrent conditional statement against the same key. Each retry is performed after a randomized sleep interval, so it can lead to statement timing out completely.\n\nIt can indicates contention over a hot row or key", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 142 + }, + "id": 90, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_contention_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]]) - $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", le=\"1.000000\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Contention", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of partially succeeded conditional statements. These statements were not committed by the coordinator, due to some replicas responding with errors or timing out. The coordinator had to propagate the error to the client. However, the statement succeeded on a minority of replicas, so may later be propagated to the rest during repair.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 142 + }, + "id": 91, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_timeout_due_to_uncertainty{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Timeout Due to Uncertainty", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of times a INSERT, UPDATE, or DELETE with conditions failed after being unable to contact enough replicas to match the consistency level", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 148 + }, + "id": 92, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Unavailable", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of times a SELECT with SERIAL consistency failed after being unable to contact enough replicas to match the consistency level", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 148 + }, + "id": 93, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Unavailable", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "Number of Paxos-repairs of INSERT, UPDATE, or DELETE with conditions.\n\nA repair is necessary when a previous Paxos statement was partialy successful. A subsequent statement then may not proceed before completing the work of its predecessor. A repair is not guaranteed to succeed, the metric indicates the number of repair attempts made", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 148 + }, + "id": 94, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_unfinished_commit{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Unfinished - Repair Attempts", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Number of Paxos-repairs of SELECT statement with SERIAL consistency.\n\nA repair is necessary when a previous Paxos statement was partialy successful. A subsequent statement then may not proceed before completing the work of its predecessor. A repair is not guaranteed to succeed, the metric indicates the number of repair attempts made", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 148 + }, + "id": 95, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_unfinished_commit{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Unfinished - Repair Attempts", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "Normally, a PREPARE Paxos-round piggy-backs the previous value along with the PREPARE response. When the coordinator is unable to obtain the previous value (or its digest) from some of the participants, or when the digests did not match, a separate repair round has to be performed.\n\nThis indicates that some Paxos queries did not run successfully to completion, e.g. because some node is overloaded, down, or there was contention around a key.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 154 + }, + "id": 96, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_failed_read_round_optimization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Failed Read-Round Optimization", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of pruning requests.\n\nA successful conditional statement deletes the intermediate state from system.paxos table using PRUNE command.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 154 + }, + "id": 97, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_prune{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Prune", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of Dropped pruning requests.\n\nA successful conditional statement deletes the intermediate state from system.paxos table using PRUNE command. If the system is busy it may not keep up with the PRUNE requests, so such requests are dropped.\n\nHigh value suggests the system is overloaded and also that system.paxos table is taking up space. If a prune is dropped, system.paxos table key and value for respective LWT transaction will stay around until next transaction against the same key or until the gc_grace_period, when it's removed by compaction.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 154 + }, + "id": 98, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_dropped_prune{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Dropped Prune", + "type": "timeseries" + } + ], + "repeat": "", + "title": "LWT", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 160 + }, + "id": 99, + "panels": [ + { + "class": "text_panel", + "content": "

CDC - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 161 + }, + "id": 100, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "The rate of CDC operations.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 163 + }, + "id": 101, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_cdc_operations_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "CDC Operations", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "The rate of failed CDC operations.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 163 + }, + "id": 102, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_cdc_operations_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Failed CDC operations", + "type": "timeseries" + } + ], + "repeat": "", + "title": "CDC", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 169 + }, + "id": 103, + "panels": [], + "repeat": "", + "title": "Memory", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Memory - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 170 + }, + "id": 104, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "Holds a current size of allocated memory in bytes.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 172 + }, + "id": 105, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(scylla_lsa_total_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "LSA total memory", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "Holds a current amount of used non-LSA memory.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 172 + }, + "id": 106, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(scylla_lsa_non_lsa_used_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Non-LSA used memory", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 178 + }, + "id": 107, + "panels": [], + "repeat": "", + "title": "Compaction", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Compaction - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 179 + }, + "id": 108, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "graph_panel_int", + "datasource": "prometheus", + "description": "Holds the number of currently active compactions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 181 + }, + "id": 109, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 4, + "targets": [ + { + "expr": "$func(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Running Compactions", + "type": "timeseries" + }, + { + "class": "percent_panel", + "datasource": "prometheus", + "description": "Percentage of CPU time used by compaction", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 181 + }, + "id": 110, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[1m])) by ([[by]]))/10", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Compactions CPU Runtime", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Shares assigned to the compaction", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 181 + }, + "id": 111, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(scylla_scheduler_shares{group=\"compaction\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Compactions Shares", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 187 + }, + "id": 112, + "panels": [], + "repeat": "scheduling_group", + "title": "Latencies - $scheduling_group", + "type": "row" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "description": "The general write latency histogram", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 188 + }, + "id": 113, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 188 + }, + "id": 114, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencya{by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 188 + }, + "id": 115, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group|$\"}", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 188 + }, + "id": 116, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "description": "The general read latency histogram", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 194 + }, + "id": 117, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]]) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 194 + }, + "id": 118, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencya{by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 194 + }, + "id": 119, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 194 + }, + "id": 120, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "dashproductreject": "no-your-pannels", + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 200 + }, + "id": 121, + "panels": [], + "repeat": "", + "title": "Your panels", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 201 + }, + "id": 122, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 203 + }, + "id": 123, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 203 + }, + "id": 124, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 209 + }, + "id": 125, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "tags": [], + "text": [ + "statement" + ], + "value": [ + "statement" + ] + }, + "dashversion": [ + ">4.3" + ], + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "SG", + "multi": true, + "name": "scheduling_group", + "options": [], + "query": "label_values(all_scheduling_group{cluster=~\"$cluster|$^\"}, scheduling_group_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Detailed", + "uid": "detailed-5-2", + "version": 5 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/ks.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/ks.json new file mode 100644 index 00000000000..58871213b21 --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/ks.json @@ -0,0 +1,1490 @@ +{ + "annotations": { + "class": "default_annotations", + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_hints_writes", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgb(255, 176, 0, 128)", + "limit": 100, + "name": "Hints Write", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints write", + "type": "tags" + }, + { + "class": "annotation_hints_sent", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "Hints Sent", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints Sent", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress{cluster=~\"$cluster|$^\"})>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + }, + { + "class": "ops_annotation", + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_node_ops_finished_percentage{cluster=~\"$cluster|$^\"}) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "ops", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Operation", + "type": "tags" + }, + { + "class": "stream_annotation", + "dashversion": [ + ">5.2", + ">2023.1" + ], + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_streaming_finished_percentage{cluster=~\"$cluster|$^\"}) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "streaming", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Streaming", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
[[cluster]]
Keyspace and table level metrics are not available. To enable, set enable_keyspace_column_family_metrics to true in scylla.yaml. Note this has a significant effect on the monitoring stack sizing.
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "panels": [ + { + "class": "wps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 3, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_column_family_write_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 4, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyaks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 5, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp95ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp99ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 7, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_column_family_read_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyaks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp95ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 10, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp99ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile read latency by [[by]]", + "type": "timeseries" + } + ], + "repeat": "table", + "title": "Latencies - $ks:$table", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 11, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 12, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 15, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "ks", + "multi": false, + "name": "ks", + "options": [], + "query": "label_values(scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"},ks)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "table", + "multi": true, + "name": "table", + "options": [], + "query": "label_values(scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\", ks=\"$ks\"},cf)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "datasource": "prometheus", + "definition": "scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"}", + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "no_ks", + "options": [], + "query": { + "query": "scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"}", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/^(scylla_column_family_write_latency_count)/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Keyspace", + "uid": "ks-5-2", + "version": 5 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/manager.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/manager.json new file mode 100644 index 00000000000..84880a9a70f --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/manager.json @@ -0,0 +1,1630 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_manager_task_failed", + "datasource": "prometheus", + "enable": true, + "expr": "sum(changes(scylla_manager_task_run_total{status=\"ERROR\", cluster=~\"$cluster|$^\"}[$__rate_interval])) by(type)>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Failed", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Task Failed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "Scylla Manager Metrics", + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster_name]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 0, + "text": "Online", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 1, + "text": "Offline", + "to": "", + "type": 1, + "value": "1" + } + ], + "noValue": " Offline", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "span": 3, + "targets": [ + { + "expr": "count(scrape_samples_scraped{job=\"scylla_manager\"}==0) OR vector(0)", + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Manager", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 3 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_manager_healthcheck_cql_rtt_ms{cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Nodes", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of nodes that Scylla manager CQL probe failed connecting to. It could indicate a network or a node problem", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 4, + "y": 3 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_manager_healthcheck_cql_status{cluster=~\"$cluster|$^\"}==-1) OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Nodes without CQL connection", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "CQL Fail", + "type": "stat" + }, + { + "class": "vertical_lcd", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 6, + "y": 3 + }, + "id": 5, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "span": 2, + "targets": [ + { + "expr": "manager:repair_progress{cluster=~\"[[cluster]]\"}", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Repair", + "type": "bargauge" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The time of the last successful repair", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "noValue": "Never", + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "dateTimeAsIsoNoDateIfToday" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 8, + "y": 3 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "manager:repair_done_ts{cluster=\"$cluster\"}*1000", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Last repair", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The time of the last failed repair", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "noValue": "Never", + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "dateTimeAsIsoNoDateIfToday" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 10, + "y": 3 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "manager:repair_fail_ts{cluster=\"$cluster\"}*1000", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Last Failure", + "type": "stat" + }, + { + "class": "vertical_lcd", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 3 + }, + "id": 8, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "span": 2, + "targets": [ + { + "expr": "manager:backup_progress{cluster=~\"[[cluster]]\"}*100", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Backup", + "type": "bargauge" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The time of the last successful backup", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "noValue": "Never", + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "dateTimeAsIsoNoDateIfToday" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 3 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "manager:backup_done_ts{cluster=\"$cluster\"}*1000", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Last Backup", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The time of the last failed backup", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "noValue": "Never", + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "dateTimeAsIsoNoDateIfToday" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 16, + "y": 3 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "manager:backup_fail_ts{cluster=\"$cluster\"}*1000", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Last Failure", + "type": "stat" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 11, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 2, + "targets": [ + { + "expr": "manager:repair_progress{cluster=~\"[[cluster]]\"}", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Repair Progress", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "si:tr/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 7 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 5, + "targets": [ + { + "expr": "sum(rate(scylla_manager_repair_token_ranges_success{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Repair Token-Range Rate", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "si:tr/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 7 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 5, + "targets": [ + { + "expr": "sum(rate(scylla_manager_repair_token_ranges_error{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Repair Token-Range Error Rate", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 2, + "targets": [ + { + "expr": "manager:backup_progress{cluster=~\"[[cluster]]\"}", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Backup Progress", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 4, + "y": 13 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(scylla_manager_backup_files_uploaded_bytes{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}) by ([[by]])", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Uploaded bytes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 13 + }, + "id": 16, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(scylla_manager_backup_files_skipped_bytes{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}) by ([[by]])", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Uploaded skipped", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 16, + "y": 13 + }, + "id": 17, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(scylla_manager_backup_files_failed_bytes{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}) by ([[by]])", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Uploaded Failed", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Shows restore progress, the remaining bytes to complete", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 18, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "sum(scylla_manager_restore_remaining_bytes{cluster=~\"[[cluster]]\", instance=~\"$instance\", shard=~\"$shard\"}) by ([[by]])", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Restore Remaining bytes", + "type": "timeseries" + }, + { + "class": "ms_panel", + "datasource": "prometheus", + "description": "Average duration time for a CQL ping operation", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 25 + }, + "id": 19, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 5, + "targets": [ + { + "expr": "avg(scylla_manager_healthcheck_cql_rtt_ms{instance=~\"$instance\", cluster=~\"[[cluster]]\"}) by ($by)", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "CQL probe duration by $by", + "type": "timeseries" + }, + { + "class": "text_panel", + "content": "

Manager Agent

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 20, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "An increase in memory can indicate an issue with the Manager agent", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 33 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 5, + "targets": [ + { + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"manager_agent.?\", instance=~\"$instance\"}) by ($by)", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Agent Memory Heap usage", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "An increase in the number of threads can indicate an issue with the Manager agent", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 33 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 5, + "targets": [ + { + "expr": "avg(go_threads{job=~\"manager_agent.?\", instance=~\"$instance\"}) by ($by)", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Agent Threads", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 23, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "3.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "Cluster", + "value": "cluster" + }, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": true, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "Keyspace", + "value": "keyspace" + }, + { + "selected": false, + "text": "Shard", + "value": "shard" + } + ], + "query": "Instance,Shard,Keyspace,Cluster", + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster_name", + "options": [], + "query": "label_values(scylla_manager_cluster_name,name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 2, + "includeAll": false, + "label": "cluster_id", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_manager_cluster_name{name=\"$cluster_name\"}, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(scylla_manager_healthcheck_cql_rtt_ms{cluster=~\"$cluster|$^\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_manager_repair_segments_total, shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "3.2", + "value": "3.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "3.2", + "value": "3.2" + } + ], + "query": "3.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Scylla Manager Metrics", + "uid": "manager-3-2", + "version": 3 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/os.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/os.json new file mode 100644 index 00000000000..6b958946f8a --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/os.json @@ -0,0 +1,2162 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "aliasColors": {}, + "class": "piechart_panel_percent", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 5 + }, + "height": "250px", + "id": 4, + "isNew": true, + "links": [], + "options": { + "displayLabels": [ + "name", + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "repeat": "node", + "targets": [ + { + "expr": "sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Free", + "metric": "", + "refId": "A", + "step": 7200 + }, + { + "expr": "(sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "B", + "step": 7200 + } + ], + "title": "Total Storage $node", + "type": "piechart" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 5, + "panels": [], + "repeat": "mount_point", + "title": "Partition $mount_point", + "type": "row" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "1-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])/sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Used disk by $by", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 7, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Used Bytes by $by", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_filesystem_free_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Free Bytes by $by", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_filesystem_files{mountpoint=\"$mount_point\", instance=~\"$node\"}- node_filesystem_files_free{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Number of files by $by", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 10, + "panels": [], + "repeat": "monitor_disk", + "title": "Disk $monitor_disk", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Disk $monitor_disk

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_writes_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Writes per $by", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_reads_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Reads per $by", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_bytes_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Writes Bps per $by", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_bytes_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Read Bps per $by", + "type": "timeseries" + }, + { + "class": "seconds_panel", + "datasource": "prometheus", + "description": "The average read time from disk", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 16, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "rate(node_disk_read_time_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])/rate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Read AWait per instance", + "type": "timeseries" + }, + { + "class": "seconds_panel", + "datasource": "prometheus", + "description": "The average write time to disk", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 29 + }, + "id": 17, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "rate(node_disk_write_time_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])/rate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Write AWait per instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 18, + "panels": [], + "repeat": "monitor_network_interface", + "title": "Network Interface $monitor_network_interface", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Network $monitor_network_interface

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 19, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "pps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_receive_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_receive_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Rx Packets", + "type": "timeseries" + }, + { + "class": "pps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_transmit_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Tx Packets", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_receive_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Rx Bps", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 23, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_transmit_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Tx Bps", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 24, + "panels": [], + "title": "CPU and Memory", + "type": "row" + }, + { + "class": "bps_panel", + "datasource": "prometheus", + "description": "The available memory, note that in a production environment we expect this to be low, Scylla would use most of the available memory when possible", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Available memory", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "description": "Percent of available memory, note that in a production environment we expect this to be low, Scylla would use most of the available memory when possible", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 51 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])/sum(node_memory_MemTotal_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Available memory", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "prometheus", + "description": "Percent of CPU used, note that in production Scylla would try to use most of the CPU and this is not a problem", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 51 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 3, + "targets": [ + { + "expr": "1-sum(rate(node_cpu_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", mode=\"idle\"}[3m])) by ([[by]])/count(node_cpu_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", mode=\"idle\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CPU used", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "CPU frequency should be set for performance.\n\n The current frequency should match the max frequency. If that is not the case, check your host configuration.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 51 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + { + "$$hashKey": "object:211", + "alias": "Max", + "color": "#F2495C" + } + ], + "span": 3, + "targets": [ + { + "expr": "max(node_cpu_scaling_frequency_max_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) or on() max(node_cpu_frequency_max_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"})", + "intervalFactor": 1, + "legendFormat": "Max", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "min(node_cpu_scaling_frequency_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]]) or on() min(node_cpu_frequency_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "CPU Frequency", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 29, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(node_filesystem_avail_bytes, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": true, + "text": "Shard", + "value": "instance,cpu" + } + ], + "query": "Cluster,DC,Instance", + "type": "custom" + }, + { + "allValue": null, + "current": { + "isNone": true, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "monitor_disk", + "options": [], + "query": "node_disk_read_bytes_total", + "refresh": 2, + "regex": "/.*device=\"([^\\\"]*)\".*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "isNone": true, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "monitor_network_interface", + "options": [], + "query": "node_network_receive_packets_total", + "refresh": 2, + "regex": "/.*device=\"([^\\\"]*)\".*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "prometheus", + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": true, + "name": "mount_point", + "options": [], + "query": "node_filesystem_avail_bytes", + "refresh": 2, + "regex": "/mountpoint=\"([^\"]*)\".*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "OS Metrics", + "uid": "OS-5-2", + "version": 5 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/platform/overview.json b/assets/monitoring/grafana/v1alpha1/dashboards/platform/overview.json new file mode 100644 index 00000000000..63f24f04320 --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/platform/overview.json @@ -0,0 +1,4181 @@ +{ + "annotations": { + "class": "default_annotations", + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_hints_writes", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgb(255, 176, 0, 128)", + "limit": 100, + "name": "Hints Write", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints write", + "type": "tags" + }, + { + "class": "annotation_hints_sent", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written{cluster=~\"$cluster|$^\"}[$__rate_interval])>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "Hints Sent", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints Sent", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress{cluster=~\"$cluster|$^\"})>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + }, + { + "class": "ops_annotation", + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_node_ops_finished_percentage{cluster=~\"$cluster|$^\"}) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "ops", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Operation", + "type": "tags" + }, + { + "class": "stream_annotation", + "dashversion": [ + ">5.2", + ">2023.1" + ], + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_streaming_finished_percentage{cluster=~\"$cluster|$^\"}) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "streaming", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Streaming", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "Scylla Cluster Metrics", + "overwrite": true, + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Cluster overview $cluster", + "type": "row" + }, + { + "class": "text_panel", + "dashproductreject": "no-version-check", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 9, + "x": 0, + "y": 1 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 9, + "y": 1 + }, + "id": 3, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 13, + "y": 1 + }, + "id": 4, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 7, + "x": 17, + "y": 1 + }, + "id": 5, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of nodes configured in the cluster.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 2 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_scylladb_current_version{job=\"scylla\", cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "# Nodes", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of unreachable nodes.\nUsually because a machine is down or unreachable.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 2 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0))", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unreachable", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The number of joining and leaving nodes.\nThe number of nodes that are up but not actively part of the cluster, either because they are still joining or because they are leaving.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 4, + "y": 2 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_node_operation_mode{cluster=~\"$cluster|$^\"}!=3)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Inactive", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 0, + "text": "Backup", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 1, + "text": "Repair", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Online", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Offline", + "to": "", + "type": 1, + "value": "-1" + } + ], + "noValue": " Offline", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 6, + "y": 2 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(scylla_manager_task_active_count{type=~\"repair\",cluster=~\"$cluster|$^\"}) or on() vector(0)) + (sum(scylla_manager_task_active_count{type=~\"backup\",cluster=~\"$cluster|$^\"})*2 or on() vector(0)) + (sum(scylla_manager_server_current_version{}) or on() vector(-1))", + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Manager", + "type": "stat" + }, + { + "class": "vertical_lcd", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 1, + "x": 8, + "y": 2 + }, + "id": 10, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(avg(scylla_manager_repair_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"repair\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0)) + (sum(avg(scylla_manager_backup_percent_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"backup\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "bargauge" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "Average Write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 9, + "y": 2 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 11, + "y": 2 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "Average Read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 13, + "y": 2 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 15, + "y": 2 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$sg\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 17, + "y": 2 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests/s", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 2 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 2 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Timeouts", + "type": "stat" + }, + { + "class": "alert_table", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 6 + }, + "options": { + "alertInstanceLabelFilter": "job!=\"scylla_manager\",advisor=\"\"", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 3, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + }, + "span": 4, + "title": "Active Alerts", + "type": "alertlist" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 18, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(wlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$sg\"}>0)", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$sg\"}>0)", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B", + "step": 1 + } + ], + "title": "Write Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A", + "step": 1 + } + ], + "title": "Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 21, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(rlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$sg\"}>0)", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$sg\"}>0)", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B", + "step": 1 + } + ], + "title": "Read Latencies", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 22, + "panels": [], + "repeat": "dc", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Information for $dc

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 23, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "vertical_lcd", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 1, + "x": 0, + "y": 15 + }, + "id": 24, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "bargauge" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "description": "The average Disk usage per [[by]].\n\n The dashed line represent the total size.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 1, + "y": 15 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "class": "desc_tooltip_options", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 5, + "targets": [ + { + "expr": "Avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])-avg(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Avg Usage {{[[by]]}}", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])", + "interval": "", + "legendFormat": "Size {{[[by]]}}", + "refId": "B" + } + ], + "title": "Average Disk Usage", + "type": "timeseries" + }, + { + "class": "graph_panel_int", + "datasource": "prometheus", + "description": "Holds the number of currently active compactions.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 15 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 2, + "targets": [ + { + "expr": "$func(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Running Compactions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "The Hits and Misses", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 8, + "y": 15 + }, + "id": 27, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Hit {{[[by]]}}", + "refId": "A", + "step": 10 + }, + { + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Misses {{[[by]]}}", + "refId": "B", + "step": 10 + } + ], + "title": "Cache Hits/Misses", + "type": "timeseries" + }, + { + "class": "small_nodes_table", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 101 + }, + { + "id": "displayName", + "value": "Load" + }, + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svr" + }, + "properties": [ + { + "id": "custom.width", + "value": 85 + }, + { + "id": "displayName", + "value": "Version" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + }, + { + "id": "displayName", + "value": "Status" + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "rgb(87, 128, 193)", + "value": 3 + }, + { + "color": "red", + "value": 4 + } + ] + } + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "Starting", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Joining", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 3, + "text": "Normal", + "to": "", + "type": 1, + "value": "3" + }, + { + "from": "", + "id": 4, + "text": "Leaving", + "to": "", + "type": 1, + "value": "4" + }, + { + "from": "", + "id": 5, + "text": "Decommissioned", + "to": "", + "type": 1, + "value": "5" + }, + { + "from": "", + "id": 6, + "text": "Draining", + "to": "", + "type": 1, + "value": "6" + }, + { + "from": "", + "id": 7, + "text": "Drained", + "to": "", + "type": 1, + "value": "7" + }, + { + "from": "", + "id": 8, + "text": "Moving", + "to": "", + "type": 1, + "value": "8" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "CQL Information Dashboard. Warning indicates that there are potential issues in the CQL Optimization", + "url": "/d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "CQL Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "1", + "id": 1, + "text": "CQL Warnings", + "to": "20", + "type": 2, + "value": "" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(87, 128, 193)", + "value": null + }, + { + "color": "dark-orange", + "value": 1 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "CQL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "OS Information Dashboard, an Error indicates there are OS related errors", + "url": "/d/OS-[[dash_version]]/os-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "OS Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "0", + "id": 2, + "text": "OS Errors", + "to": "100000", + "type": 2, + "value": "0" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(87, 128, 193)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "OS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Cordinator and Replica node errors", + "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Errors", + "to": "", + "type": 1, + "value": "1" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(87, 128, 193)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": " " + } + ] + }, + { + "dashversion": [ + ">5.1", + ">2022.2" + ], + "matcher": { + "id": "byName", + "options": "Value #F" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "How many live nodes this node sees", + "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "Live" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Detailed view", + "url": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}&from=${__from}&to=${__to}" + } + ] + } + ] + }, + { + "dashversion": [ + ">5.2", + ">2023.1" + ], + "matcher": { + "id": "byName", + "options": "Value #G" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 101 + }, + { + "id": "displayName", + "value": "Streaming" + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 10, + "x": 14, + "y": 15 + }, + "id": 28, + "scopedVars": { + "dc": { + "selected": false, + "text": "datacenter1", + "value": "datacenter1" + } + }, + "span": 8, + "targets": [ + { + "expr": "0*scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"} + on (instance) group_left() scylla_node_operation_mode{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} ) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "sum(scylla_errors:nodes_total{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "expr": "(sum(scylla_cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(scylla_cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "dashversion": [ + ">5.1", + ">2022.2" + ], + "expr": "sum(scylla_gossip_live{cluster=~\"$cluster|$^\"})by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "dashversion": [ + ">5.2", + ">2023.1" + ], + "expr": "(min(scylla_streaming_finished_percentage{cluster=~\"$cluster|$^\", dc=~\"$dc\"}*100) by (instance) < 100) or on (instance) 0*sum(scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + } + ], + "title": "Nodes", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "instance", + "svr", + "Value #A", + "Value #B", + "Value #C", + "Value #D", + "Value #E", + "Value #F", + "Value #G" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 5, + "Value #B": 7, + "Value #C": 3, + "Value #D": 1, + "Value #E": 2, + "Value #F": 6, + "Value #G": 8, + "instance": 0, + "svr": 4 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 32 + }, + "id": 29, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + { + "alias": "1 Day Ago", + "dashLength": 4, + "dashes": true + }, + { + "alias": "1 Week Ago", + "dashLength": 2, + "dashes": true + } + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Writes", + "refId": "A", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1d)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1d))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "1 Day Ago", + "refId": "B", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1w)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_write_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m] offset 1w))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "1 Week Ago", + "refId": "C", + "step": 1 + } + ], + "title": "Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 32 + }, + "id": 30, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(wlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}>0) by ([[by]],sg)", + "intervalFactor": 1, + "legendFormat": "95% {{[[by]]}}", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(wlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}>0) by ([[by]],sg)", + "intervalFactor": 1, + "legendFormat": "99% {{[[by]]}}", + "refId": "B", + "step": 1 + } + ], + "title": "Write Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 10, + "y": 32 + }, + "id": 31, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Writes {{[[by]]}}", + "refId": "A", + "step": 10 + } + ], + "title": "Write Timeouts by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 32 + }, + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + { + "alias": "1 Day Ago", + "dashLength": 4, + "dashes": true + }, + { + "alias": "1 Week Ago", + "dashLength": 2, + "dashes": true + } + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]]) or on([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1d)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1d))", + "intervalFactor": 1, + "legendFormat": "1 Day Ago", + "refId": "B", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1w)) or on ([[by]]) $func(rate(scylla_storage_proxy_coordinator_read_latency_summary_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\",scheduling_group_name=~\"$sg\"}[1m] offset 1w))", + "intervalFactor": 1, + "legendFormat": "1 Week Ago", + "refId": "C", + "step": 1 + } + ], + "title": "Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 32 + }, + "id": 33, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(rlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}>0) by([[by]])", + "intervalFactor": 1, + "legendFormat": "95% {{[[by]]}}", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(rlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$sg\"}>0) by([[by]])", + "intervalFactor": 1, + "legendFormat": "99% {{[[by]]}}", + "refId": "B", + "step": 1 + } + ], + "title": "Read Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 10, + "y": 38 + }, + "id": 34, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])+rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])+rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Read {{[[by]]}}", + "refId": "A", + "step": 10 + } + ], + "title": "Read Timeouts by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "dashproductreject": "no-version-check", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 10, + "x": 14, + "y": 38 + }, + "id": 35, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "plain_text", + "dashproduc": "no-version-check", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 10, + "x": 14, + "y": 38 + }, + "id": 36, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 37, + "panels": [], + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Advisor

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 38, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "advisor_table", + "dashversion": ">4.1", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "dashboard" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/${__data.fields.dashboard}-[[dash_version]]?refresh=30s&orgId=1&var-by=instance&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "advisor" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "displayName", + "value": "Category" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "severity" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Open an issue", + "url": "https://github.com/scylladb/scylla/issues/new?body=description%3D${__data.fields[4]}%0ASource%3DAdvisor%0AScylla-versions%3D${all_scyllas_versions}%0Ascylla-monitoring%3D${monitoring_version}%0Acluster%3D${count_dc}%0Aname%3D${cluster}%0A%0A" + } + ] + }, + { + "id": "mappings", + "value": [ + { + "from": "0", + "id": 1, + "text": "\ud83d\udd14", + "to": "10", + "type": 2, + "value": "" + } + ] + }, + { + "id": "displayName", + "value": "Report" + }, + { + "id": "custom.width", + "value": 65 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "summary" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "${__data.fields.description}\n\n click for more information", + "url": "https://monitoring.docs.scylladb.com/branch-4.4/use-monitoring/advisor/${__data.fields.alertname}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alertname" + }, + "properties": [ + { + "id": "displayName", + "value": "." + }, + { + "id": "custom.width", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "description" + }, + "properties": [ + { + "id": "displayName", + "value": "." + }, + { + "id": "custom.width", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 39, + "links": [], + "options": { + "showHeader": true + }, + "targets": [ + { + "active": true, + "annotations": true, + "datasource": "prometheus", + "expr": "ALERTS{advisor!=\"\"}", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "target": "Query" + } + ], + "title": "", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "advisor", + "dashboard", + "description", + "severity", + "alertname", + "summary", + "Time" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 1, + "advisor": 2, + "dashboard": 3, + "severity": 0, + "summary": 4 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "dashproductreject": "no-your-pannels", + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 40, + "panels": [], + "repeat": "", + "title": "Your panels", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 41, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 44, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - 4.4.5
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "5.2" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "DC", + "value": "dc" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": true, + "text": "DC", + "value": "dc" + }, + { + "selected": false, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\"},shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": false, + "name": "mount_point", + "options": [], + "query": "node_filesystem_avail_bytes", + "refresh": 2, + "regex": "/mountpoint=\"([^\"]*)\".*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "selected": true, + "text": [ + "statement" + ], + "value": [ + "statement" + ] + }, + "dashversion": [ + ">4.3" + ], + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "SG", + "multi": true, + "name": "sg", + "options": [], + "query": "label_values(rlatencyp99{cluster=~\"$cluster\", scheduling_group_name!~\"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache\"},scheduling_group_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5-2", + "value": "5-2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "dash_version", + "options": [ + { + "selected": true, + "text": "5-2", + "value": "5-2" + } + ], + "query": "5-2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "all_scyllas_versions", + "options": [], + "query": "label_values(scylla_scylladb_current_version{cluster=~\"$cluster|$^\"}, version)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", + "error": null, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "count_dc", + "options": [], + "query": { + "query": "query_result(count(up{job=\"scylla\"}) by (dc))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "/(?\\{dc=\"[^\"]+\".* \\d+) .*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "5.2", + "value": "5.2" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "5.2", + "value": "5.2" + } + ], + "query": "5.2", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "4.4.5", + "value": "4.4.5" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "4.4.5", + "value": "4.4.5" + } + ], + "query": "4.4.5", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Overview", + "uid": "overview-5-2", + "version": 1 +} diff --git a/assets/monitoring/grafana/v1alpha1/dashboards/saas/overview.json b/assets/monitoring/grafana/v1alpha1/dashboards/saas/overview.json new file mode 100644 index 00000000000..ab717d5430d --- /dev/null +++ b/assets/monitoring/grafana/v1alpha1/dashboards/saas/overview.json @@ -0,0 +1,4664 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_manager_task", + "datasource": "prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress)>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + }, + { + "class": "ops_annotation", + "datasource": "prometheus", + "enable": true, + "expr": "10*min(scylla_node_ops_finished_percentage) by (ops, dc,instance) < 10", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "ops", + "showIn": 0, + "tagKeys": "ops,dc,instance", + "tags": [], + "titleFormat": "Operation", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[$__rate_interval])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "Scylla Cluster Metrics", + "overwrite": true, + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Cluster overview $cluster", + "type": "row" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_transport_requests_served{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])) + (sum(rate(scylla_thrift_served{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[60s])) or on() vector(0))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests/s", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "Average Write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 5, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencyp95{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "95% Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 7, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "Average Read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 9, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 11, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencyp95{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "95% Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "99% read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 13, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name!=\"streaming\"}>0)", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 15, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"} )", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 17, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "$func(rate(scylla_database_total_reads_failed{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "R Failed", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 19, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes_failed{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "W Failed", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 21, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Timeouts", + "type": "stat" + }, + { + "class": "alert_table", + "columns": [], + "datasource": "alertmanager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 13, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 4, + "targets": [ + { + "active": true, + "annotations": true, + "filters": "job!=\"scylla_manager\",advisor=\"\"", + "legendFormat": "{{description}}", + "refId": "A", + "target": "Query" + } + ], + "title": "Active Alerts", + "transform": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "summary", + "instance" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "instance": 1, + "summary": 2 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 15, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(wlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(wlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B", + "step": 1 + } + ], + "title": "Write Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 5 + }, + "id": 16, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A", + "step": 1 + } + ], + "title": "Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(rlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(rlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B", + "step": 1 + } + ], + "title": "Read Latencies", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 18, + "panels": [], + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Advisor

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 19, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 20, + "options": { + "content": "

Balance

\nAn Imbalance between shards or nodes may indicates a potential problem", + "mode": "html" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + }, + { + "class": "advisor_table", + "dashversion": ">4.1", + "datasource": "alertmanager", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "dashboard" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/${__data.fields.dashboard}-[[dash_version]]?refresh=30s&orgId=1&var-by=instance&from=${__from}&to=${__to}" + } + ] + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "advisor" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "displayName", + "value": "Category" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "severity" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Open an issue", + "url": "https://github.com/scylladb/scylla/issues/new?body=description%3D${__data.fields[4]}%0ASource%3DAdvisor%0AScylla-versions%3D${all_scyllas_versions}%0Ascylla-monitoring%3D${monitoring_version}%0Acluster%3D${count_dc}%0Aname%3D${cluster}%0A%0A" + } + ] + }, + { + "id": "mappings", + "value": [ + { + "from": "0", + "id": 1, + "text": "\ud83d\udd14", + "to": "10", + "type": 2, + "value": "" + } + ] + }, + { + "id": "displayName", + "value": "Report" + }, + { + "id": "custom.width", + "value": 65 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "summary" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "${__data.fields.description}\n\n click for more information", + "url": "https://monitoring.docs.scylladb.com/branch-master/use-monitoring/advisor/${__data.fields.alertname}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alertname" + }, + "properties": [ + { + "id": "displayName", + "value": "." + }, + { + "id": "custom.width", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "description" + }, + "properties": [ + { + "id": "displayName", + "value": "." + }, + { + "id": "custom.width", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 21, + "links": [], + "options": { + "showHeader": true + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "active": true, + "annotations": true, + "filters": "advisor!=\"\"", + "legendFormat": "{{description}}", + "refId": "A", + "target": "Query" + } + ], + "title": "", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "advisor", + "dashboard", + "description", + "severity", + "alertname", + "summary", + "Time" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 1, + "advisor": 2, + "dashboard": 3, + "severity": 0, + "summary": 4 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "small_stat_error", + "datasource": "prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 16 + }, + "id": 22, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "max(abs(sum(scylla_transport_current_connections{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by (instance,shard)-scalar(avg(scylla_transport_current_connections{cluster=~\"$cluster|$^\", dc=~\"$dc\"})))) - 8", + "hide": false, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 16 + }, + "id": 23, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "max(abs(rate(scylla_cql_updates{conditional=\"no\", dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_updates{conditional=\"no\", dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_updates{conditional=\"no\", dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "max(abs(rate(scylla_cql_inserts{conditional=\"no\", dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_inserts{conditional=\"no\", dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_inserts{conditional=\"no\", dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "max(abs(rate(scylla_cql_reads{ dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_reads{ dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_reads{ dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "max(abs(rate(scylla_cql_deletes{conditional=\"no\", dc=~\"$dc\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_deletes{conditional=\"no\", dc=~\"$dc\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_deletes{conditional=\"no\", dc=~\"$dc\"}[1m])) by(dc)+100) by(dc))-3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CQL OPs", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 16, + "y": 16 + }, + "id": 24, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "((max(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "((max(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Latency", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 18, + "y": 16 + }, + "id": 25, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "((max(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "((max(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"})-scalar(avg(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name!=\"streaming\"}>0))+100)-3)", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Latency", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "prometheus", + "description": "A shard that reads more from the cache could be an indication for hot partition", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 16 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "((rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]) - rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))- scalar(avg(rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]) - rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))))/scalar(stddev(rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]) - rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m]))+100)-3", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Cache", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "prometheus", + "description": "A single shard that reads more from sstables, could be that a node is slow", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 16 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "max(abs(scylla_database_active_reads{ dc=~\"$dc\"} - scalar(avg(scylla_database_active_reads{ dc=~\"$dc\"})))/scalar(stddev(scylla_database_active_reads{ dc=~\"$dc\"})+0.001))-3", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "SSTable", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 28, + "panels": [], + "repeat": "dc", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Information for $dc

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 29, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "vertical_lcd", + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 1, + "x": 0, + "y": 25 + }, + "id": 30, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} )", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "bargauge" + }, + { + "class": "bytes_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 1, + "y": 25 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 5, + "targets": [ + { + "expr": "Avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\"}) by ([[by]])-avg(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Avg Usage {{[[by]]}}", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\"}) by ([[by]])", + "interval": "", + "legendFormat": "Size {{[[by]]}}", + "refId": "B" + } + ], + "title": "Disk Size by $by", + "type": "timeseries" + }, + { + "class": "graph_panel_int", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 25 + }, + "id": 32, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "span": 2, + "targets": [ + { + "expr": "$func(scylla_compaction_manager_compactions{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Running Compactions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "The Hits and Misses", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 8, + "y": 25 + }, + "id": 33, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Hit {{[[by]]}}", + "refId": "A", + "step": 10 + }, + { + "expr": "$func(rate(scylla_cache_row_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Misses {{[[by]]}}", + "refId": "B", + "step": 10 + } + ], + "title": "Cache Rows Hits/Misses", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Write attempts - include all writes that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 14, + "y": 25 + }, + "id": 34, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + { + "alias": "1 Day Ago", + "dashLength": 4, + "dashes": true + }, + { + "alias": "1 Week Ago", + "dashLength": 2, + "dashes": true + } + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Writes", + "refId": "A", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m] offset 1d))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "1 Day Ago", + "refId": "B", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m] offset 1w))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "1 Week Ago", + "refId": "C", + "step": 1 + } + ], + "title": "Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 25 + }, + "id": 35, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(wlatencyp95{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "95% {{[[by]]}}", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(wlatencyp99{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "99% {{[[by]]}}", + "refId": "B", + "step": 1 + } + ], + "title": "Write Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 31 + }, + "id": 36, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Writes {{[[by]]}}", + "refId": "A", + "step": 10 + } + ], + "title": "Write Timeouts by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 31 + }, + "id": 37, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])+rate(scylla_storage_proxy_coordinator_cas_read_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])+rate(scylla_storage_proxy_coordinator_range_timeouts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Read {{[[by]]}}", + "refId": "A", + "step": 10 + } + ], + "title": "Read Timeouts by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "The Hits and Misses", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 8, + "y": 31 + }, + "id": 38, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])-$func(rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Hit {{[[by]]}}", + "refId": "A", + "step": 10 + }, + { + "expr": "$func(rate(scylla_cache_reads_with_misses{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Misses {{[[by]]}}", + "refId": "B", + "step": 10 + } + ], + "title": "Cache Reads Hits/Misses", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Read attempts - include all reads that reached the coordinator node, even if they will eventually fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 14, + "y": 31 + }, + "id": 39, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + { + "alias": "1 Day Ago", + "dashLength": 4, + "dashes": true + }, + { + "alias": "1 Week Ago", + "dashLength": 2, + "dashes": true + } + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}[1m] offset 1d))", + "intervalFactor": 1, + "legendFormat": "1 Day Ago", + "refId": "B", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}[1m] offset 1w))", + "intervalFactor": 1, + "legendFormat": "1 Week Ago", + "refId": "C", + "step": 1 + } + ], + "title": "Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 31 + }, + "id": 40, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "avg(rlatencyp95{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by([[by]])", + "intervalFactor": 1, + "legendFormat": "95% {{[[by]]}}", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(rlatencyp99{by=\"[[by]]\", cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name!=\"streaming\"}>0) by([[by]])", + "intervalFactor": 1, + "legendFormat": "99% {{[[by]]}}", + "refId": "B", + "step": 1 + } + ], + "title": "Read Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL INSERT commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL SELECT commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Reads", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL DELETE commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Number of CQL UPDATE commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{cluster=~\"$cluster|$^\", dc=~\"$dc\", who=\"internal\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "amount of CQL connections currently established", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 43 + }, + "id": 45, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(scylla_transport_current_connections{cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Client CQL connections by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL batches command, each batched command is counted once", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 43 + }, + "id": 46, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_batches{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL command batched. Each batch would add the number of commands inside the batch", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 43 + }, + "id": 47, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_statements_in_batches{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Command In Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "prometheus", + "description": "Counts the number of SELECT statements with BYPASS CACHE option", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 43 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_select_bypass_caches{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "BYPASS CACHE", + "type": "timeseries" + }, + { + "class": "graph_panel", + "dashversion": [ + ">4.4", + ">2021.1" + ], + "datasource": "prometheus", + "description": "CQL errors by type, only active errors are shown", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 49 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_transport_cql_errors_total{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]],type) >0", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Errors [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of CQL row reads", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 49 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_rows_read{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Row Reads [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "prometheus", + "description": "Number of reads using secondary indexes", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 49 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_secondary_index_reads{cluster=~\"$cluster|$^\", dc=~\"$dc\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Secondary indexes Reads [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 52, + "panels": [], + "repeat": "", + "title": "Your panels", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 53, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "class": "fieldConfig_defaults_custom", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 55, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 56, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "DC", + "value": "dc" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": true, + "text": "DC", + "value": "dc" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": false, + "name": "mount_point", + "options": [ + { + "selected": true, + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + } + ], + "query": "/var/lib/scylla", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "custom", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "all_scyllas_versions", + "options": [], + "query": "label_values(scylla_scylladb_current_version{cluster=~\"$cluster|$^\"}, version)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", + "error": null, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "count_dc", + "options": [], + "query": { + "query": "query_result(count(up{job=~\"$cluster|$^\"}) by (dc))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "/(?\\{dc=\"[^\"]+\".* \\d+) .*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "CQL Overview", + "uid": "cql-overview", + "version": 1 +} diff --git a/assets/monitoring/grafana/v1alpha1/deployment.yaml b/assets/monitoring/grafana/v1alpha1/deployment.yaml index 1f83553c092..6448636f52f 100644 --- a/assets/monitoring/grafana/v1alpha1/deployment.yaml +++ b/assets/monitoring/grafana/v1alpha1/deployment.yaml @@ -23,9 +23,35 @@ spec: {{- .affinity | toYAML | nindent 8 }} tolerations: {{- .tolerations | toYAML | nindent 8 }} + initContainers: + - name: gzip + image: registry.access.redhat.com/ubi9/ubi:9.2-755.1696515532@sha256:762ee03f2cce3af7f8c8f3eda26f2aeb50678fb7cc3b78f856546970faa04c67 + command: + - /usr/bin/bash + - -euExo + - pipefail + - -O + - inherit_errexit + - -c + args: + - | + find /var/run/configmaps -type d -mindepth 1 -maxdepth 1 | while read -r d; do + mkdir /var/run/decompressed-configmaps/"${d##*/}" + find "${d}" -maxdepth 1 -name '*.gz.base64' -exec cp -L -t /var/run/decompressed-configmaps/"${d##*/}" {} + + done + find /var/run/decompressed-configmaps -name '*.gz.base64' | while read -r f; do + base64 -d "${f}" > "${f%.base64}" + rm "${f}" + done + find /var/run/decompressed-configmaps -name '*.gz' -exec gzip -d {} + + volumeMounts: + - name: decompressed-configmaps + mountPath: /var/run/decompressed-configmaps + - name: grafana-scylladb-dashboards + mountPath: /var/run/configmaps/grafana-scylladb-dashboards containers: - name: grafana - image: docker.io/grafana/grafana:9.3.4 + image: docker.io/grafana/grafana:9.5.12@sha256:7d2f2a8b7aebe30bf3f9ae0f190e508e571b43f65753ba3b1b1adf0800bc9256 command: - grafana-server - --packaging=docker @@ -67,8 +93,9 @@ spec: volumeMounts: - name: grafana-configs mountPath: /var/run/configmaps/grafana-configs - - name: grafana-scylladb-dashboards + - name: decompressed-configmaps mountPath: /var/run/dashboards/scylladb + subPath: grafana-scylladb-dashboards - name: grafana-provisioning mountPath: /var/run/configmaps/grafana-provisioning/access-control/access-control.yaml subPath: access-control.yaml @@ -98,6 +125,9 @@ spec: - name: grafana-storage mountPath: /var/lib/grafana volumes: + - name: decompressed-configmaps + emptyDir: + sizeLimit: 50Mi - name: grafana-configs configMap: name: "{{ .scyllaDBMonitoringName }}-grafana-configs" diff --git a/assets/monitoring/grafana/v1alpha1/registry.go b/assets/monitoring/grafana/v1alpha1/registry.go index 84bfb1fddb5..7e2f0dd8030 100644 --- a/assets/monitoring/grafana/v1alpha1/registry.go +++ b/assets/monitoring/grafana/v1alpha1/registry.go @@ -1,9 +1,17 @@ package v1alpha1 import ( + "bytes" + "compress/gzip" + "embed" _ "embed" + "encoding/base64" + "fmt" + "io/fs" + "path/filepath" "github.com/scylladb/scylla-operator/pkg/assets" + "github.com/scylladb/scylla-operator/pkg/helpers" "github.com/scylladb/scylla-operator/pkg/scheme" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -36,13 +44,17 @@ var ( grafanaProvisioningConfigMapTemplateString string GrafanaProvisioningConfigMapTemplate = ParseObjectTemplateOrDie[*corev1.ConfigMap]("grafana-provisioning-cm", grafanaProvisioningConfigMapTemplateString) - //go:embed "dashboards-platform.cm.yaml" - grafanaDashboardsPlatformConfigMapTemplateString string - GrafanaDashboardsPlatformConfigMapTemplate = ParseObjectTemplateOrDie[*corev1.ConfigMap]("grafana-dashboards-platform-cm", grafanaDashboardsPlatformConfigMapTemplateString) + //go:embed "dashboards.cm.yaml" + grafanaDashboardsConfigMapTemplateString string + GrafanaDashboardsConfigMapTemplate = ParseObjectTemplateOrDie[*corev1.ConfigMap]("grafana-dashboards-cm", grafanaDashboardsConfigMapTemplateString) - //go:embed "dashboards-saas.cm.yaml" - grafanaDashboardsSAASConfigMapTemplateString string - GrafanaDashboardsSAASConfigMapTemplate = ParseObjectTemplateOrDie[*corev1.ConfigMap]("grafana-dashboards-saas-cm", grafanaDashboardsSAASConfigMapTemplateString) + //go:embed "dashboards/platform/*.json" + grafanaDashboardsPlatformFS embed.FS + GrafanaDashboardsPlatform = helpers.Must(gzipMapData(helpers.Must(parseDashboardsFromFS(grafanaDashboardsPlatformFS, "dashboards/platform")))) + + //go:embed "dashboards/saas/*.json" + grafanaDashboardsSAASFS embed.FS + GrafanaDashboardsSAAS = helpers.Must(gzipMapData(helpers.Must(parseDashboardsFromFS(grafanaDashboardsSAASFS, "dashboards/saas")))) //go:embed "service.yaml" grafanaServiceTemplateString string @@ -52,3 +64,65 @@ var ( grafanaIngressTemplateString string GrafanaIngressTemplate = ParseObjectTemplateOrDie[*networkingv1.Ingress]("grafana-ingress", grafanaIngressTemplateString) ) + +func parseDashboardsFromFS(files embed.FS, root string) (map[string]string, error) { + res := map[string]string{} + + err := fs.WalkDir(files, root, func(p string, de fs.DirEntry, err error) error { + if err != nil { + return err + } + + if p == root || de.IsDir() { + return nil + } + + data, err := fs.ReadFile(files, p) + if err != nil { + return fmt.Errorf("can't read file %q: %w", p, err) + } + relPath, err := filepath.Rel(root, p) + if err != nil { + return fmt.Errorf("can't compute relative path to file %q: %w", p, err) + } + res[relPath] = string(data) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("can't walk fs at root %q: %w", root, err) + } + + return res, nil +} + +func gzipMapData(uncompressedMap map[string]string) (map[string]string, error) { + res := make(map[string]string, len(uncompressedMap)) + for k, v := range uncompressedMap { + var buf bytes.Buffer + b64Writer := base64.NewEncoder(base64.StdEncoding, &buf) + + zw, err := gzip.NewWriterLevel(b64Writer, gzip.BestCompression) + if err != nil { + return nil, fmt.Errorf("can't create gzip writer: %w", err) + } + + _, err = zw.Write([]byte(v)) + if err != nil { + return nil, fmt.Errorf("can't write value of key %q into gzip writer: %w", k, err) + } + err = zw.Close() + if err != nil { + return nil, fmt.Errorf("can't close gzip writer for key %q: %w", k, err) + } + + err = b64Writer.Close() + if err != nil { + return nil, fmt.Errorf("can't close base64 writer for key %q: %w", k, err) + } + + res[fmt.Sprintf("%s.gz.base64", k)] = buf.String() + } + + return res, nil +} diff --git a/assets/monitoring/prometheus/v1/alerts.prometheusrule.yaml b/assets/monitoring/prometheus/v1/alerts.prometheusrule.yaml index 66a7151997d..78afc8c2779 100644 --- a/assets/monitoring/prometheus/v1/alerts.prometheusrule.yaml +++ b/assets/monitoring/prometheus/v1/alerts.prometheusrule.yaml @@ -5,174 +5,151 @@ metadata: labels: scylla-operator.scylladb.com/scylladbmonitoring-name: "{{ .scyllaDBMonitoringName }}" spec: +{{` groups: - name: scylla.rules - rules: {{` + rules: - alert: cqlNonPrepared expr: cql:non_prepared > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "cqlOptimization" dashboard: "cql" - annotations: description: 'Some queries are non-prepared' summary: non prepared statments - alert: cql:non_paged_no_system expr: cql:non_paged > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "cqlOptimization" dashboard: "cql" status: "1" - annotations: description: 'Some SELECT queries are non-paged' summary: non paged statments - alert: cqlNoTokenAware expr: cql:non_token_aware > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "cqlOptimization" dashboard: "cql" - annotations: description: 'Some queries are not token-aware' summary: non token aware statments - - alert: cqlReverseOrder - expr: cql:reverse_queries > 0 - for: 10s - labels: - severity: "1" - advisor: "cqlOptimization" - dashboard: "cql" - annotations: - description: 'Some queries use reverse order' - summary: reverse order queries - alert: cqlAllowFiltering expr: cql:allow_filtering > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "cqlOptimization" dashboard: "cql" - annotations: description: 'Some queries use ALLOW FILTERING' summary: Allow filtering queries - alert: cqlCLAny expr: cql:any_queries > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "cqlOptimization" dashboard: "cql" - annotations: description: 'Some queries use Consistency Level: ANY' summary: non prepared statments - alert: cqlCLAll expr: cql:all_queries > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "cqlOptimization" dashboard: "cql" - annotations: description: 'Some queries use Consistency Level: ALL' summary: non prepared statments - alert: nonBalancedcqlTraffic expr: abs(rate(scylla_cql_updates{conditional="no"}[1m]) - scalar(avg(rate(scylla_cql_updates{conditional="no"}[1m]))))/scalar(stddev(rate(scylla_cql_updates{conditional="no"}[1m]))+100) > 2 - for: 10s + for: 3m labels: - severity: "1" + severity: "info" status: "1" advisor: "balanced" dashboard: "cql" - annotations: description: 'CQL queries are not balanced among shards {{ $labels.instance }} shard {{ $labels.shard }}' summary: CQL queries are not balanced - alert: nodeLocalErrors expr: sum(errors:local_failed) by (cluster, instance) > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "operationError" dashboard: "scylla-detailed" - annotations: description: 'Some operation failed at the replica side' summary: Replica side Level error - alert: nodeIOErrors expr: sum(rate(scylla_reactor_aio_errors[60s])) by (cluster, instance) > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "operationError" dashboard: "OS-master" - annotations: description: 'IO Errors can indicate a node with a faulty disk {{ $labels.instance }}' summary: IO Disk Error - alert: nodeCLErrors expr: sum(errors:operation_unavailable) by (cluster) > 0 for: 10s labels: - severity: "1" + severity: "info" advisor: "operationError" dashboard: "scylla-detailed" - annotations: description: 'Some operation failed due to consistency level' summary: Consistency Level error - alert: preparedCacheEviction expr: sum(rate(scylla_cql_prepared_cache_evictions[2m])) by (cluster) + sum(rate(scylla_cql_authorized_prepared_statements_cache_evictions[2m])) by (cluster) > 100 for: 5m labels: - severity: "1" + severity: "info" advisor: "preparedEviction" dashboard: "scylla-detailed" - annotations: description: 'The prepared-statement cache is being continuously evicted, which could indicate a problem in your prepared-statement usage logic.' summary: Prepared cache eviction - alert: heavyCompaction expr: max(scylla_scheduler_shares{group="compaction"}) by (cluster) >= 1000 for: 20m labels: - severity: "1" + severity: "info" advisor: "heavyCompaction" dashboard: "scylla-detailed" - annotations: description: 'Compaction load increases to a level it can interfere with the system behaviour. If this persists set the compaction share to a static level.' summary: Heavy compaction load - alert: shedRequests expr: max(sum(rate(scylla_transport_requests_shed[60s])) by (instance,cluster)/sum(rate(scylla_transport_requests_served{}[60s])) by (instance, cluster)) by(cluster) > 0.01 for: 5m labels: - severity: "1" + severity: "info" advisor: "systemOverload" dashboard: "scylla-detailed" - annotations: description: 'More than 1% of the requests got shed, this is an indication of an overload, consider system resize.' summary: System is overloaded - alert: cappedTombstone expr: changes(scylla_sstables_capped_tombstone_deletion_time[1h]) > 0 for: 1m labels: - severity: "1" + severity: "info" advisor: "cappedTombstone" dashboard: "scylla-detailed" - annotations: description: 'Tombstone delete time was set too far in the future and was capped' summary: Tobmstone delete time is capped - alert: InstanceDown expr: up{job="scylla"} == 0 for: 30s labels: - severity: "2" + severity: "warn" annotations: description: '{{ $labels.instance }} has been down for more than 30 seconds.' summary: Instance {{ $labels.instance }} down - alert: InstanceDown - expr: absent(scylla_transport_requests_served{job="scylla", shard="0"}) + expr: sum(up{job="scylla"}>0)by(instance) unless sum(scylla_transport_requests_served{shard="0"}) by(instance) for: 1m labels: - severity: "2" + severity: "warn" annotations: description: '{{ $labels.instance }} instance is shutting down.' summary: Instance {{ $labels.instance }} down @@ -180,7 +157,7 @@ spec: expr: scylla_node_operation_mode > 3 for: 30s labels: - severity: "2" + severity: "warn" annotations: description: '{{ $labels.instance }} instance is shutting down.' summary: Instance {{ $labels.instance }} down @@ -189,7 +166,7 @@ spec: * 100 < 35 for: 30s labels: - severity: "2" + severity: "warn" annotations: description: '{{ $labels.instance }} has less than 35% free disk space.' summary: Instance {{ $labels.instance }} low disk space @@ -198,7 +175,7 @@ spec: * 100 < 25 for: 30s labels: - severity: "3" + severity: "error" annotations: description: '{{ $labels.instance }} has less than 25% free disk space.' summary: Instance {{ $labels.instance }} low disk space @@ -207,7 +184,7 @@ spec: * 100 < 15 for: 30s labels: - severity: "4" + severity: "critical" annotations: description: '{{ $labels.instance }} has less than 15% free disk space.' summary: Instance {{ $labels.instance }} low disk space @@ -216,7 +193,7 @@ spec: * 100 < 20 for: 30s labels: - severity: "3" + severity: "error" annotations: description: '{{ $labels.instance }} has less than 20% free disk space on the root partition.' summary: Instance {{ $labels.instance }} low disk space @@ -224,7 +201,7 @@ spec: expr: scylla_manager_healthcheck_cql_status == -1 for: 30s labels: - severity: "2" + severity: "warn" annotations: description: '{{ $labels.host }} has denied CQL connection for more than 30 seconds.' summary: Instance {{ $labels.host }} no CQL connection @@ -232,7 +209,7 @@ spec: expr: wlatencyp95{by="instance"} > 100000 for: 5m labels: - severity: "1" + severity: "info" annotations: description: '{{ $labels.instance }} has 95% high latency for more than 5 minutes.' summary: Instance {{ $labels.instance }} High Write Latency @@ -240,7 +217,7 @@ spec: expr: wlatencya{by="instance"} >10000 for: 5m labels: - severity: "1" + severity: "info" annotations: description: '{{ $labels.instance }} has average high latency for more than 5 minutes.' summary: Instance {{ $labels.instance }} High Write Latency @@ -248,7 +225,7 @@ spec: expr: rlatencyp95{by="instance"} > 100000 for: 5m labels: - severity: "1" + severity: "info" annotations: description: '{{ $labels.instance }} has 95% high latency for more than 5 minutes.' summary: Instance {{ $labels.instance }} High Read Latency @@ -256,7 +233,7 @@ spec: expr: rlatencya{by="instance"} >10000 for: 5m labels: - severity: "1" + severity: "info" annotations: description: '{{ $labels.instance }} has average high latency for more than 5 minutes.' summary: Instance {{ $labels.instance }} High Read Latency @@ -264,7 +241,7 @@ spec: expr: (sum(scylla_manager_scheduler_run_total{type=~"backup", status="ERROR"}) or vector(0)) - (sum(scylla_manager_scheduler_run_total{type=~"backup", status="ERROR"} offset 3m) or vector(0)) > 0 for: 10s labels: - severity: "1" + severity: "info" annotations: description: 'Backup failed' summary: Backup task failed @@ -272,7 +249,7 @@ spec: expr: (sum(scylla_manager_scheduler_run_total{type=~"repair", status="ERROR"}) or vector(0)) - (sum(scylla_manager_scheduler_run_total{type=~"repair", status="ERROR"} offset 3m) or vector(0)) > 0 for: 10s labels: - severity: "1" + severity: "info" annotations: description: 'Repair failed' summary: Repair task failed @@ -280,7 +257,7 @@ spec: expr: resets(scylla_gossip_heart_beat[1h])>0 for: 10s labels: - severity: "1" + severity: "info" annotations: description: 'Node restarted' summary: Instance {{ $labels.instance }} restarted @@ -288,8 +265,29 @@ spec: expr: changes(node_vmstat_oom_kill[1h])>0 for: 10s labels: - severity: "2" + severity: "warn" annotations: description: 'OOM Kill on {{ $labels.instance }}' summary: A process was terminated on Instance {{ $labels.instance }} + - alert: tooManyFiles + expr: (node_filesystem_files - node_filesystem_files_free) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>20000 + for: 10s + labels: + severity: "info" + description: 'Over 20k open files per shard {{ $labels.instance }}' + summary: There are over 20K open files per shard on Insace {{ $labels.instance }} + - alert: tooManyFiles + expr: (node_filesystem_files - node_filesystem_files_free) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>30000 + for: 10s + labels: + severity: "warn" + description: 'Over 30k open files per shard {{ $labels.instance }}' + summary: There are over 30K open files per shard on Insace {{ $labels.instance }} + - alert: tooManyFiles + expr: (node_filesystem_files - node_filesystem_files_free) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>40000 + for: 10s + labels: + severity: "error" + description: 'Over 40k open files per shard {{ $labels.instance }}' + summary: There are over 40K open files per shard on Insace {{ $labels.instance }} `}} diff --git a/assets/monitoring/prometheus/v1/prometheus.yaml b/assets/monitoring/prometheus/v1/prometheus.yaml index 7765d380f90..d7d69bd985f 100644 --- a/assets/monitoring/prometheus/v1/prometheus.yaml +++ b/assets/monitoring/prometheus/v1/prometheus.yaml @@ -3,6 +3,7 @@ kind: Prometheus metadata: name: "{{ .scyllaDBMonitoringName }}" spec: + version: "v2.44.0" serviceAccountName: "{{ .scyllaDBMonitoringName }}-prometheus" securityContext: runAsNonRoot: true diff --git a/assets/monitoring/prometheus/v1/recording.prometheusrule.yaml b/assets/monitoring/prometheus/v1/recording.prometheusrule.yaml index 3386f9d730a..d4bfa00b0d8 100644 --- a/assets/monitoring/prometheus/v1/recording.prometheusrule.yaml +++ b/assets/monitoring/prometheus/v1/recording.prometheusrule.yaml @@ -1,10 +1,11 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: - name: "{{ .scyllaDBMonitoringName }}-recoding" + name: "{{ .scyllaDBMonitoringName }}-recording" labels: scylla-operator.scylladb.com/scylladbmonitoring-name: "{{ .scyllaDBMonitoringName }}" spec: +{{` groups: - name: scylla.rules rules: @@ -43,19 +44,20 @@ spec: - record: errors:nodes_total expr: errors:local_failed + errors:operation_unavailable - record: manager:repair_done_ts - expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="DONE",type="repair"}[60s])) by (cluster) > 0) or on(cluster) manager:repair_done_ts + expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="DONE",type="repair"}[2m])) by (cluster) > 0) or on(cluster) manager:repair_done_ts - record: manager:backup_done_ts - expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="DONE",type="backup"}[60s])) by (cluster) > 0) or on(cluster) manager:backup_done_ts + expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="DONE",type="backup"}[2m])) by (cluster) > 0) or on(cluster) manager:backup_done_ts - record: manager:repair_fail_ts - expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="ERROR",type="repair"}[60s])) by (cluster) > 0) or on(cluster) manager:repair_fail_ts + expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="ERROR",type="repair"}[2m])) by (cluster) > 0) or on(cluster) manager:repair_fail_ts - record: manager:backup_fail_ts - expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="ERROR",type="backup"}[60s])) by (cluster) > 0) or on(cluster) manager:backup_fail_ts + expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="ERROR",type="backup"}[2m])) by (cluster) > 0) or on(cluster) manager:backup_fail_ts - record: manager:repair_progress expr: (max(scylla_manager_scheduler_run_indicator{type="repair"}) by (cluster) >bool 0)*((max(scylla_manager_repair_token_ranges_total) by(cluster)<= 0)*0 or on(cluster) (sum(scylla_manager_repair_token_ranges_success>=0) by (cluster) + sum(scylla_manager_repair_token_ranges_error>=0) by (cluster))/sum(scylla_manager_repair_token_ranges_total>=0) by (cluster)) - record: scylla_manager_repair_progress expr: sum(manager:repair_progress) by (cluster) labels: level: "1" + dd: "1" by: "cluster" - record: manager:backup_progress expr: (max(scylla_manager_scheduler_run_indicator{type="backup"}) by (cluster) >bool 0)*((max(scylla_manager_backup_files_size_bytes) by(cluster)<= 0)*0 or on(cluster) (sum(scylla_manager_backup_files_uploaded_bytes) by (cluster) + sum(scylla_manager_backup_files_skipped_bytes) by (cluster) + sum(scylla_manager_backup_files_failed_bytes)by(cluster))/sum(scylla_manager_backup_files_size_bytes>=0) by (cluster)) @@ -63,167 +65,441 @@ spec: expr: sum(manager:backup_progress) by (cluster) labels: level: "1" + dd: "1" by: "cluster" - record: wlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: wlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) labels: by: "instance" level: "1" + dd: "1" - record: wlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, dc, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) labels: by: "dc" level: "1" + dd: "1" - record: wlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) labels: by: "cluster" level: "1" + dd: "1" - record: rlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: rlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) labels: by: "instance" level: "1" + dd: "1" - record: rlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, dc, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) labels: by: "dc" level: "1" + dd: "1" - record: rlatencyp99 - expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, scheduling_group_name, le)) + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) labels: by: "cluster" level: "1" + dd: "1" - record: wlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: wlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) labels: by: "instance" level: "1" + dd: "1" - record: wlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, dc, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) labels: by: "dc" level: "1" + dd: "1" - record: wlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{}[60s])) by (cluster, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) labels: by: "cluster" level: "1" + dd: "1" - record: rlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: rlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) labels: by: "instance" level: "1" + dd: "1" - record: rlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, dc, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) labels: by: "dc" level: "1" + dd: "1" - record: rlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{}[60s])) by (cluster, scheduling_group_name, le)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) labels: by: "cluster" level: "1" + dd: "1" - record: wlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{}[60s])) by (cluster, dc, instance,scheduling_group_name, shard)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{}[60s])) by (cluster, dc, instance, scheduling_group_name, shard) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: wlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{}[60s])) by (cluster, dc, instance,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{}[60s])) by (cluster, dc, scheduling_group_name, instance) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) labels: by: "instance" level: "1" + dd: "1" - record: wlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{}[60s])) by (cluster, dc,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{}[60s])) by (cluster, scheduling_group_name, dc) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) labels: by: "dc" level: "1" + dd: "1" - record: wlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{}[60s])) by (cluster,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{}[60s])) by (cluster, scheduling_group_name) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) labels: by: "cluster" level: "1" + dd: "1" - record: rlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{}[60s])) by (cluster, dc, instance, shard,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{}[60s])) by (cluster, dc, instance, shard, scheduling_group_name) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: rlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{}[60s])) by (cluster, dc, instance,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{}[60s])) by (cluster, dc, instance, scheduling_group_name) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) labels: by: "instance" level: "1" + dd: "1" - record: rlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{}[60s])) by (cluster, dc,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{}[60s])) by (cluster, dc, scheduling_group_name) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) labels: by: "dc" level: "1" + dd: "1" - record: rlatencya - expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{}[60s])) by (cluster,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{}[60s])) by (cluster, scheduling_group_name) + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) labels: by: "cluster" level: "1" + dd: "1" + - record: wlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{shard=~".+", scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + labels: + by: "instance,shard" + - record: wlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + labels: + by: "instance" + + - record: wlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) + labels: + by: "dc" + - record: wlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) + labels: + by: "cluster" + - record: rlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{shard=~".+", scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + labels: + by: "instance,shard" + - record: rlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + labels: + by: "instance" + - record: rlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) + labels: + by: "dc" + - record: rlatencyp99 + expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) + labels: + by: "cluster" + - record: wlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{shard=~".+", scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + labels: + by: "instance,shard" + - record: wlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + labels: + by: "instance" + - record: wlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) + labels: + by: "dc" + - record: wlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) + labels: + by: "cluster" + - record: rlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{shard=~".+", scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + labels: + by: "instance,shard" + - record: rlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + labels: + by: "instance" + - record: rlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) + labels: + by: "dc" + - record: rlatencyp95 + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) + labels: + by: "cluster" + - record: wlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{shard=~".+", scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + labels: + by: "instance,shard" + - record: wlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + labels: + by: "instance" + - record: wlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) + labels: + by: "dc" + - record: wlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) + labels: + by: "cluster" + - record: rlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{shard=~".+", scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le)) + labels: + by: "instance,shard" + - record: rlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le)) + labels: + by: "instance" + - record: rlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le)) + labels: + by: "dc" + - record: rlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le)) + labels: + by: "cluster" - record: casrlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: casrlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{}[60s])) by (cluster, dc, instance, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, le, scheduling_group_name)) labels: by: "instance" level: "1" + dd: "1" - record: casrlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{}[60s])) by (cluster, dc, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, le, scheduling_group_name)) labels: by: "dc" level: "1" + dd: "1" - record: casrlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{}[60s])) by (cluster, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, le, scheduling_group_name)) labels: by: "cluster" level: "1" + dd: "1" - record: caswlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name)) labels: by: "instance,shard" - level: "1" + level: "2" + dd: "2" - record: caswlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{}[60s])) by (cluster, dc, instance, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, le, scheduling_group_name)) labels: by: "instance" level: "1" + dd: "1" - record: caswlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{}[60s])) by (cluster, dc, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, le, scheduling_group_name)) labels: by: "dc" level: "1" + dd: "1" - record: caswlatencyp95 - expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{}[60s])) by (cluster, le, scheduling_group_name)) + expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, le, scheduling_group_name)) + labels: + by: "cluster" + level: "1" + dd: "1" + - record: casrlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name)) + labels: + by: "instance,shard" + level: "2" + dd: "2" + - record: casrlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, le, scheduling_group_name)) + labels: + by: "instance" + level: "1" + dd: "1" + - record: casrlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, le, scheduling_group_name)) + labels: + by: "dc" + level: "1" + dd: "1" + - record: casrlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, le, scheduling_group_name)) + labels: + by: "cluster" + level: "1" + dd: "1" + - record: caswlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{shard=~".+", scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name)) + labels: + by: "instance,shard" + level: "2" + dd: "2" + - record: caswlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, le, scheduling_group_name)) + labels: + by: "instance" + level: "1" + dd: "1" + - record: caswlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, le, scheduling_group_name)) + labels: + by: "dc" + level: "1" + dd: "1" + - record: caswlatencya + expr: histogram_quantile(0.5, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, le, scheduling_group_name)) labels: by: "cluster" level: "1" + dd: "1" - record: all_scheduling_group - expr: sum(scylla_storage_proxy_coordinator_write_latency_count>0) by (cluster, scheduling_group_name) - + expr: sum by (cluster, scheduling_group_name) (scylla_storage_proxy_coordinator_write_latency_count + scylla_storage_proxy_coordinator_read_latency_count) > 0 + - record: wlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf)) + labels: + by: "instance,shard" + - record: wlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf)) + labels: + by: "instance" + - record: wlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf)) + labels: + by: "dc" + - record: wlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, le,ks,cf)) + labels: + by: "cluster" + - record: rlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf)) + labels: + by: "instance,shard" + - record: rlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf)) + labels: + by: "instance" + - record: rlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf)) + labels: + by: "dc" + - record: rlatencyp99ks + expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, le,ks,cf)) + labels: + by: "cluster" + - record: wlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf)) + labels: + by: "instance,shard" + - record: wlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf)) + labels: + by: "instance" + - record: wlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf)) + labels: + by: "dc" + - record: wlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, le,ks,cf)) + labels: + by: "cluster" + - record: rlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf)) + labels: + by: "instance,shard" + - record: rlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf)) + labels: + by: "instance" + - record: rlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf)) + labels: + by: "dc" + - record: rlatencyp95ks + expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, le,ks,cf)) + labels: + by: "cluster" + - record: wlatencyaks + expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster, dc, instance, shard,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster, dc, instance, shard,ks,cf) + labels: + by: "instance,shard" + - record: wlatencyaks + expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster, dc, instance,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster, dc, instance,ks,cf) + labels: + by: "instance" + - record: wlatencyaks + expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster, dc,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster, dc,ks,cf) + labels: + by: "dc" + - record: wlatencyaks + expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster,ks,cf) + labels: + by: "cluster" + - record: rlatencyaks + expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster, dc, instance, shard,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster, dc, instance, shard,ks,cf) + labels: + by: "instance,shard" + - record: rlatencyaks + expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster, dc, instance,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster, dc, instance,ks,cf) + labels: + by: "instance" + - record: rlatencyaks + expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster, dc,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster, dc,ks,cf) + labels: + by: "dc" + - record: rlatencyaks + expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster,ks,cf) + labels: + by: "cluster" +`}} diff --git a/pkg/controller/scylladbmonitoring/sync_grafana.go b/pkg/controller/scylladbmonitoring/sync_grafana.go index 77f8707733f..3081b0a67ab 100644 --- a/pkg/controller/scylladbmonitoring/sync_grafana.go +++ b/pkg/controller/scylladbmonitoring/sync_grafana.go @@ -147,18 +147,20 @@ func makeGrafanaDashboards(sm *scyllav1alpha1.ScyllaDBMonitoring) (*corev1.Confi t = *sm.Spec.Type } + var dashboards map[string]string switch t { case scyllav1alpha1.ScyllaDBMonitoringTypePlatform: - return grafanav1alpha1assets.GrafanaDashboardsPlatformConfigMapTemplate.RenderObject(map[string]any{ - "scyllaDBMonitoringName": sm.Name, - }) + dashboards = grafanav1alpha1assets.GrafanaDashboardsPlatform case scyllav1alpha1.ScyllaDBMonitoringTypeSAAS: - return grafanav1alpha1assets.GrafanaDashboardsSAASConfigMapTemplate.RenderObject(map[string]any{ - "scyllaDBMonitoringName": sm.Name, - }) + dashboards = grafanav1alpha1assets.GrafanaDashboardsSAAS default: return nil, "", fmt.Errorf("unkown monitoring type: %q", t) } + + return grafanav1alpha1assets.GrafanaDashboardsConfigMapTemplate.RenderObject(map[string]any{ + "scyllaDBMonitoringName": sm.Name, + "dashboards": dashboards, + }) } func makeGrafanaProvisionings(sm *scyllav1alpha1.ScyllaDBMonitoring) (*corev1.ConfigMap, string, error) { @@ -280,7 +282,7 @@ func (smc *Controller) syncGrafana( var requiredDeployment *appsv1.Deployment // Trigger restart for inputs that are not live reloaded. - grafanaRestartHash, hashErr := hash.HashObjects(requiredConfigsCM, requiredProvisioningsCM) + grafanaRestartHash, hashErr := hash.HashObjects(requiredConfigsCM, requiredProvisioningsCM, requiredDahsboardsCM) if hashErr != nil { renderErrors = append(renderErrors, hashErr) } else { diff --git a/pkg/controller/scylladbmonitoring/sync_grafana_test.go b/pkg/controller/scylladbmonitoring/sync_grafana_test.go index d52160bca15..19de2077ac3 100644 --- a/pkg/controller/scylladbmonitoring/sync_grafana_test.go +++ b/pkg/controller/scylladbmonitoring/sync_grafana_test.go @@ -1,13 +1,20 @@ package scylladbmonitoring import ( + "crypto/sha512" + "encoding/base64" + "fmt" "reflect" "strings" "testing" "github.com/google/go-cmp/cmp" scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/pointer" + "github.com/scylladb/scylla-operator/pkg/scheme" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) func Test_makeGrafanaIngress(t *testing.T) { @@ -125,3 +132,98 @@ spec: }) } } + +func Test_makeGrafanaDashboards(t *testing.T) { + tt := []struct { + name string + sm *scyllav1alpha1.ScyllaDBMonitoring + expectedString string + expectedErr error + }{ + { + name: "renders data for default SaaS type", + sm: &scyllav1alpha1.ScyllaDBMonitoring{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sm-name", + }, + Spec: scyllav1alpha1.ScyllaDBMonitoringSpec{ + Type: nil, + }, + }, + expectedString: strings.TrimLeft(` +apiVersion: v1 +data: + overview.json.gz.base64: '<12812 bytes, hash: "wX2HNiQFcoq3ZHp/fxfzwHDcBs4L14XT5Ainng==">' +kind: ConfigMap +metadata: + creationTimestamp: null + name: sm-name-grafana-scylladb-dashboards +`, "\n"), + expectedErr: nil, + }, + { + name: "renders data for platform type", + sm: &scyllav1alpha1.ScyllaDBMonitoring{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sm-name", + }, + Spec: scyllav1alpha1.ScyllaDBMonitoringSpec{ + Type: pointer.Ptr(scyllav1alpha1.ScyllaDBMonitoringTypePlatform), + }, + }, + expectedString: strings.TrimLeft(` +apiVersion: v1 +data: + advanced.json.gz.base64: '<10636 bytes, hash: "2FtFj4vQKwaZldf3EpV/lquuiKnceSFkKCvs/A==">' + alternator.json.gz.base64: '<29564 bytes, hash: "kG1P7l18/b7pqlBGM1J085vpzK5rn3RpaJSsYA==">' + cql.json.gz.base64: '<26344 bytes, hash: "JTpUOpCmNdZJ/2OcOPwSdqAkIpllDu/7VsRziw==">' + detailed.json.gz.base64: '<32768 bytes, hash: "vD71wxAYWcAKgKeelKh70O0X2zevBjM7TILEVA==">' + ks.json.gz.base64: '<19488 bytes, hash: "uiuutDdzdfGODAwu7/g84QpwwDsxbtfzt9RDOA==">' + manager.json.gz.base64: '<19420 bytes, hash: "mbcqlRXTOcSVJbjutBrQZ4DzKqXCIiHtkD3jxA==">' + os.json.gz.base64: '<19788 bytes, hash: "UIjUH/4tsJHHOZZ+ZbVOopQ8ScMY8KkYIRb2ZQ==">' + overview.json.gz.base64: '<14204 bytes, hash: "dYi0WNJhWxsZNIf9ZChTqo9YRFiq4Zu2gNMAFQ==">' +kind: ConfigMap +metadata: + creationTimestamp: null + name: sm-name-grafana-scylladb-dashboards +`, "\n"), + expectedErr: nil, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + cm, _, err := makeGrafanaDashboards(tc.sm) + if !reflect.DeepEqual(err, tc.expectedErr) { + t.Errorf("expected and got errors differ:\n%s", cmp.Diff(tc.expectedErr, err)) + } + + // To avoid embedding thousands of lines we make a sum. + totalSize := 0 + for k, v := range cm.Data { + hash := sha512.Sum512_224([]byte(v)) + cm.Data[k] = fmt.Sprintf("<%d bytes, hash: %q>", len(v), base64.StdEncoding.EncodeToString(hash[:])) + totalSize += len(v) + } + + t.Logf("ConfigMap %q contains %d byte(s) of data", cm.Name, totalSize) + + objStringBytes, err := runtime.Encode(scheme.DefaultYamlSerializer, cm) + if err != nil { + t.Fatal(err) + } + + objString := string(objStringBytes) + if objString != tc.expectedString { + t.Errorf("expected and got strings differ:\n%s", cmp.Diff( + strings.Split(tc.expectedString, "\n"), + strings.Split(objString, "\n"), + )) + } + + // Make sure we don't outgrow max CM size. + if totalSize > corev1.MaxSecretSize { + t.Errorf("configmap size too large: size %d exceeded max size %d", totalSize, corev1.MaxSecretSize) + } + }) + } +} diff --git a/pkg/controller/scylladbmonitoring/sync_prometheus_test.go b/pkg/controller/scylladbmonitoring/sync_prometheus_test.go index 3cf6168fa3e..b6406200bc4 100644 --- a/pkg/controller/scylladbmonitoring/sync_prometheus_test.go +++ b/pkg/controller/scylladbmonitoring/sync_prometheus_test.go @@ -248,6 +248,7 @@ kind: Prometheus metadata: name: "sm-name" spec: + version: "v2.44.0" serviceAccountName: "sm-name-prometheus" securityContext: runAsNonRoot: true @@ -323,6 +324,7 @@ kind: Prometheus metadata: name: "sm-name" spec: + version: "v2.44.0" serviceAccountName: "sm-name-prometheus" securityContext: runAsNonRoot: true