Skip to content

Commit

Permalink
scylla-advanced: set the graph tooltip in desc order, add filter for …
Browse files Browse the repository at this point in the history 
…class and group
  • Loading branch information
@amnonh
amnonh committed Mar 1, 2023
1 parent 29e9b95 commit 2d77ccb
Showing 1 changed file with 85 additions and 12 deletions.
97 changes: 85 additions & 12 deletions grafana/scylla-advanced.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,15 @@
{
"expr": "max(rate(scylla_io_queue_total_delay_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])",
"intervalFactor": 1,
"legendFormat": "IO queue {{[[by]]}}",
"legendFormat": "",
"metric": "seastar_io_queue_delay",
"refId": "A",
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "$classes I/O Queue delay by [[by]]"
},
{
Expand All @@ -70,12 +73,15 @@
{
"expr": "max(scylla_io_queue_queue_length{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])",
"intervalFactor": 1,
"legendFormat": "queue length {{[[by]]}}",
"legendFormat": "",
"metric": "seastar_io_queue_delay",
"refId": "A",
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "$classes Queue length by [[by]]"
},
{
Expand All @@ -91,6 +97,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "$classes I/O Queue bandwidth by [[by]]"
},
{
Expand All @@ -106,6 +115,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "$classes I/O Queue IOPS by [[by]]"
},
{
Expand All @@ -115,12 +127,14 @@
{
"expr": "max(rate(scylla_io_queue_total_exec_sec{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])/rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]]) or on() max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])",
"intervalFactor": 1,
"legendFormat": "Disk {{[[by]]}}",
"metric": "scylla_io_queue_total_exec_sec",
"refId": "A",
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Disk $classes I/O Queue delay by [[by]]"
},
{
Expand All @@ -129,13 +143,15 @@
"targets": [
{
"expr": "max(scylla_io_queue_disk_queue_length{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])",
"intervalFactor": 1,
"legendFormat": "disk queue length {{[[by]]}}",
"intervalFactor": 1,
"metric": "scylla_io_queue_disk_queue_length",
"refId": "A",
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "DISK $classes Queue length by [[by]]"
}
],
Expand Down Expand Up @@ -197,6 +213,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Time used by [[by]] - $group",
"description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. This graph shows how much time was spent in $group group"
},
Expand All @@ -214,6 +233,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Time spent in task quota violations by [[by]] - $group",
"description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer. This may cause latency issues"
},
Expand All @@ -231,6 +253,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Scheduler shares [[by]] - $group",
"description": "Shares assigned to the $group. Shares determine how Scylla reactor distributes the task quotas between groups (Higher share gets more quotas)"
}
Expand Down Expand Up @@ -274,6 +299,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Local Reads Error by [[by]]",
"description": "Number of Read requests that failed due to an 'unavailable' error"
},
Expand All @@ -290,6 +318,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Local Write Error by [[by]]",
"description": "Number of write requests that failed due to an 'unavailable' error"
},
Expand Down Expand Up @@ -319,6 +350,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Reads Unavailable Error by [[by]]",
"description": "Number of Read requests that failed due to an 'unavailable' error"
},
Expand All @@ -335,6 +369,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Write Unavailable Error by [[by]]",
"description": "Number of write requests that failed on a local Node"
},
Expand All @@ -351,6 +388,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Range Unavailable Error by [[by]]",
"description": "Number of write requests that failed on a local Node"
}
Expand All @@ -373,6 +413,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "AIO Error by [[by]]",
"description": "Number of AIO Errors"
},
Expand All @@ -389,6 +432,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Ignored Future By [[by]]",
"description": "Total number of abandoned failed futures, futures destroyed while still containing an exception."
},
Expand All @@ -405,6 +451,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "C++ Exceptions [[by]]",
"description": "Number of C++ exceptions thrown.\n\n An exception by itself does not indicate a problem"
}
Expand Down Expand Up @@ -449,6 +498,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Avg reserved disk space by [[by]]",
"description": "Holds the size of disk space in bytes reserved for data so far. A too high value indicates that we have some bottleneck in the writing to sstables path"
},
Expand All @@ -465,6 +517,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Avg used disk space by [[by]]",
"description": "Holds the size of disk space in bytes used for data so far. A too high value indicates that we have some bottleneck in the writing to sstables path"
},
Expand All @@ -481,6 +536,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Avg flush by [[by]]",
"description": "Counts a number of times the flush() method was called for a file"
},
Expand All @@ -497,6 +555,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Segments by [[by]]",
"description": "Holds the current number of segments"
},
Expand All @@ -513,6 +574,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Avg flush limit exceeded by [[by]]",
"description": "Counts a number of times a flush limit was exceeded. A non-zero value indicates that there are too many pending flush operations (see pending_flushes) and some of them will be blocked till the total amount of pending flush operations drops below 5."
},
Expand All @@ -529,6 +593,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Pending allocations by [[by]]",
"description": "Holds the number of currently pending allocations. A non-zero value indicates that we have a bottleneck in the disk write flow."
},
Expand All @@ -545,6 +612,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Pending flush by [[by]]",
"description": "Counts a number of requests blocked due to memory pressure. A non-zero value indicates that the commitlog memory quota is not enough to serve the required amount of requests."
},
Expand All @@ -561,6 +631,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Unused segments by [[by]]",
"description": "Holds the current number of unused segments. A non-zero value indicates that the disk write path became temporary slow."
},
Expand All @@ -577,6 +650,9 @@
"step": 30
}
],
"options": {
"class":"desc_tooltip_options"
},
"title": "Allocating segments by [[by]]",
"description": "Holds the number of not closed segments that still have some free space. This value should not get too high."
}
Expand Down Expand Up @@ -663,20 +739,17 @@
"class": "template_variable_all",
"label": "classes",
"name": "classes",
"hide": 2,
"hide": 0,
"query": "label_values(scylla_io_queue_delay,class)",
"sort": 3
"sort": 1
},
{
"class": "template_variable_all",
"label": "group",
"name": "group",
"hide": 2,
"hide": 0,
"query": "label_values(scylla_scheduler_time_spent_on_task_quota_violations_ms,group)",
"sort": 3
},
{
"class": "adhoc_filter"
"sort": 1
},
{
"class": "aggregation_function",
Expand Down

0 comments on commit 2d77ccb

Please sign in to comment.